Lines Matching refs:Builder
61 insertRMWLLSCLoop(IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
74 IRBuilder<> &Builder, Type *ResultType, Value *Addr,
325 IRBuilder<> Builder(I);
327 auto LeadingFence = TLI->emitLeadingFence(Builder, Order, IsStore, IsLoad);
329 auto TrailingFence = TLI->emitTrailingFence(Builder, Order, IsStore, IsLoad);
331 // because there is no easy way of setting Builder insertion point after
361 IRBuilder<> Builder(LI);
366 Value *NewAddr = Builder.CreateBitCast(Addr, PT);
368 auto *NewLI = Builder.CreateLoad(NewAddr);
374 Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType());
387 [](IRBuilder<> &Builder, Value *Loaded) { return Loaded; });
398 IRBuilder<> Builder(LI);
404 TLI->emitLoadLinked(Builder, LI->getPointerOperand(), LI->getOrdering());
405 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
414 IRBuilder<> Builder(LI);
420 Value *Pair = Builder.CreateAtomicCmpXchg(
423 Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");
440 IRBuilder<> Builder(SI);
444 Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy);
449 Value *NewAddr = Builder.CreateBitCast(Addr, PT);
451 StoreInst *NewSI = Builder.CreateStore(NewVal, NewAddr);
467 IRBuilder<> Builder(SI);
469 Builder.CreateAtomicRMW(AtomicRMWInst::Xchg, SI->getPointerOperand(),
477 static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
481 Value* Pair = Builder.CreateAtomicCmpXchg(
484 Success = Builder.CreateExtractValue(Pair, 1, "success");
485 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
490 static Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder,
497 return Builder.CreateAdd(Loaded, Inc, "new");
499 return Builder.CreateSub(Loaded, Inc, "new");
501 return Builder.CreateAnd(Loaded, Inc, "new");
503 return Builder.CreateNot(Builder.CreateAnd(Loaded, Inc), "new");
505 return Builder.CreateOr(Loaded, Inc, "new");
507 return Builder.CreateXor(Loaded, Inc, "new");
509 NewVal = Builder.CreateICmpSGT(Loaded, Inc);
510 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
512 NewVal = Builder.CreateICmpSLE(Loaded, Inc);
513 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
515 NewVal = Builder.CreateICmpUGT(Loaded, Inc);
516 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
518 NewVal = Builder.CreateICmpULE(Loaded, Inc);
519 return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
536 auto PerformOp = [&](IRBuilder<> &Builder, Value *Loaded) {
537 return performAtomicOp(AI->getOperation(), Builder, Loaded,
591 static PartwordMaskValues createMaskInstrs(IRBuilder<> &Builder, Instruction *I,
613 Value *AddrInt = Builder.CreatePtrToInt(Addr, DL.getIntPtrType(Ctx));
614 Ret.AlignedAddr = Builder.CreateIntToPtr(
615 Builder.CreateAnd(AddrInt, ~(uint64_t)(WordSize - 1)), WordPtrType,
618 Value *PtrLSB = Builder.CreateAnd(AddrInt, WordSize - 1, "PtrLSB");
621 Ret.ShiftAmt = Builder.CreateShl(PtrLSB, 3);
625 Builder.CreateShl(Builder.CreateXor(PtrLSB, WordSize - ValueSize), 3);
628 Ret.ShiftAmt = Builder.CreateTrunc(Ret.ShiftAmt, Ret.WordType, "ShiftAmt");
629 Ret.Mask = Builder.CreateShl(
632 Ret.Inv_Mask = Builder.CreateNot(Ret.Mask, "Inv_Mask");
641 IRBuilder<> &Builder, Value *Loaded,
646 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
647 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, Shifted_Inc);
654 return performAtomicOp(Op, Builder, Loaded, Shifted_Inc);
660 Value *NewVal = performAtomicOp(Op, Builder, Loaded, Shifted_Inc);
661 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);
662 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
663 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);
673 Value *Loaded_Shiftdown = Builder.CreateTrunc(
674 Builder.CreateLShr(Loaded, PMV.ShiftAmt), PMV.ValueType);
675 Value *NewVal = performAtomicOp(Op, Builder, Loaded_Shiftdown, Inc);
676 Value *NewVal_Shiftup = Builder.CreateShl(
677 Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
678 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);
679 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shiftup);
701 IRBuilder<> Builder(AI);
704 createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
708 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
711 auto PerformPartwordOp = [&](IRBuilder<> &Builder, Value *Loaded) {
712 return performMaskedAtomicOp(AI->getOperation(), Builder, Loaded,
719 insertRMWCmpXchgLoop(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder,
721 Value *FinalOldResult = Builder.CreateTrunc(
722 Builder.CreateLShr(OldResult, PMV.ShiftAmt), PMV.ValueType);
769 IRBuilder<> Builder(CI);
770 LLVMContext &Ctx = Builder.getContext();
783 Builder.SetInsertPoint(BB);
786 Builder, CI, CI->getCompareOperand()->getType(), Addr, WordSize);
790 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);
792 Builder.CreateShl(Builder.CreateZExt(Cmp, PMV.WordType), PMV.ShiftAmt);
796 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);
798 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);
799 Builder.CreateBr(LoopBB);
802 Builder.SetInsertPoint(LoopBB);
803 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);
807 Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);
808 Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);
809 AtomicCmpXchgInst *NewCI = Builder.CreateAtomicCmpXchg(
820 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
821 Builder.CreateExtractValue(NewCI, 1);
824 Builder.CreateBr(EndBB);
826 Builder.CreateCondBr(Success, EndBB, FailureBB);
829 Builder.SetInsertPoint(FailureBB);
833 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);
834 Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut);
835 Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB);
841 Builder.SetInsertPoint(CI);
843 Value *FinalOldVal = Builder.CreateTrunc(
844 Builder.CreateLShr(OldVal, PMV.ShiftAmt), PMV.ValueType);
846 Res = Builder.CreateInsertValue(Res, FinalOldVal, 0);
847 Res = Builder.CreateInsertValue(Res, Success, 1);
856 IRBuilder<> Builder(I);
858 insertRMWLLSCLoop(Builder, ResultType, Addr, MemOpOrder, PerformOp);
865 IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
868 LLVMContext &Ctx = Builder.getContext();
869 BasicBlock *BB = Builder.GetInsertBlock();
885 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
891 Builder.SetInsertPoint(BB);
892 Builder.CreateBr(LoopBB);
895 Builder.SetInsertPoint(LoopBB);
896 Value *Loaded = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
898 Value *NewVal = PerformOp(Builder, Loaded);
901 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);
902 Value *TryAgain = Builder.CreateICmpNE(
904 Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);
906 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
920 IRBuilder<> Builder(CI);
925 Value *NewAddr = Builder.CreateBitCast(Addr, PT);
927 Value *NewCmp = Builder.CreatePtrToInt(CI->getCompareOperand(), NewTy);
928 Value *NewNewVal = Builder.CreatePtrToInt(CI->getNewValOperand(), NewTy);
931 auto *NewCI = Builder.CreateAtomicCmpXchg(NewAddr, NewCmp, NewNewVal,
939 Value *OldVal = Builder.CreateExtractValue(NewCI, 0);
940 Value *Succ = Builder.CreateExtractValue(NewCI, 1);
942 OldVal = Builder.CreateIntToPtr(OldVal, CI->getCompareOperand()->getType());
945 Res = Builder.CreateInsertValue(Res, OldVal, 0);
946 Res = Builder.CreateInsertValue(Res, Succ, 1);
1043 IRBuilder<> Builder(CI);
1049 Builder.SetInsertPoint(BB);
1051 TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true,
1053 Builder.CreateBr(StartBB);
1056 Builder.SetInsertPoint(StartBB);
1057 Value *UnreleasedLoad = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
1058 Value *ShouldStore = Builder.CreateICmpEQ(
1063 Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);
1065 Builder.SetInsertPoint(ReleasingStoreBB);
1067 TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true,
1069 Builder.CreateBr(TryStoreBB);
1071 Builder.SetInsertPoint(TryStoreBB);
1073 Builder, CI->getNewValOperand(), Addr, MemOpOrder);
1074 StoreSuccess = Builder.CreateICmpEQ(
1077 Builder.CreateCondBr(StoreSuccess, SuccessBB,
1080 Builder.SetInsertPoint(ReleasedLoadBB);
1083 SecondLoad = TLI->emitLoadLinked(Builder, Addr, MemOpOrder);
1084 ShouldStore = Builder.CreateICmpEQ(SecondLoad, CI->getCompareOperand(),
1089 Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB);
1091 Builder.CreateUnreachable();
1095 Builder.SetInsertPoint(SuccessBB);
1097 TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true,
1099 Builder.CreateBr(ExitBB);
1101 Builder.SetInsertPoint(NoStoreBB);
1105 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);
1106 Builder.CreateBr(FailureBB);
1108 Builder.SetInsertPoint(FailureBB);
1110 TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true,
1112 Builder.CreateBr(ExitBB);
1118 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1119 PHINode *Success = Builder.CreatePHI(Type::getInt1Ty(Ctx), 2);
1123 // Setup the builder so we can create any PHIs we need.
1128 Builder.SetInsertPoint(TryStoreBB, TryStoreBB->begin());
1129 PHINode *TryStoreLoaded = Builder.CreatePHI(UnreleasedLoad->getType(), 2);
1133 Builder.SetInsertPoint(NoStoreBB, NoStoreBB->begin());
1134 PHINode *NoStoreLoaded = Builder.CreatePHI(UnreleasedLoad->getType(), 2);
1138 Builder.SetInsertPoint(ExitBB, ++ExitBB->begin());
1139 PHINode *ExitLoaded = Builder.CreatePHI(UnreleasedLoad->getType(), 2);
1173 Res = Builder.CreateInsertValue(UndefValue::get(CI->getType()), Loaded, 0);
1174 Res = Builder.CreateInsertValue(Res, Success, 1);
1212 IRBuilder<> &Builder, Type *ResultTy, Value *Addr,
1216 LLVMContext &Ctx = Builder.getContext();
1217 BasicBlock *BB = Builder.GetInsertBlock();
1236 BB->splitBasicBlock(Builder.GetInsertPoint(), "atomicrmw.end");
1243 Builder.SetInsertPoint(BB);
1244 LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr);
1247 Builder.CreateBr(LoopBB);
1250 Builder.SetInsertPoint(LoopBB);
1251 PHINode *Loaded = Builder.CreatePHI(ResultTy, 2, "loaded");
1254 Value *NewVal = PerformOp(Builder, Loaded);
1259 CreateCmpXchg(Builder, Addr, Loaded, NewVal,
1268 Builder.CreateCondBr(Success, ExitBB, LoopBB);
1270 Builder.SetInsertPoint(ExitBB, ExitBB->begin());
1277 IRBuilder<> Builder(AI);
1279 Builder, AI->getType(), AI->getPointerOperand(), AI->getOrdering(),
1280 [&](IRBuilder<> &Builder, Value *Loaded) {
1281 return performAtomicOp(AI->getOperation(), Builder, Loaded,
1429 expandAtomicRMWToCmpXchg(I, [this](IRBuilder<> &Builder, Value *Addr,
1434 AtomicCmpXchgInst *Pair = Builder.CreateAtomicCmpXchg(
1437 Success = Builder.CreateExtractValue(Pair, 1, "success");
1438 NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
1461 IRBuilder<> Builder(I);
1545 Builder.CreateBitCast(PointerOperand, Type::getInt8PtrTy(Ctx));
1553 Builder.CreateBitCast(AllocaCASExpected, Type::getInt8PtrTy(Ctx));
1554 Builder.CreateLifetimeStart(AllocaCASExpected_i8, SizeVal64);
1555 Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment);
1563 Builder.CreateBitOrPointerCast(ValueOperand, SizedIntTy);
1569 Builder.CreateBitCast(AllocaValue, Type::getInt8PtrTy(Ctx));
1570 Builder.CreateLifetimeStart(AllocaValue_i8, SizeVal64);
1571 Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment);
1581 Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx));
1582 Builder.CreateLifetimeStart(AllocaResult_i8, SizeVal64);
1609 CallInst *Call = Builder.CreateCall(LibcallFn, Args);
1615 Builder.CreateLifetimeEnd(AllocaValue_i8, SizeVal64);
1623 Builder.CreateAlignedLoad(AllocaCASExpected, AllocaAlignment);
1624 Builder.CreateLifetimeEnd(AllocaCASExpected_i8, SizeVal64);
1625 V = Builder.CreateInsertValue(V, ExpectedOut, 0);
1626 V = Builder.CreateInsertValue(V, Result, 1);
1631 V = Builder.CreateBitOrPointerCast(Result, I->getType());
1633 V = Builder.CreateAlignedLoad(AllocaResult, AllocaAlignment);
1634 Builder.CreateLifetimeEnd(AllocaResult_i8, SizeVal64);
