Lines Matching full:cgf
30 CodeGenFunction &CGF;
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
47 ASTContext &C = CGF.getContext();
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173 return CGF.CGM.getSize(size);
204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
291 Address TempAlloca = CGF.CreateMemTemp(
298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
303 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
332 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
334 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
350 CGF.Builder.CreateMemSet(
351 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
352 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
357 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
364 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
367 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
374 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
375 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
380 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
385 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
389 CGF
391 CGF.Builder.SetInsertPoint(StoreExpectedBB);
393 CGF.Builder.CreateStore(Old, Val1);
395 CGF.Builder.CreateBr(ContinueBB);
397 CGF.Builder.SetInsertPoint(ContinueBB);
399 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
405 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
437 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
445 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
448 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
450 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
452 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
454 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
461 CGF.Builder.SetInsertPoint(MonotonicBB);
462 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
464 CGF.Builder.CreateBr(ContBB);
467 CGF.Builder.SetInsertPoint(AcquireBB);
468 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
470 CGF.Builder.CreateBr(ContBB);
471 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
477 CGF.Builder.SetInsertPoint(SeqCstBB);
478 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
480 CGF.Builder.CreateBr(ContBB);
481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
485 CGF.Builder.SetInsertPoint(ContBB);
488 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
500 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
504 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
510 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
515 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
516 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
518 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
520 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
521 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
523 CGF.Builder.SetInsertPoint(StrongBB);
524 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
526 CGF.Builder.CreateBr(ContBB);
528 CGF.Builder.SetInsertPoint(WeakBB);
529 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
531 CGF.Builder.CreateBr(ContBB);
533 CGF.Builder.SetInsertPoint(ContBB);
540 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
543 CGF.Builder.CreateStore(Load, Dest);
550 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
551 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
611 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
613 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
620 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
622 Result = CGF.Builder.CreateNot(Result);
623 CGF.Builder.CreateStore(Result, Dest);
629 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
630 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
631 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
637 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
642 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
643 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
645 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
646 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
648 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
649 Val = CGF.EmitLoadOfScalar(Ptr, false,
650 CGF.getContext().getPointerType(ValTy),
656 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
657 CGF.getContext().VoidPtrTy);
1145 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1146 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1151 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1154 CGF.Builder.CreateMemCpy(Tmp, Addr,
1172 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1176 return CGF.convertTempToRValue(addr, getValueType(), loc);
1180 return RValue::get(CGF.Builder.CreateLoad(addr));
1182 return CGF.EmitLoadOfBitfieldLValue(
1186 return CGF.EmitLoadOfLValue(
1190 return CGF
1207 ? CGF.ConvertTypeForMem(ValueTy)
1211 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1213 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1215 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1232 CGF.Builder.CreateStore(IntVal, CastTemp)
1242 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1243 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1244 CGF.getContext().VoidPtrTy);
1245 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1246 CGF.getContext().VoidPtrTy);
1248 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1249 CGF.getContext().IntTy);
1250 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1257 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1264 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1346 CGF.EmitAggregateCopy(getAtomicAddress(),
1364 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1366 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1380 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1381 AtomicInfo Atomics(CGF, TempLV);
1392 return CGF.EmitToMemory(Value, ValueTy);
1395 CGF.getLLVMContext(),
1398 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1400 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1409 return CGF.Builder.CreateLoad(Addr);
1417 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1425 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1426 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1438 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1439 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1440 CGF.getContext().VoidPtrTy);
1441 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1442 CGF.getContext().VoidPtrTy);
1443 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1444 CGF.getContext().VoidPtrTy);
1446 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1447 CGF.getContext().IntTy);
1449 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1450 CGF.getContext().IntTy);
1451 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1452 CGF.getContext().BoolTy, Args);
1492 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1500 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1532 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1537 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1540 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1553 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1554 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1555 CGF.EmitBlock(ContBB);
1559 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1560 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1565 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1570 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1571 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1582 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1583 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1584 auto *CurBB = CGF.Builder.GetInsertBlock();
1585 CGF.EmitBlock(ContBB);
1586 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1593 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1597 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1598 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1601 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1602 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1603 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1630 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1640 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1641 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1642 CGF.EmitBlock(ContBB);
1646 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1647 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1649 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1654 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1655 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1665 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1666 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1667 auto *CurBB = CGF.Builder.GetInsertBlock();
1668 CGF.EmitBlock(ContBB);
1669 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1676 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1678 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1679 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1682 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1683 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1684 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);