1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the code for emitting atomic operations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenFunction.h" 15 #include "CGCall.h" 16 #include "CGRecordLayout.h" 17 #include "CodeGenModule.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "llvm/ADT/StringExtras.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/Intrinsics.h" 23 #include "llvm/IR/Operator.h" 24 25 using namespace clang; 26 using namespace CodeGen; 27 28 namespace { 29 class AtomicInfo { 30 CodeGenFunction &CGF; 31 QualType AtomicTy; 32 QualType ValueTy; 33 uint64_t AtomicSizeInBits; 34 uint64_t ValueSizeInBits; 35 CharUnits AtomicAlign; 36 CharUnits ValueAlign; 37 CharUnits LValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 88 VoidPtrAddr, OffsetInChars.getQuantity()); 89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 90 VoidPtrAddr, 91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 92 "atomic_bitfield_base"); 93 BFI = OrigBFI; 94 BFI.Offset = Offset; 95 BFI.StorageSize = AtomicSizeInBits; 96 BFI.StorageOffset += OffsetInChars; 97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 98 BFI, lvalue.getType(), 99 lvalue.getAlignmentSource()); 100 LVal.setTBAAInfo(lvalue.getTBAAInfo()); 101 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 102 if (AtomicTy.isNull()) { 103 llvm::APInt Size( 104 /*numBits=*/32, 105 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 106 AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 107 /*IndexTypeQuals=*/0); 108 } 109 AtomicAlign = ValueAlign = lvalue.getAlignment(); 110 } else if (lvalue.isVectorElt()) { 111 ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 112 ValueSizeInBits = C.getTypeSize(ValueTy); 113 AtomicTy = lvalue.getType(); 114 AtomicSizeInBits = C.getTypeSize(AtomicTy); 115 AtomicAlign = ValueAlign = lvalue.getAlignment(); 116 LVal = lvalue; 117 } else { 118 assert(lvalue.isExtVectorElt()); 119 ValueTy = lvalue.getType(); 120 ValueSizeInBits = C.getTypeSize(ValueTy); 121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 122 lvalue.getType(), lvalue.getExtVectorAddress() 123 .getElementType()->getVectorNumElements()); 124 AtomicSizeInBits = C.getTypeSize(AtomicTy); 125 AtomicAlign = ValueAlign = lvalue.getAlignment(); 126 LVal = lvalue; 127 } 128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 129 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 130 } 131 132 QualType getAtomicType() const { return AtomicTy; } 133 QualType getValueType() const { return ValueTy; } 134 CharUnits getAtomicAlignment() const { return AtomicAlign; } 135 CharUnits getValueAlignment() const { return ValueAlign; } 136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 139 bool shouldUseLibcall() const { return UseLibcall; } 140 const LValue &getAtomicLValue() const { return LVal; } 141 llvm::Value *getAtomicPointer() const { 142 if (LVal.isSimple()) 143 return LVal.getPointer(); 144 else if (LVal.isBitField()) 145 return LVal.getBitFieldPointer(); 146 else if (LVal.isVectorElt()) 147 return LVal.getVectorPointer(); 148 assert(LVal.isExtVectorElt()); 149 return LVal.getExtVectorPointer(); 150 } 151 Address getAtomicAddress() const { 152 return Address(getAtomicPointer(), getAtomicAlignment()); 153 } 154 155 Address getAtomicAddressAsAtomicIntPointer() const { 156 return emitCastToAtomicIntPointer(getAtomicAddress()); 157 } 158 159 /// Is the atomic size larger than the underlying value type? 160 /// 161 /// Note that the absence of padding does not mean that atomic 162 /// objects are completely interchangeable with non-atomic 163 /// objects: we might have promoted the alignment of a type 164 /// without making it bigger. 165 bool hasPadding() const { 166 return (ValueSizeInBits != AtomicSizeInBits); 167 } 168 169 bool emitMemSetZeroIfNecessary() const; 170 171 llvm::Value *getAtomicSizeValue() const { 172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 173 return CGF.CGM.getSize(size); 174 } 175 176 /// Cast the given pointer to an integer pointer suitable for atomic 177 /// operations if the source. 178 Address emitCastToAtomicIntPointer(Address Addr) const; 179 180 /// If Addr is compatible with the iN that will be used for an atomic 181 /// operation, bitcast it. Otherwise, create a temporary that is suitable 182 /// and copy the value across. 183 Address convertToAtomicIntPointer(Address Addr) const; 184 185 /// Turn an atomic-layout object into an r-value. 186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 187 SourceLocation loc, bool AsValue) const; 188 189 /// \brief Converts a rvalue to integer value. 190 llvm::Value *convertRValueToInt(RValue RVal) const; 191 192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 193 AggValueSlot ResultSlot, 194 SourceLocation Loc, bool AsValue) const; 195 196 /// Copy an atomic r-value into atomic-layout memory. 197 void emitCopyIntoMemory(RValue rvalue) const; 198 199 /// Project an l-value down to the value field. 200 LValue projectValue() const { 201 assert(LVal.isSimple()); 202 Address addr = getAtomicAddress(); 203 if (hasPadding()) 204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 205 206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 207 LVal.getAlignmentSource(), LVal.getTBAAInfo()); 208 } 209 210 /// \brief Emits atomic load. 211 /// \returns Loaded value. 212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 213 bool AsValue, llvm::AtomicOrdering AO, 214 bool IsVolatile); 215 216 /// \brief Emits atomic compare-and-exchange sequence. 217 /// \param Expected Expected value. 218 /// \param Desired Desired value. 219 /// \param Success Atomic ordering for success operation. 220 /// \param Failure Atomic ordering for failed operation. 221 /// \param IsWeak true if atomic operation is weak, false otherwise. 222 /// \returns Pair of values: previous value from storage (value type) and 223 /// boolean flag (i1 type) with true if success and false otherwise. 224 std::pair<RValue, llvm::Value *> 225 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 226 llvm::AtomicOrdering Success = 227 llvm::AtomicOrdering::SequentiallyConsistent, 228 llvm::AtomicOrdering Failure = 229 llvm::AtomicOrdering::SequentiallyConsistent, 230 bool IsWeak = false); 231 232 /// \brief Emits atomic update. 233 /// \param AO Atomic ordering. 234 /// \param UpdateOp Update operation for the current lvalue. 235 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 236 const llvm::function_ref<RValue(RValue)> &UpdateOp, 237 bool IsVolatile); 238 /// \brief Emits atomic update. 239 /// \param AO Atomic ordering. 240 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 241 bool IsVolatile); 242 243 /// Materialize an atomic r-value in atomic-layout memory. 244 Address materializeRValue(RValue rvalue) const; 245 246 /// \brief Creates temp alloca for intermediate operations on atomic value. 247 Address CreateTempAlloca() const; 248 private: 249 bool requiresMemSetZero(llvm::Type *type) const; 250 251 252 /// \brief Emits atomic load as a libcall. 253 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 254 llvm::AtomicOrdering AO, bool IsVolatile); 255 /// \brief Emits atomic load as LLVM instruction. 256 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 257 /// \brief Emits atomic compare-and-exchange op as a libcall. 258 llvm::Value *EmitAtomicCompareExchangeLibcall( 259 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 260 llvm::AtomicOrdering Success = 261 llvm::AtomicOrdering::SequentiallyConsistent, 262 llvm::AtomicOrdering Failure = 263 llvm::AtomicOrdering::SequentiallyConsistent); 264 /// \brief Emits atomic compare-and-exchange op as LLVM instruction. 265 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 266 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 267 llvm::AtomicOrdering Success = 268 llvm::AtomicOrdering::SequentiallyConsistent, 269 llvm::AtomicOrdering Failure = 270 llvm::AtomicOrdering::SequentiallyConsistent, 271 bool IsWeak = false); 272 /// \brief Emit atomic update as libcalls. 273 void 274 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 275 const llvm::function_ref<RValue(RValue)> &UpdateOp, 276 bool IsVolatile); 277 /// \brief Emit atomic update as LLVM instructions. 278 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 279 const llvm::function_ref<RValue(RValue)> &UpdateOp, 280 bool IsVolatile); 281 /// \brief Emit atomic update as libcalls. 282 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 283 bool IsVolatile); 284 /// \brief Emit atomic update as LLVM instructions. 285 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 286 bool IsVolatile); 287 }; 288 } 289 290 Address AtomicInfo::CreateTempAlloca() const { 291 Address TempAlloca = CGF.CreateMemTemp( 292 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 293 : AtomicTy, 294 getAtomicAlignment(), 295 "atomic-temp"); 296 // Cast to pointer to value type for bitfields. 297 if (LVal.isBitField()) 298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 299 TempAlloca, getAtomicAddress().getType()); 300 return TempAlloca; 301 } 302 303 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 304 StringRef fnName, 305 QualType resultType, 306 CallArgList &args) { 307 const CGFunctionInfo &fnInfo = 308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 310 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 311 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); 312 } 313 314 /// Does a store of the given IR type modify the full expected width? 315 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 316 uint64_t expectedSize) { 317 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 318 } 319 320 /// Does the atomic type require memsetting to zero before initialization? 321 /// 322 /// The IR type is provided as a way of making certain queries faster. 323 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 324 // If the atomic type has size padding, we definitely need a memset. 325 if (hasPadding()) return true; 326 327 // Otherwise, do some simple heuristics to try to avoid it: 328 switch (getEvaluationKind()) { 329 // For scalars and complexes, check whether the store size of the 330 // type uses the full size. 331 case TEK_Scalar: 332 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 333 case TEK_Complex: 334 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 335 AtomicSizeInBits / 2); 336 337 // Padding in structs has an undefined bit pattern. User beware. 338 case TEK_Aggregate: 339 return false; 340 } 341 llvm_unreachable("bad evaluation kind"); 342 } 343 344 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 345 assert(LVal.isSimple()); 346 llvm::Value *addr = LVal.getPointer(); 347 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 348 return false; 349 350 CGF.Builder.CreateMemSet( 351 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 352 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 353 LVal.getAlignment().getQuantity()); 354 return true; 355 } 356 357 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 358 Address Dest, Address Ptr, 359 Address Val1, Address Val2, 360 uint64_t Size, 361 llvm::AtomicOrdering SuccessOrder, 362 llvm::AtomicOrdering FailureOrder) { 363 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 364 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 365 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 366 367 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 368 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder); 369 Pair->setVolatile(E->isVolatile()); 370 Pair->setWeak(IsWeak); 371 372 // Cmp holds the result of the compare-exchange operation: true on success, 373 // false on failure. 374 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 375 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 376 377 // This basic block is used to hold the store instruction if the operation 378 // failed. 379 llvm::BasicBlock *StoreExpectedBB = 380 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 381 382 // This basic block is the exit point of the operation, we should end up 383 // here regardless of whether or not the operation succeeded. 384 llvm::BasicBlock *ContinueBB = 385 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 386 387 // Update Expected if Expected isn't equal to Old, otherwise branch to the 388 // exit point. 389 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 390 391 CGF.Builder.SetInsertPoint(StoreExpectedBB); 392 // Update the memory at Expected with Old's value. 393 CGF.Builder.CreateStore(Old, Val1); 394 // Finally, branch to the exit point. 395 CGF.Builder.CreateBr(ContinueBB); 396 397 CGF.Builder.SetInsertPoint(ContinueBB); 398 // Update the memory at Dest with Cmp's value. 399 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 400 } 401 402 /// Given an ordering required on success, emit all possible cmpxchg 403 /// instructions to cope with the provided (but possibly only dynamically known) 404 /// FailureOrder. 405 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 406 bool IsWeak, Address Dest, Address Ptr, 407 Address Val1, Address Val2, 408 llvm::Value *FailureOrderVal, 409 uint64_t Size, 410 llvm::AtomicOrdering SuccessOrder) { 411 llvm::AtomicOrdering FailureOrder; 412 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 413 auto FOS = FO->getSExtValue(); 414 if (!llvm::isValidAtomicOrderingCABI(FOS)) 415 FailureOrder = llvm::AtomicOrdering::Monotonic; 416 else 417 switch ((llvm::AtomicOrderingCABI)FOS) { 418 case llvm::AtomicOrderingCABI::relaxed: 419 case llvm::AtomicOrderingCABI::release: 420 case llvm::AtomicOrderingCABI::acq_rel: 421 FailureOrder = llvm::AtomicOrdering::Monotonic; 422 break; 423 case llvm::AtomicOrderingCABI::consume: 424 case llvm::AtomicOrderingCABI::acquire: 425 FailureOrder = llvm::AtomicOrdering::Acquire; 426 break; 427 case llvm::AtomicOrderingCABI::seq_cst: 428 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 429 break; 430 } 431 if (isStrongerThan(FailureOrder, SuccessOrder)) { 432 // Don't assert on undefined behavior "failure argument shall be no 433 // stronger than the success argument". 434 FailureOrder = 435 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 436 } 437 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 438 FailureOrder); 439 return; 440 } 441 442 // Create all the relevant BB's 443 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 444 *SeqCstBB = nullptr; 445 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 446 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 447 SuccessOrder != llvm::AtomicOrdering::Release) 448 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 449 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 450 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 451 452 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 453 454 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 455 456 // Emit all the different atomics 457 458 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 459 // doesn't matter unless someone is crazy enough to use something that 460 // doesn't fold to a constant for the ordering. 461 CGF.Builder.SetInsertPoint(MonotonicBB); 462 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 463 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic); 464 CGF.Builder.CreateBr(ContBB); 465 466 if (AcquireBB) { 467 CGF.Builder.SetInsertPoint(AcquireBB); 468 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 469 Size, SuccessOrder, llvm::AtomicOrdering::Acquire); 470 CGF.Builder.CreateBr(ContBB); 471 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 472 AcquireBB); 473 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 474 AcquireBB); 475 } 476 if (SeqCstBB) { 477 CGF.Builder.SetInsertPoint(SeqCstBB); 478 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 479 llvm::AtomicOrdering::SequentiallyConsistent); 480 CGF.Builder.CreateBr(ContBB); 481 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 482 SeqCstBB); 483 } 484 485 CGF.Builder.SetInsertPoint(ContBB); 486 } 487 488 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 489 Address Ptr, Address Val1, Address Val2, 490 llvm::Value *IsWeak, llvm::Value *FailureOrder, 491 uint64_t Size, llvm::AtomicOrdering Order) { 492 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 493 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 494 495 switch (E->getOp()) { 496 case AtomicExpr::AO__c11_atomic_init: 497 llvm_unreachable("Already handled!"); 498 499 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 500 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 501 FailureOrder, Size, Order); 502 return; 503 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 504 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 505 FailureOrder, Size, Order); 506 return; 507 case AtomicExpr::AO__atomic_compare_exchange: 508 case AtomicExpr::AO__atomic_compare_exchange_n: { 509 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 510 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 511 Val1, Val2, FailureOrder, Size, Order); 512 } else { 513 // Create all the relevant BB's 514 llvm::BasicBlock *StrongBB = 515 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 516 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 517 llvm::BasicBlock *ContBB = 518 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 519 520 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 521 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 522 523 CGF.Builder.SetInsertPoint(StrongBB); 524 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 525 FailureOrder, Size, Order); 526 CGF.Builder.CreateBr(ContBB); 527 528 CGF.Builder.SetInsertPoint(WeakBB); 529 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 530 FailureOrder, Size, Order); 531 CGF.Builder.CreateBr(ContBB); 532 533 CGF.Builder.SetInsertPoint(ContBB); 534 } 535 return; 536 } 537 case AtomicExpr::AO__c11_atomic_load: 538 case AtomicExpr::AO__atomic_load_n: 539 case AtomicExpr::AO__atomic_load: { 540 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 541 Load->setAtomic(Order); 542 Load->setVolatile(E->isVolatile()); 543 CGF.Builder.CreateStore(Load, Dest); 544 return; 545 } 546 547 case AtomicExpr::AO__c11_atomic_store: 548 case AtomicExpr::AO__atomic_store: 549 case AtomicExpr::AO__atomic_store_n: { 550 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 551 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 552 Store->setAtomic(Order); 553 Store->setVolatile(E->isVolatile()); 554 return; 555 } 556 557 case AtomicExpr::AO__c11_atomic_exchange: 558 case AtomicExpr::AO__atomic_exchange_n: 559 case AtomicExpr::AO__atomic_exchange: 560 Op = llvm::AtomicRMWInst::Xchg; 561 break; 562 563 case AtomicExpr::AO__atomic_add_fetch: 564 PostOp = llvm::Instruction::Add; 565 // Fall through. 566 case AtomicExpr::AO__c11_atomic_fetch_add: 567 case AtomicExpr::AO__atomic_fetch_add: 568 Op = llvm::AtomicRMWInst::Add; 569 break; 570 571 case AtomicExpr::AO__atomic_sub_fetch: 572 PostOp = llvm::Instruction::Sub; 573 // Fall through. 574 case AtomicExpr::AO__c11_atomic_fetch_sub: 575 case AtomicExpr::AO__atomic_fetch_sub: 576 Op = llvm::AtomicRMWInst::Sub; 577 break; 578 579 case AtomicExpr::AO__atomic_and_fetch: 580 PostOp = llvm::Instruction::And; 581 // Fall through. 582 case AtomicExpr::AO__c11_atomic_fetch_and: 583 case AtomicExpr::AO__atomic_fetch_and: 584 Op = llvm::AtomicRMWInst::And; 585 break; 586 587 case AtomicExpr::AO__atomic_or_fetch: 588 PostOp = llvm::Instruction::Or; 589 // Fall through. 590 case AtomicExpr::AO__c11_atomic_fetch_or: 591 case AtomicExpr::AO__atomic_fetch_or: 592 Op = llvm::AtomicRMWInst::Or; 593 break; 594 595 case AtomicExpr::AO__atomic_xor_fetch: 596 PostOp = llvm::Instruction::Xor; 597 // Fall through. 598 case AtomicExpr::AO__c11_atomic_fetch_xor: 599 case AtomicExpr::AO__atomic_fetch_xor: 600 Op = llvm::AtomicRMWInst::Xor; 601 break; 602 603 case AtomicExpr::AO__atomic_nand_fetch: 604 PostOp = llvm::Instruction::And; // the NOT is special cased below 605 // Fall through. 606 case AtomicExpr::AO__atomic_fetch_nand: 607 Op = llvm::AtomicRMWInst::Nand; 608 break; 609 } 610 611 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 612 llvm::AtomicRMWInst *RMWI = 613 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order); 614 RMWI->setVolatile(E->isVolatile()); 615 616 // For __atomic_*_fetch operations, perform the operation again to 617 // determine the value which was written. 618 llvm::Value *Result = RMWI; 619 if (PostOp) 620 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 621 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 622 Result = CGF.Builder.CreateNot(Result); 623 CGF.Builder.CreateStore(Result, Dest); 624 } 625 626 // This function emits any expression (scalar, complex, or aggregate) 627 // into a temporary alloca. 628 static Address 629 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 630 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 631 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 632 /*Init*/ true); 633 return DeclPtr; 634 } 635 636 static void 637 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 638 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 639 SourceLocation Loc, CharUnits SizeInChars) { 640 if (UseOptimizedLibcall) { 641 // Load value and pass it to the function directly. 642 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 643 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 644 ValTy = 645 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 646 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 647 SizeInBits)->getPointerTo(); 648 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 649 Val = CGF.EmitLoadOfScalar(Ptr, false, 650 CGF.getContext().getPointerType(ValTy), 651 Loc); 652 // Coerce the value into an appropriately sized integer type. 653 Args.add(RValue::get(Val), ValTy); 654 } else { 655 // Non-optimized functions always take a reference. 656 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 657 CGF.getContext().VoidPtrTy); 658 } 659 } 660 661 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 662 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 663 QualType MemTy = AtomicTy; 664 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 665 MemTy = AT->getValueType(); 666 CharUnits sizeChars, alignChars; 667 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 668 uint64_t Size = sizeChars.getQuantity(); 669 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 670 bool UseLibcall = (sizeChars != alignChars || 671 getContext().toBits(sizeChars) > MaxInlineWidthInBits); 672 673 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 674 675 Address Val1 = Address::invalid(); 676 Address Val2 = Address::invalid(); 677 Address Dest = Address::invalid(); 678 Address Ptr(EmitScalarExpr(E->getPtr()), alignChars); 679 680 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { 681 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 682 EmitAtomicInit(E->getVal1(), lvalue); 683 return RValue::get(nullptr); 684 } 685 686 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 687 688 switch (E->getOp()) { 689 case AtomicExpr::AO__c11_atomic_init: 690 llvm_unreachable("Already handled above with EmitAtomicInit!"); 691 692 case AtomicExpr::AO__c11_atomic_load: 693 case AtomicExpr::AO__atomic_load_n: 694 break; 695 696 case AtomicExpr::AO__atomic_load: 697 Dest = EmitPointerWithAlignment(E->getVal1()); 698 break; 699 700 case AtomicExpr::AO__atomic_store: 701 Val1 = EmitPointerWithAlignment(E->getVal1()); 702 break; 703 704 case AtomicExpr::AO__atomic_exchange: 705 Val1 = EmitPointerWithAlignment(E->getVal1()); 706 Dest = EmitPointerWithAlignment(E->getVal2()); 707 break; 708 709 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 710 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 711 case AtomicExpr::AO__atomic_compare_exchange_n: 712 case AtomicExpr::AO__atomic_compare_exchange: 713 Val1 = EmitPointerWithAlignment(E->getVal1()); 714 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 715 Val2 = EmitPointerWithAlignment(E->getVal2()); 716 else 717 Val2 = EmitValToTemp(*this, E->getVal2()); 718 OrderFail = EmitScalarExpr(E->getOrderFail()); 719 if (E->getNumSubExprs() == 6) 720 IsWeak = EmitScalarExpr(E->getWeak()); 721 break; 722 723 case AtomicExpr::AO__c11_atomic_fetch_add: 724 case AtomicExpr::AO__c11_atomic_fetch_sub: 725 if (MemTy->isPointerType()) { 726 // For pointer arithmetic, we're required to do a bit of math: 727 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 728 // ... but only for the C11 builtins. The GNU builtins expect the 729 // user to multiply by sizeof(T). 730 QualType Val1Ty = E->getVal1()->getType(); 731 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 732 CharUnits PointeeIncAmt = 733 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 734 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 735 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 736 Val1 = Temp; 737 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 738 break; 739 } 740 // Fall through. 741 case AtomicExpr::AO__atomic_fetch_add: 742 case AtomicExpr::AO__atomic_fetch_sub: 743 case AtomicExpr::AO__atomic_add_fetch: 744 case AtomicExpr::AO__atomic_sub_fetch: 745 case AtomicExpr::AO__c11_atomic_store: 746 case AtomicExpr::AO__c11_atomic_exchange: 747 case AtomicExpr::AO__atomic_store_n: 748 case AtomicExpr::AO__atomic_exchange_n: 749 case AtomicExpr::AO__c11_atomic_fetch_and: 750 case AtomicExpr::AO__c11_atomic_fetch_or: 751 case AtomicExpr::AO__c11_atomic_fetch_xor: 752 case AtomicExpr::AO__atomic_fetch_and: 753 case AtomicExpr::AO__atomic_fetch_or: 754 case AtomicExpr::AO__atomic_fetch_xor: 755 case AtomicExpr::AO__atomic_fetch_nand: 756 case AtomicExpr::AO__atomic_and_fetch: 757 case AtomicExpr::AO__atomic_or_fetch: 758 case AtomicExpr::AO__atomic_xor_fetch: 759 case AtomicExpr::AO__atomic_nand_fetch: 760 Val1 = EmitValToTemp(*this, E->getVal1()); 761 break; 762 } 763 764 QualType RValTy = E->getType().getUnqualifiedType(); 765 766 // The inlined atomics only function on iN types, where N is a power of 2. We 767 // need to make sure (via temporaries if necessary) that all incoming values 768 // are compatible. 769 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 770 AtomicInfo Atomics(*this, AtomicVal); 771 772 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 773 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 774 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 775 if (Dest.isValid()) 776 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 777 else if (E->isCmpXChg()) 778 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 779 else if (!RValTy->isVoidType()) 780 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 781 782 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 783 if (UseLibcall) { 784 bool UseOptimizedLibcall = false; 785 switch (E->getOp()) { 786 case AtomicExpr::AO__c11_atomic_init: 787 llvm_unreachable("Already handled above with EmitAtomicInit!"); 788 789 case AtomicExpr::AO__c11_atomic_fetch_add: 790 case AtomicExpr::AO__atomic_fetch_add: 791 case AtomicExpr::AO__c11_atomic_fetch_and: 792 case AtomicExpr::AO__atomic_fetch_and: 793 case AtomicExpr::AO__c11_atomic_fetch_or: 794 case AtomicExpr::AO__atomic_fetch_or: 795 case AtomicExpr::AO__atomic_fetch_nand: 796 case AtomicExpr::AO__c11_atomic_fetch_sub: 797 case AtomicExpr::AO__atomic_fetch_sub: 798 case AtomicExpr::AO__c11_atomic_fetch_xor: 799 case AtomicExpr::AO__atomic_fetch_xor: 800 case AtomicExpr::AO__atomic_add_fetch: 801 case AtomicExpr::AO__atomic_and_fetch: 802 case AtomicExpr::AO__atomic_nand_fetch: 803 case AtomicExpr::AO__atomic_or_fetch: 804 case AtomicExpr::AO__atomic_sub_fetch: 805 case AtomicExpr::AO__atomic_xor_fetch: 806 // For these, only library calls for certain sizes exist. 807 UseOptimizedLibcall = true; 808 break; 809 810 case AtomicExpr::AO__c11_atomic_load: 811 case AtomicExpr::AO__c11_atomic_store: 812 case AtomicExpr::AO__c11_atomic_exchange: 813 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 814 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 815 case AtomicExpr::AO__atomic_load_n: 816 case AtomicExpr::AO__atomic_load: 817 case AtomicExpr::AO__atomic_store_n: 818 case AtomicExpr::AO__atomic_store: 819 case AtomicExpr::AO__atomic_exchange_n: 820 case AtomicExpr::AO__atomic_exchange: 821 case AtomicExpr::AO__atomic_compare_exchange_n: 822 case AtomicExpr::AO__atomic_compare_exchange: 823 // Only use optimized library calls for sizes for which they exist. 824 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 825 UseOptimizedLibcall = true; 826 break; 827 } 828 829 CallArgList Args; 830 if (!UseOptimizedLibcall) { 831 // For non-optimized library calls, the size is the first parameter 832 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 833 getContext().getSizeType()); 834 } 835 // Atomic address is the first or second parameter 836 Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())), 837 getContext().VoidPtrTy); 838 839 std::string LibCallName; 840 QualType LoweredMemTy = 841 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 842 QualType RetTy; 843 bool HaveRetTy = false; 844 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 845 switch (E->getOp()) { 846 case AtomicExpr::AO__c11_atomic_init: 847 llvm_unreachable("Already handled!"); 848 849 // There is only one libcall for compare an exchange, because there is no 850 // optimisation benefit possible from a libcall version of a weak compare 851 // and exchange. 852 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 853 // void *desired, int success, int failure) 854 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 855 // int success, int failure) 856 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 857 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 858 case AtomicExpr::AO__atomic_compare_exchange: 859 case AtomicExpr::AO__atomic_compare_exchange_n: 860 LibCallName = "__atomic_compare_exchange"; 861 RetTy = getContext().BoolTy; 862 HaveRetTy = true; 863 Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())), 864 getContext().VoidPtrTy); 865 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 866 MemTy, E->getExprLoc(), sizeChars); 867 Args.add(RValue::get(Order), getContext().IntTy); 868 Order = OrderFail; 869 break; 870 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 871 // int order) 872 // T __atomic_exchange_N(T *mem, T val, int order) 873 case AtomicExpr::AO__c11_atomic_exchange: 874 case AtomicExpr::AO__atomic_exchange_n: 875 case AtomicExpr::AO__atomic_exchange: 876 LibCallName = "__atomic_exchange"; 877 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 878 MemTy, E->getExprLoc(), sizeChars); 879 break; 880 // void __atomic_store(size_t size, void *mem, void *val, int order) 881 // void __atomic_store_N(T *mem, T val, int order) 882 case AtomicExpr::AO__c11_atomic_store: 883 case AtomicExpr::AO__atomic_store: 884 case AtomicExpr::AO__atomic_store_n: 885 LibCallName = "__atomic_store"; 886 RetTy = getContext().VoidTy; 887 HaveRetTy = true; 888 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 889 MemTy, E->getExprLoc(), sizeChars); 890 break; 891 // void __atomic_load(size_t size, void *mem, void *return, int order) 892 // T __atomic_load_N(T *mem, int order) 893 case AtomicExpr::AO__c11_atomic_load: 894 case AtomicExpr::AO__atomic_load: 895 case AtomicExpr::AO__atomic_load_n: 896 LibCallName = "__atomic_load"; 897 break; 898 // T __atomic_add_fetch_N(T *mem, T val, int order) 899 // T __atomic_fetch_add_N(T *mem, T val, int order) 900 case AtomicExpr::AO__atomic_add_fetch: 901 PostOp = llvm::Instruction::Add; 902 // Fall through. 903 case AtomicExpr::AO__c11_atomic_fetch_add: 904 case AtomicExpr::AO__atomic_fetch_add: 905 LibCallName = "__atomic_fetch_add"; 906 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 907 LoweredMemTy, E->getExprLoc(), sizeChars); 908 break; 909 // T __atomic_and_fetch_N(T *mem, T val, int order) 910 // T __atomic_fetch_and_N(T *mem, T val, int order) 911 case AtomicExpr::AO__atomic_and_fetch: 912 PostOp = llvm::Instruction::And; 913 // Fall through. 914 case AtomicExpr::AO__c11_atomic_fetch_and: 915 case AtomicExpr::AO__atomic_fetch_and: 916 LibCallName = "__atomic_fetch_and"; 917 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 918 MemTy, E->getExprLoc(), sizeChars); 919 break; 920 // T __atomic_or_fetch_N(T *mem, T val, int order) 921 // T __atomic_fetch_or_N(T *mem, T val, int order) 922 case AtomicExpr::AO__atomic_or_fetch: 923 PostOp = llvm::Instruction::Or; 924 // Fall through. 925 case AtomicExpr::AO__c11_atomic_fetch_or: 926 case AtomicExpr::AO__atomic_fetch_or: 927 LibCallName = "__atomic_fetch_or"; 928 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 929 MemTy, E->getExprLoc(), sizeChars); 930 break; 931 // T __atomic_sub_fetch_N(T *mem, T val, int order) 932 // T __atomic_fetch_sub_N(T *mem, T val, int order) 933 case AtomicExpr::AO__atomic_sub_fetch: 934 PostOp = llvm::Instruction::Sub; 935 // Fall through. 936 case AtomicExpr::AO__c11_atomic_fetch_sub: 937 case AtomicExpr::AO__atomic_fetch_sub: 938 LibCallName = "__atomic_fetch_sub"; 939 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 940 LoweredMemTy, E->getExprLoc(), sizeChars); 941 break; 942 // T __atomic_xor_fetch_N(T *mem, T val, int order) 943 // T __atomic_fetch_xor_N(T *mem, T val, int order) 944 case AtomicExpr::AO__atomic_xor_fetch: 945 PostOp = llvm::Instruction::Xor; 946 // Fall through. 947 case AtomicExpr::AO__c11_atomic_fetch_xor: 948 case AtomicExpr::AO__atomic_fetch_xor: 949 LibCallName = "__atomic_fetch_xor"; 950 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 951 MemTy, E->getExprLoc(), sizeChars); 952 break; 953 // T __atomic_nand_fetch_N(T *mem, T val, int order) 954 // T __atomic_fetch_nand_N(T *mem, T val, int order) 955 case AtomicExpr::AO__atomic_nand_fetch: 956 PostOp = llvm::Instruction::And; // the NOT is special cased below 957 // Fall through. 958 case AtomicExpr::AO__atomic_fetch_nand: 959 LibCallName = "__atomic_fetch_nand"; 960 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 961 MemTy, E->getExprLoc(), sizeChars); 962 break; 963 } 964 965 // Optimized functions have the size in their name. 966 if (UseOptimizedLibcall) 967 LibCallName += "_" + llvm::utostr(Size); 968 // By default, assume we return a value of the atomic type. 969 if (!HaveRetTy) { 970 if (UseOptimizedLibcall) { 971 // Value is returned directly. 972 // The function returns an appropriately sized integer type. 973 RetTy = getContext().getIntTypeForBitwidth( 974 getContext().toBits(sizeChars), /*Signed=*/false); 975 } else { 976 // Value is returned through parameter before the order. 977 RetTy = getContext().VoidTy; 978 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 979 getContext().VoidPtrTy); 980 } 981 } 982 // order is always the last parameter 983 Args.add(RValue::get(Order), 984 getContext().IntTy); 985 986 // PostOp is only needed for the atomic_*_fetch operations, and 987 // thus is only needed for and implemented in the 988 // UseOptimizedLibcall codepath. 989 assert(UseOptimizedLibcall || !PostOp); 990 991 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 992 // The value is returned directly from the libcall. 993 if (E->isCmpXChg()) 994 return Res; 995 996 // The value is returned directly for optimized libcalls but the expr 997 // provided an out-param. 998 if (UseOptimizedLibcall && Res.getScalarVal()) { 999 llvm::Value *ResVal = Res.getScalarVal(); 1000 if (PostOp) { 1001 llvm::Value *LoadVal1 = Args[1].RV.getScalarVal(); 1002 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1003 } 1004 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1005 ResVal = Builder.CreateNot(ResVal); 1006 1007 Builder.CreateStore( 1008 ResVal, 1009 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1010 } 1011 1012 if (RValTy->isVoidType()) 1013 return RValue::get(nullptr); 1014 1015 return convertTempToRValue( 1016 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1017 RValTy, E->getExprLoc()); 1018 } 1019 1020 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1021 E->getOp() == AtomicExpr::AO__atomic_store || 1022 E->getOp() == AtomicExpr::AO__atomic_store_n; 1023 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1024 E->getOp() == AtomicExpr::AO__atomic_load || 1025 E->getOp() == AtomicExpr::AO__atomic_load_n; 1026 1027 if (isa<llvm::ConstantInt>(Order)) { 1028 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1029 // We should not ever get to a case where the ordering isn't a valid C ABI 1030 // value, but it's hard to enforce that in general. 1031 if (llvm::isValidAtomicOrderingCABI(ord)) 1032 switch ((llvm::AtomicOrderingCABI)ord) { 1033 case llvm::AtomicOrderingCABI::relaxed: 1034 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1035 llvm::AtomicOrdering::Monotonic); 1036 break; 1037 case llvm::AtomicOrderingCABI::consume: 1038 case llvm::AtomicOrderingCABI::acquire: 1039 if (IsStore) 1040 break; // Avoid crashing on code with undefined behavior 1041 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1042 llvm::AtomicOrdering::Acquire); 1043 break; 1044 case llvm::AtomicOrderingCABI::release: 1045 if (IsLoad) 1046 break; // Avoid crashing on code with undefined behavior 1047 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1048 llvm::AtomicOrdering::Release); 1049 break; 1050 case llvm::AtomicOrderingCABI::acq_rel: 1051 if (IsLoad || IsStore) 1052 break; // Avoid crashing on code with undefined behavior 1053 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1054 llvm::AtomicOrdering::AcquireRelease); 1055 break; 1056 case llvm::AtomicOrderingCABI::seq_cst: 1057 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1058 llvm::AtomicOrdering::SequentiallyConsistent); 1059 break; 1060 } 1061 if (RValTy->isVoidType()) 1062 return RValue::get(nullptr); 1063 1064 return convertTempToRValue( 1065 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1066 RValTy, E->getExprLoc()); 1067 } 1068 1069 // Long case, when Order isn't obviously constant. 1070 1071 // Create all the relevant BB's 1072 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1073 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1074 *SeqCstBB = nullptr; 1075 MonotonicBB = createBasicBlock("monotonic", CurFn); 1076 if (!IsStore) 1077 AcquireBB = createBasicBlock("acquire", CurFn); 1078 if (!IsLoad) 1079 ReleaseBB = createBasicBlock("release", CurFn); 1080 if (!IsLoad && !IsStore) 1081 AcqRelBB = createBasicBlock("acqrel", CurFn); 1082 SeqCstBB = createBasicBlock("seqcst", CurFn); 1083 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1084 1085 // Create the switch for the split 1086 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1087 // doesn't matter unless someone is crazy enough to use something that 1088 // doesn't fold to a constant for the ordering. 1089 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1090 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1091 1092 // Emit all the different atomics 1093 Builder.SetInsertPoint(MonotonicBB); 1094 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 1095 Size, llvm::AtomicOrdering::Monotonic); 1096 Builder.CreateBr(ContBB); 1097 if (!IsStore) { 1098 Builder.SetInsertPoint(AcquireBB); 1099 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 1100 Size, llvm::AtomicOrdering::Acquire); 1101 Builder.CreateBr(ContBB); 1102 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1103 AcquireBB); 1104 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1105 AcquireBB); 1106 } 1107 if (!IsLoad) { 1108 Builder.SetInsertPoint(ReleaseBB); 1109 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 1110 Size, llvm::AtomicOrdering::Release); 1111 Builder.CreateBr(ContBB); 1112 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1113 ReleaseBB); 1114 } 1115 if (!IsLoad && !IsStore) { 1116 Builder.SetInsertPoint(AcqRelBB); 1117 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 1118 Size, llvm::AtomicOrdering::AcquireRelease); 1119 Builder.CreateBr(ContBB); 1120 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1121 AcqRelBB); 1122 } 1123 Builder.SetInsertPoint(SeqCstBB); 1124 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 1125 Size, llvm::AtomicOrdering::SequentiallyConsistent); 1126 Builder.CreateBr(ContBB); 1127 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1128 SeqCstBB); 1129 1130 // Cleanup and return 1131 Builder.SetInsertPoint(ContBB); 1132 if (RValTy->isVoidType()) 1133 return RValue::get(nullptr); 1134 1135 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1136 return convertTempToRValue( 1137 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1138 RValTy, E->getExprLoc()); 1139 } 1140 1141 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1142 unsigned addrspace = 1143 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1144 llvm::IntegerType *ty = 1145 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1146 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1147 } 1148 1149 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1150 llvm::Type *Ty = Addr.getElementType(); 1151 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1152 if (SourceSizeInBits != AtomicSizeInBits) { 1153 Address Tmp = CreateTempAlloca(); 1154 CGF.Builder.CreateMemCpy(Tmp, Addr, 1155 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1156 Addr = Tmp; 1157 } 1158 1159 return emitCastToAtomicIntPointer(Addr); 1160 } 1161 1162 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1163 AggValueSlot resultSlot, 1164 SourceLocation loc, 1165 bool asValue) const { 1166 if (LVal.isSimple()) { 1167 if (EvaluationKind == TEK_Aggregate) 1168 return resultSlot.asRValue(); 1169 1170 // Drill into the padding structure if we have one. 1171 if (hasPadding()) 1172 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 1173 1174 // Otherwise, just convert the temporary to an r-value using the 1175 // normal conversion routine. 1176 return CGF.convertTempToRValue(addr, getValueType(), loc); 1177 } 1178 if (!asValue) 1179 // Get RValue from temp memory as atomic for non-simple lvalues 1180 return RValue::get(CGF.Builder.CreateLoad(addr)); 1181 if (LVal.isBitField()) 1182 return CGF.EmitLoadOfBitfieldLValue( 1183 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1184 LVal.getAlignmentSource())); 1185 if (LVal.isVectorElt()) 1186 return CGF.EmitLoadOfLValue( 1187 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1188 LVal.getAlignmentSource()), loc); 1189 assert(LVal.isExtVectorElt()); 1190 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1191 addr, LVal.getExtVectorElts(), LVal.getType(), 1192 LVal.getAlignmentSource())); 1193 } 1194 1195 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1196 AggValueSlot ResultSlot, 1197 SourceLocation Loc, 1198 bool AsValue) const { 1199 // Try not to in some easy cases. 1200 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1201 if (getEvaluationKind() == TEK_Scalar && 1202 (((!LVal.isBitField() || 1203 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1204 !hasPadding()) || 1205 !AsValue)) { 1206 auto *ValTy = AsValue 1207 ? CGF.ConvertTypeForMem(ValueTy) 1208 : getAtomicAddress().getType()->getPointerElementType(); 1209 if (ValTy->isIntegerTy()) { 1210 assert(IntVal->getType() == ValTy && "Different integer types."); 1211 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1212 } else if (ValTy->isPointerTy()) 1213 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1214 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1215 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1216 } 1217 1218 // Create a temporary. This needs to be big enough to hold the 1219 // atomic integer. 1220 Address Temp = Address::invalid(); 1221 bool TempIsVolatile = false; 1222 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1223 assert(!ResultSlot.isIgnored()); 1224 Temp = ResultSlot.getAddress(); 1225 TempIsVolatile = ResultSlot.isVolatile(); 1226 } else { 1227 Temp = CreateTempAlloca(); 1228 } 1229 1230 // Slam the integer into the temporary. 1231 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1232 CGF.Builder.CreateStore(IntVal, CastTemp) 1233 ->setVolatile(TempIsVolatile); 1234 1235 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1236 } 1237 1238 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1239 llvm::AtomicOrdering AO, bool) { 1240 // void __atomic_load(size_t size, void *mem, void *return, int order); 1241 CallArgList Args; 1242 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1243 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1244 CGF.getContext().VoidPtrTy); 1245 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1246 CGF.getContext().VoidPtrTy); 1247 Args.add( 1248 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1249 CGF.getContext().IntTy); 1250 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1251 } 1252 1253 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1254 bool IsVolatile) { 1255 // Okay, we're doing this natively. 1256 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1257 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1258 Load->setAtomic(AO); 1259 1260 // Other decoration. 1261 if (IsVolatile) 1262 Load->setVolatile(true); 1263 if (LVal.getTBAAInfo()) 1264 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1265 return Load; 1266 } 1267 1268 /// An LValue is a candidate for having its loads and stores be made atomic if 1269 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1270 /// performing such an operation can be performed without a libcall. 1271 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1272 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1273 AtomicInfo AI(*this, LV); 1274 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1275 // An atomic is inline if we don't need to use a libcall. 1276 bool AtomicIsInline = !AI.shouldUseLibcall(); 1277 // MSVC doesn't seem to do this for types wider than a pointer. 1278 if (getContext().getTypeSize(LV.getType()) > 1279 getContext().getTypeSize(getContext().getIntPtrType())) 1280 return false; 1281 return IsVolatile && AtomicIsInline; 1282 } 1283 1284 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1285 AggValueSlot Slot) { 1286 llvm::AtomicOrdering AO; 1287 bool IsVolatile = LV.isVolatileQualified(); 1288 if (LV.getType()->isAtomicType()) { 1289 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1290 } else { 1291 AO = llvm::AtomicOrdering::Acquire; 1292 IsVolatile = true; 1293 } 1294 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1295 } 1296 1297 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1298 bool AsValue, llvm::AtomicOrdering AO, 1299 bool IsVolatile) { 1300 // Check whether we should use a library call. 1301 if (shouldUseLibcall()) { 1302 Address TempAddr = Address::invalid(); 1303 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1304 assert(getEvaluationKind() == TEK_Aggregate); 1305 TempAddr = ResultSlot.getAddress(); 1306 } else 1307 TempAddr = CreateTempAlloca(); 1308 1309 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1310 1311 // Okay, turn that back into the original value or whole atomic (for 1312 // non-simple lvalues) type. 1313 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1314 } 1315 1316 // Okay, we're doing this natively. 1317 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1318 1319 // If we're ignoring an aggregate return, don't do anything. 1320 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1321 return RValue::getAggregate(Address::invalid(), false); 1322 1323 // Okay, turn that back into the original value or atomic (for non-simple 1324 // lvalues) type. 1325 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1326 } 1327 1328 /// Emit a load from an l-value of atomic type. Note that the r-value 1329 /// we produce is an r-value of the atomic *value* type. 1330 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1331 llvm::AtomicOrdering AO, bool IsVolatile, 1332 AggValueSlot resultSlot) { 1333 AtomicInfo Atomics(*this, src); 1334 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1335 IsVolatile); 1336 } 1337 1338 /// Copy an r-value into memory as part of storing to an atomic type. 1339 /// This needs to create a bit-pattern suitable for atomic operations. 1340 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1341 assert(LVal.isSimple()); 1342 // If we have an r-value, the rvalue should be of the atomic type, 1343 // which means that the caller is responsible for having zeroed 1344 // any padding. Just do an aggregate copy of that type. 1345 if (rvalue.isAggregate()) { 1346 CGF.EmitAggregateCopy(getAtomicAddress(), 1347 rvalue.getAggregateAddress(), 1348 getAtomicType(), 1349 (rvalue.isVolatileQualified() 1350 || LVal.isVolatileQualified())); 1351 return; 1352 } 1353 1354 // Okay, otherwise we're copying stuff. 1355 1356 // Zero out the buffer if necessary. 1357 emitMemSetZeroIfNecessary(); 1358 1359 // Drill past the padding if present. 1360 LValue TempLVal = projectValue(); 1361 1362 // Okay, store the rvalue in. 1363 if (rvalue.isScalar()) { 1364 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1365 } else { 1366 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1367 } 1368 } 1369 1370 1371 /// Materialize an r-value into memory for the purposes of storing it 1372 /// to an atomic type. 1373 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1374 // Aggregate r-values are already in memory, and EmitAtomicStore 1375 // requires them to be values of the atomic type. 1376 if (rvalue.isAggregate()) 1377 return rvalue.getAggregateAddress(); 1378 1379 // Otherwise, make a temporary and materialize into it. 1380 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1381 AtomicInfo Atomics(CGF, TempLV); 1382 Atomics.emitCopyIntoMemory(rvalue); 1383 return TempLV.getAddress(); 1384 } 1385 1386 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1387 // If we've got a scalar value of the right size, try to avoid going 1388 // through memory. 1389 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1390 llvm::Value *Value = RVal.getScalarVal(); 1391 if (isa<llvm::IntegerType>(Value->getType())) 1392 return CGF.EmitToMemory(Value, ValueTy); 1393 else { 1394 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1395 CGF.getLLVMContext(), 1396 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1397 if (isa<llvm::PointerType>(Value->getType())) 1398 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1399 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1400 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1401 } 1402 } 1403 // Otherwise, we need to go through memory. 1404 // Put the r-value in memory. 1405 Address Addr = materializeRValue(RVal); 1406 1407 // Cast the temporary to the atomic int type and pull a value out. 1408 Addr = emitCastToAtomicIntPointer(Addr); 1409 return CGF.Builder.CreateLoad(Addr); 1410 } 1411 1412 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1413 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1414 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1415 // Do the atomic store. 1416 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1417 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1418 ExpectedVal, DesiredVal, 1419 Success, Failure); 1420 // Other decoration. 1421 Inst->setVolatile(LVal.isVolatileQualified()); 1422 Inst->setWeak(IsWeak); 1423 1424 // Okay, turn that back into the original value type. 1425 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1426 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1427 return std::make_pair(PreviousVal, SuccessFailureVal); 1428 } 1429 1430 llvm::Value * 1431 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1432 llvm::Value *DesiredAddr, 1433 llvm::AtomicOrdering Success, 1434 llvm::AtomicOrdering Failure) { 1435 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1436 // void *desired, int success, int failure); 1437 CallArgList Args; 1438 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1439 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1440 CGF.getContext().VoidPtrTy); 1441 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1442 CGF.getContext().VoidPtrTy); 1443 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1444 CGF.getContext().VoidPtrTy); 1445 Args.add(RValue::get( 1446 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1447 CGF.getContext().IntTy); 1448 Args.add(RValue::get( 1449 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1450 CGF.getContext().IntTy); 1451 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1452 CGF.getContext().BoolTy, Args); 1453 1454 return SuccessFailureRVal.getScalarVal(); 1455 } 1456 1457 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1458 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1459 llvm::AtomicOrdering Failure, bool IsWeak) { 1460 if (isStrongerThan(Failure, Success)) 1461 // Don't assert on undefined behavior "failure argument shall be no stronger 1462 // than the success argument". 1463 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1464 1465 // Check whether we should use a library call. 1466 if (shouldUseLibcall()) { 1467 // Produce a source address. 1468 Address ExpectedAddr = materializeRValue(Expected); 1469 Address DesiredAddr = materializeRValue(Desired); 1470 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1471 DesiredAddr.getPointer(), 1472 Success, Failure); 1473 return std::make_pair( 1474 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1475 SourceLocation(), /*AsValue=*/false), 1476 Res); 1477 } 1478 1479 // If we've got a scalar value of the right size, try to avoid going 1480 // through memory. 1481 auto *ExpectedVal = convertRValueToInt(Expected); 1482 auto *DesiredVal = convertRValueToInt(Desired); 1483 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1484 Failure, IsWeak); 1485 return std::make_pair( 1486 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1487 SourceLocation(), /*AsValue=*/false), 1488 Res.second); 1489 } 1490 1491 static void 1492 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1493 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1494 Address DesiredAddr) { 1495 RValue UpRVal; 1496 LValue AtomicLVal = Atomics.getAtomicLValue(); 1497 LValue DesiredLVal; 1498 if (AtomicLVal.isSimple()) { 1499 UpRVal = OldRVal; 1500 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1501 } else { 1502 // Build new lvalue for temp address 1503 Address Ptr = Atomics.materializeRValue(OldRVal); 1504 LValue UpdateLVal; 1505 if (AtomicLVal.isBitField()) { 1506 UpdateLVal = 1507 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1508 AtomicLVal.getType(), 1509 AtomicLVal.getAlignmentSource()); 1510 DesiredLVal = 1511 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1512 AtomicLVal.getType(), 1513 AtomicLVal.getAlignmentSource()); 1514 } else if (AtomicLVal.isVectorElt()) { 1515 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1516 AtomicLVal.getType(), 1517 AtomicLVal.getAlignmentSource()); 1518 DesiredLVal = LValue::MakeVectorElt( 1519 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1520 AtomicLVal.getAlignmentSource()); 1521 } else { 1522 assert(AtomicLVal.isExtVectorElt()); 1523 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1524 AtomicLVal.getType(), 1525 AtomicLVal.getAlignmentSource()); 1526 DesiredLVal = LValue::MakeExtVectorElt( 1527 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1528 AtomicLVal.getAlignmentSource()); 1529 } 1530 UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1531 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1532 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1533 } 1534 // Store new value in the corresponding memory area 1535 RValue NewRVal = UpdateOp(UpRVal); 1536 if (NewRVal.isScalar()) { 1537 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1538 } else { 1539 assert(NewRVal.isComplex()); 1540 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1541 /*isInit=*/false); 1542 } 1543 } 1544 1545 void AtomicInfo::EmitAtomicUpdateLibcall( 1546 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1547 bool IsVolatile) { 1548 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1549 1550 Address ExpectedAddr = CreateTempAlloca(); 1551 1552 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1553 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1554 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1555 CGF.EmitBlock(ContBB); 1556 Address DesiredAddr = CreateTempAlloca(); 1557 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1558 requiresMemSetZero(getAtomicAddress().getElementType())) { 1559 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1560 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1561 } 1562 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1563 AggValueSlot::ignored(), 1564 SourceLocation(), /*AsValue=*/false); 1565 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1566 auto *Res = 1567 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1568 DesiredAddr.getPointer(), 1569 AO, Failure); 1570 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1571 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1572 } 1573 1574 void AtomicInfo::EmitAtomicUpdateOp( 1575 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1576 bool IsVolatile) { 1577 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1578 1579 // Do the atomic load. 1580 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1581 // For non-simple lvalues perform compare-and-swap procedure. 1582 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1583 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1584 auto *CurBB = CGF.Builder.GetInsertBlock(); 1585 CGF.EmitBlock(ContBB); 1586 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1587 /*NumReservedValues=*/2); 1588 PHI->addIncoming(OldVal, CurBB); 1589 Address NewAtomicAddr = CreateTempAlloca(); 1590 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1591 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1592 requiresMemSetZero(getAtomicAddress().getElementType())) { 1593 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1594 } 1595 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1596 SourceLocation(), /*AsValue=*/false); 1597 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1598 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1599 // Try to write new value using cmpxchg operation 1600 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1601 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1602 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1603 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1604 } 1605 1606 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1607 RValue UpdateRVal, Address DesiredAddr) { 1608 LValue AtomicLVal = Atomics.getAtomicLValue(); 1609 LValue DesiredLVal; 1610 // Build new lvalue for temp address 1611 if (AtomicLVal.isBitField()) { 1612 DesiredLVal = 1613 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1614 AtomicLVal.getType(), 1615 AtomicLVal.getAlignmentSource()); 1616 } else if (AtomicLVal.isVectorElt()) { 1617 DesiredLVal = 1618 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1619 AtomicLVal.getType(), 1620 AtomicLVal.getAlignmentSource()); 1621 } else { 1622 assert(AtomicLVal.isExtVectorElt()); 1623 DesiredLVal = LValue::MakeExtVectorElt( 1624 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1625 AtomicLVal.getAlignmentSource()); 1626 } 1627 DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1628 // Store new value in the corresponding memory area 1629 assert(UpdateRVal.isScalar()); 1630 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1631 } 1632 1633 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1634 RValue UpdateRVal, bool IsVolatile) { 1635 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1636 1637 Address ExpectedAddr = CreateTempAlloca(); 1638 1639 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1640 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1641 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1642 CGF.EmitBlock(ContBB); 1643 Address DesiredAddr = CreateTempAlloca(); 1644 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1645 requiresMemSetZero(getAtomicAddress().getElementType())) { 1646 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1647 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1648 } 1649 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1650 auto *Res = 1651 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1652 DesiredAddr.getPointer(), 1653 AO, Failure); 1654 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1655 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1656 } 1657 1658 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1659 bool IsVolatile) { 1660 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1661 1662 // Do the atomic load. 1663 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1664 // For non-simple lvalues perform compare-and-swap procedure. 1665 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1666 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1667 auto *CurBB = CGF.Builder.GetInsertBlock(); 1668 CGF.EmitBlock(ContBB); 1669 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1670 /*NumReservedValues=*/2); 1671 PHI->addIncoming(OldVal, CurBB); 1672 Address NewAtomicAddr = CreateTempAlloca(); 1673 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1674 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1675 requiresMemSetZero(getAtomicAddress().getElementType())) { 1676 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1677 } 1678 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1679 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1680 // Try to write new value using cmpxchg operation 1681 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1682 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1683 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1684 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1685 } 1686 1687 void AtomicInfo::EmitAtomicUpdate( 1688 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1689 bool IsVolatile) { 1690 if (shouldUseLibcall()) { 1691 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1692 } else { 1693 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1694 } 1695 } 1696 1697 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1698 bool IsVolatile) { 1699 if (shouldUseLibcall()) { 1700 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1701 } else { 1702 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1703 } 1704 } 1705 1706 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1707 bool isInit) { 1708 bool IsVolatile = lvalue.isVolatileQualified(); 1709 llvm::AtomicOrdering AO; 1710 if (lvalue.getType()->isAtomicType()) { 1711 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1712 } else { 1713 AO = llvm::AtomicOrdering::Release; 1714 IsVolatile = true; 1715 } 1716 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1717 } 1718 1719 /// Emit a store to an l-value of atomic type. 1720 /// 1721 /// Note that the r-value is expected to be an r-value *of the atomic 1722 /// type*; this means that for aggregate r-values, it should include 1723 /// storage for any padding that was necessary. 1724 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1725 llvm::AtomicOrdering AO, bool IsVolatile, 1726 bool isInit) { 1727 // If this is an aggregate r-value, it should agree in type except 1728 // maybe for address-space qualification. 1729 assert(!rvalue.isAggregate() || 1730 rvalue.getAggregateAddress().getElementType() 1731 == dest.getAddress().getElementType()); 1732 1733 AtomicInfo atomics(*this, dest); 1734 LValue LVal = atomics.getAtomicLValue(); 1735 1736 // If this is an initialization, just put the value there normally. 1737 if (LVal.isSimple()) { 1738 if (isInit) { 1739 atomics.emitCopyIntoMemory(rvalue); 1740 return; 1741 } 1742 1743 // Check whether we should use a library call. 1744 if (atomics.shouldUseLibcall()) { 1745 // Produce a source address. 1746 Address srcAddr = atomics.materializeRValue(rvalue); 1747 1748 // void __atomic_store(size_t size, void *mem, void *val, int order) 1749 CallArgList args; 1750 args.add(RValue::get(atomics.getAtomicSizeValue()), 1751 getContext().getSizeType()); 1752 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 1753 getContext().VoidPtrTy); 1754 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 1755 getContext().VoidPtrTy); 1756 args.add( 1757 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 1758 getContext().IntTy); 1759 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 1760 return; 1761 } 1762 1763 // Okay, we're doing this natively. 1764 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 1765 1766 // Do the atomic store. 1767 Address addr = 1768 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 1769 intValue = Builder.CreateIntCast( 1770 intValue, addr.getElementType(), /*isSigned=*/false); 1771 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 1772 1773 // Initializations don't need to be atomic. 1774 if (!isInit) 1775 store->setAtomic(AO); 1776 1777 // Other decoration. 1778 if (IsVolatile) 1779 store->setVolatile(true); 1780 if (dest.getTBAAInfo()) 1781 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 1782 return; 1783 } 1784 1785 // Emit simple atomic update operation. 1786 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 1787 } 1788 1789 /// Emit a compare-and-exchange op for atomic type. 1790 /// 1791 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 1792 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 1793 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 1794 AggValueSlot Slot) { 1795 // If this is an aggregate r-value, it should agree in type except 1796 // maybe for address-space qualification. 1797 assert(!Expected.isAggregate() || 1798 Expected.getAggregateAddress().getElementType() == 1799 Obj.getAddress().getElementType()); 1800 assert(!Desired.isAggregate() || 1801 Desired.getAggregateAddress().getElementType() == 1802 Obj.getAddress().getElementType()); 1803 AtomicInfo Atomics(*this, Obj); 1804 1805 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 1806 IsWeak); 1807 } 1808 1809 void CodeGenFunction::EmitAtomicUpdate( 1810 LValue LVal, llvm::AtomicOrdering AO, 1811 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 1812 AtomicInfo Atomics(*this, LVal); 1813 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 1814 } 1815 1816 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 1817 AtomicInfo atomics(*this, dest); 1818 1819 switch (atomics.getEvaluationKind()) { 1820 case TEK_Scalar: { 1821 llvm::Value *value = EmitScalarExpr(init); 1822 atomics.emitCopyIntoMemory(RValue::get(value)); 1823 return; 1824 } 1825 1826 case TEK_Complex: { 1827 ComplexPairTy value = EmitComplexExpr(init); 1828 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 1829 return; 1830 } 1831 1832 case TEK_Aggregate: { 1833 // Fix up the destination if the initializer isn't an expression 1834 // of atomic type. 1835 bool Zeroed = false; 1836 if (!init->getType()->isAtomicType()) { 1837 Zeroed = atomics.emitMemSetZeroIfNecessary(); 1838 dest = atomics.projectValue(); 1839 } 1840 1841 // Evaluate the expression directly into the destination. 1842 AggValueSlot slot = AggValueSlot::forLValue(dest, 1843 AggValueSlot::IsNotDestructed, 1844 AggValueSlot::DoesNotNeedGCBarriers, 1845 AggValueSlot::IsNotAliased, 1846 Zeroed ? AggValueSlot::IsZeroed : 1847 AggValueSlot::IsNotZeroed); 1848 1849 EmitAggExpr(init, slot); 1850 return; 1851 } 1852 } 1853 llvm_unreachable("bad evaluation kind"); 1854 } 1855
