1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file contains the code for emitting atomic operations. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CodeGenFunction.h" 15 #include "CGCall.h" 16 #include "CodeGenModule.h" 17 #include "clang/AST/ASTContext.h" 18 #include "llvm/IR/DataLayout.h" 19 #include "llvm/IR/Intrinsics.h" 20 #include "llvm/IR/Operator.h" 21 22 using namespace clang; 23 using namespace CodeGen; 24 25 // The ABI values for various atomic memory orderings. 26 enum AtomicOrderingKind { 27 AO_ABI_memory_order_relaxed = 0, 28 AO_ABI_memory_order_consume = 1, 29 AO_ABI_memory_order_acquire = 2, 30 AO_ABI_memory_order_release = 3, 31 AO_ABI_memory_order_acq_rel = 4, 32 AO_ABI_memory_order_seq_cst = 5 33 }; 34 35 namespace { 36 class AtomicInfo { 37 CodeGenFunction &CGF; 38 QualType AtomicTy; 39 QualType ValueTy; 40 uint64_t AtomicSizeInBits; 41 uint64_t ValueSizeInBits; 42 CharUnits AtomicAlign; 43 CharUnits ValueAlign; 44 CharUnits LValueAlign; 45 TypeEvaluationKind EvaluationKind; 46 bool UseLibcall; 47 public: 48 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) { 49 assert(lvalue.isSimple()); 50 51 AtomicTy = lvalue.getType(); 52 ValueTy = AtomicTy->castAs<AtomicType>()->getValueType(); 53 EvaluationKind = CGF.getEvaluationKind(ValueTy); 54 55 ASTContext &C = CGF.getContext(); 56 57 uint64_t valueAlignInBits; 58 llvm::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy); 59 60 uint64_t atomicAlignInBits; 61 llvm::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy); 62 63 assert(ValueSizeInBits <= AtomicSizeInBits); 64 assert(valueAlignInBits <= atomicAlignInBits); 65 66 AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits); 67 ValueAlign = C.toCharUnitsFromBits(valueAlignInBits); 68 if (lvalue.getAlignment().isZero()) 69 lvalue.setAlignment(AtomicAlign); 70 71 UseLibcall = 72 (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) || 73 AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth()); 74 } 75 76 QualType getAtomicType() const { return AtomicTy; } 77 QualType getValueType() const { return ValueTy; } 78 CharUnits getAtomicAlignment() const { return AtomicAlign; } 79 CharUnits getValueAlignment() const { return ValueAlign; } 80 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 81 uint64_t getValueSizeInBits() const { return AtomicSizeInBits; } 82 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 83 bool shouldUseLibcall() const { return UseLibcall; } 84 85 /// Is the atomic size larger than the underlying value type? 86 /// 87 /// Note that the absence of padding does not mean that atomic 88 /// objects are completely interchangeable with non-atomic 89 /// objects: we might have promoted the alignment of a type 90 /// without making it bigger. 91 bool hasPadding() const { 92 return (ValueSizeInBits != AtomicSizeInBits); 93 } 94 95 void emitMemSetZeroIfNecessary(LValue dest) const; 96 97 llvm::Value *getAtomicSizeValue() const { 98 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 99 return CGF.CGM.getSize(size); 100 } 101 102 /// Cast the given pointer to an integer pointer suitable for 103 /// atomic operations. 104 llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; 105 106 /// Turn an atomic-layout object into an r-value. 107 RValue convertTempToRValue(llvm::Value *addr, 108 AggValueSlot resultSlot) const; 109 110 /// Copy an atomic r-value into atomic-layout memory. 111 void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const; 112 113 /// Project an l-value down to the value field. 114 LValue projectValue(LValue lvalue) const { 115 llvm::Value *addr = lvalue.getAddress(); 116 if (hasPadding()) 117 addr = CGF.Builder.CreateStructGEP(addr, 0); 118 119 return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(), 120 CGF.getContext(), lvalue.getTBAAInfo()); 121 } 122 123 /// Materialize an atomic r-value in atomic-layout memory. 124 llvm::Value *materializeRValue(RValue rvalue) const; 125 126 private: 127 bool requiresMemSetZero(llvm::Type *type) const; 128 }; 129 } 130 131 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 132 StringRef fnName, 133 QualType resultType, 134 CallArgList &args) { 135 const CGFunctionInfo &fnInfo = 136 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args, 137 FunctionType::ExtInfo(), RequiredArgs::All); 138 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 139 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 140 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); 141 } 142 143 /// Does a store of the given IR type modify the full expected width? 144 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 145 uint64_t expectedSize) { 146 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 147 } 148 149 /// Does the atomic type require memsetting to zero before initialization? 150 /// 151 /// The IR type is provided as a way of making certain queries faster. 152 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 153 // If the atomic type has size padding, we definitely need a memset. 154 if (hasPadding()) return true; 155 156 // Otherwise, do some simple heuristics to try to avoid it: 157 switch (getEvaluationKind()) { 158 // For scalars and complexes, check whether the store size of the 159 // type uses the full size. 160 case TEK_Scalar: 161 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 162 case TEK_Complex: 163 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 164 AtomicSizeInBits / 2); 165 166 // Just be pessimistic about aggregates. 167 case TEK_Aggregate: 168 return true; 169 } 170 llvm_unreachable("bad evaluation kind"); 171 } 172 173 void AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const { 174 llvm::Value *addr = dest.getAddress(); 175 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 176 return; 177 178 CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 179 AtomicSizeInBits / 8, 180 dest.getAlignment().getQuantity()); 181 } 182 183 static void 184 EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, 185 llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2, 186 uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) { 187 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 188 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 189 190 switch (E->getOp()) { 191 case AtomicExpr::AO__c11_atomic_init: 192 llvm_unreachable("Already handled!"); 193 194 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 195 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 196 case AtomicExpr::AO__atomic_compare_exchange: 197 case AtomicExpr::AO__atomic_compare_exchange_n: { 198 // Note that cmpxchg only supports specifying one ordering and 199 // doesn't support weak cmpxchg, at least at the moment. 200 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 201 LoadVal1->setAlignment(Align); 202 llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2); 203 LoadVal2->setAlignment(Align); 204 llvm::AtomicCmpXchgInst *CXI = 205 CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order); 206 CXI->setVolatile(E->isVolatile()); 207 llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1); 208 StoreVal1->setAlignment(Align); 209 llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1); 210 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 211 return; 212 } 213 214 case AtomicExpr::AO__c11_atomic_load: 215 case AtomicExpr::AO__atomic_load_n: 216 case AtomicExpr::AO__atomic_load: { 217 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 218 Load->setAtomic(Order); 219 Load->setAlignment(Size); 220 Load->setVolatile(E->isVolatile()); 221 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest); 222 StoreDest->setAlignment(Align); 223 return; 224 } 225 226 case AtomicExpr::AO__c11_atomic_store: 227 case AtomicExpr::AO__atomic_store: 228 case AtomicExpr::AO__atomic_store_n: { 229 assert(!Dest && "Store does not return a value"); 230 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 231 LoadVal1->setAlignment(Align); 232 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 233 Store->setAtomic(Order); 234 Store->setAlignment(Size); 235 Store->setVolatile(E->isVolatile()); 236 return; 237 } 238 239 case AtomicExpr::AO__c11_atomic_exchange: 240 case AtomicExpr::AO__atomic_exchange_n: 241 case AtomicExpr::AO__atomic_exchange: 242 Op = llvm::AtomicRMWInst::Xchg; 243 break; 244 245 case AtomicExpr::AO__atomic_add_fetch: 246 PostOp = llvm::Instruction::Add; 247 // Fall through. 248 case AtomicExpr::AO__c11_atomic_fetch_add: 249 case AtomicExpr::AO__atomic_fetch_add: 250 Op = llvm::AtomicRMWInst::Add; 251 break; 252 253 case AtomicExpr::AO__atomic_sub_fetch: 254 PostOp = llvm::Instruction::Sub; 255 // Fall through. 256 case AtomicExpr::AO__c11_atomic_fetch_sub: 257 case AtomicExpr::AO__atomic_fetch_sub: 258 Op = llvm::AtomicRMWInst::Sub; 259 break; 260 261 case AtomicExpr::AO__atomic_and_fetch: 262 PostOp = llvm::Instruction::And; 263 // Fall through. 264 case AtomicExpr::AO__c11_atomic_fetch_and: 265 case AtomicExpr::AO__atomic_fetch_and: 266 Op = llvm::AtomicRMWInst::And; 267 break; 268 269 case AtomicExpr::AO__atomic_or_fetch: 270 PostOp = llvm::Instruction::Or; 271 // Fall through. 272 case AtomicExpr::AO__c11_atomic_fetch_or: 273 case AtomicExpr::AO__atomic_fetch_or: 274 Op = llvm::AtomicRMWInst::Or; 275 break; 276 277 case AtomicExpr::AO__atomic_xor_fetch: 278 PostOp = llvm::Instruction::Xor; 279 // Fall through. 280 case AtomicExpr::AO__c11_atomic_fetch_xor: 281 case AtomicExpr::AO__atomic_fetch_xor: 282 Op = llvm::AtomicRMWInst::Xor; 283 break; 284 285 case AtomicExpr::AO__atomic_nand_fetch: 286 PostOp = llvm::Instruction::And; 287 // Fall through. 288 case AtomicExpr::AO__atomic_fetch_nand: 289 Op = llvm::AtomicRMWInst::Nand; 290 break; 291 } 292 293 llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 294 LoadVal1->setAlignment(Align); 295 llvm::AtomicRMWInst *RMWI = 296 CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order); 297 RMWI->setVolatile(E->isVolatile()); 298 299 // For __atomic_*_fetch operations, perform the operation again to 300 // determine the value which was written. 301 llvm::Value *Result = RMWI; 302 if (PostOp) 303 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 304 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 305 Result = CGF.Builder.CreateNot(Result); 306 llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest); 307 StoreDest->setAlignment(Align); 308 } 309 310 // This function emits any expression (scalar, complex, or aggregate) 311 // into a temporary alloca. 312 static llvm::Value * 313 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 314 llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 315 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 316 /*Init*/ true); 317 return DeclPtr; 318 } 319 320 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { 321 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 322 QualType MemTy = AtomicTy; 323 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 324 MemTy = AT->getValueType(); 325 CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy); 326 uint64_t Size = sizeChars.getQuantity(); 327 CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy); 328 unsigned Align = alignChars.getQuantity(); 329 unsigned MaxInlineWidthInBits = 330 getContext().getTargetInfo().getMaxAtomicInlineWidth(); 331 bool UseLibcall = (Size != Align || 332 getContext().toBits(sizeChars) > MaxInlineWidthInBits); 333 334 llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0; 335 Ptr = EmitScalarExpr(E->getPtr()); 336 337 if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { 338 assert(!Dest && "Init does not return a value"); 339 LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext()); 340 EmitAtomicInit(E->getVal1(), lvalue); 341 return RValue::get(0); 342 } 343 344 Order = EmitScalarExpr(E->getOrder()); 345 346 switch (E->getOp()) { 347 case AtomicExpr::AO__c11_atomic_init: 348 llvm_unreachable("Already handled!"); 349 350 case AtomicExpr::AO__c11_atomic_load: 351 case AtomicExpr::AO__atomic_load_n: 352 break; 353 354 case AtomicExpr::AO__atomic_load: 355 Dest = EmitScalarExpr(E->getVal1()); 356 break; 357 358 case AtomicExpr::AO__atomic_store: 359 Val1 = EmitScalarExpr(E->getVal1()); 360 break; 361 362 case AtomicExpr::AO__atomic_exchange: 363 Val1 = EmitScalarExpr(E->getVal1()); 364 Dest = EmitScalarExpr(E->getVal2()); 365 break; 366 367 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 368 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 369 case AtomicExpr::AO__atomic_compare_exchange_n: 370 case AtomicExpr::AO__atomic_compare_exchange: 371 Val1 = EmitScalarExpr(E->getVal1()); 372 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 373 Val2 = EmitScalarExpr(E->getVal2()); 374 else 375 Val2 = EmitValToTemp(*this, E->getVal2()); 376 OrderFail = EmitScalarExpr(E->getOrderFail()); 377 // Evaluate and discard the 'weak' argument. 378 if (E->getNumSubExprs() == 6) 379 EmitScalarExpr(E->getWeak()); 380 break; 381 382 case AtomicExpr::AO__c11_atomic_fetch_add: 383 case AtomicExpr::AO__c11_atomic_fetch_sub: 384 if (MemTy->isPointerType()) { 385 // For pointer arithmetic, we're required to do a bit of math: 386 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 387 // ... but only for the C11 builtins. The GNU builtins expect the 388 // user to multiply by sizeof(T). 389 QualType Val1Ty = E->getVal1()->getType(); 390 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 391 CharUnits PointeeIncAmt = 392 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 393 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 394 Val1 = CreateMemTemp(Val1Ty, ".atomictmp"); 395 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty)); 396 break; 397 } 398 // Fall through. 399 case AtomicExpr::AO__atomic_fetch_add: 400 case AtomicExpr::AO__atomic_fetch_sub: 401 case AtomicExpr::AO__atomic_add_fetch: 402 case AtomicExpr::AO__atomic_sub_fetch: 403 case AtomicExpr::AO__c11_atomic_store: 404 case AtomicExpr::AO__c11_atomic_exchange: 405 case AtomicExpr::AO__atomic_store_n: 406 case AtomicExpr::AO__atomic_exchange_n: 407 case AtomicExpr::AO__c11_atomic_fetch_and: 408 case AtomicExpr::AO__c11_atomic_fetch_or: 409 case AtomicExpr::AO__c11_atomic_fetch_xor: 410 case AtomicExpr::AO__atomic_fetch_and: 411 case AtomicExpr::AO__atomic_fetch_or: 412 case AtomicExpr::AO__atomic_fetch_xor: 413 case AtomicExpr::AO__atomic_fetch_nand: 414 case AtomicExpr::AO__atomic_and_fetch: 415 case AtomicExpr::AO__atomic_or_fetch: 416 case AtomicExpr::AO__atomic_xor_fetch: 417 case AtomicExpr::AO__atomic_nand_fetch: 418 Val1 = EmitValToTemp(*this, E->getVal1()); 419 break; 420 } 421 422 if (!E->getType()->isVoidType() && !Dest) 423 Dest = CreateMemTemp(E->getType(), ".atomicdst"); 424 425 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 426 if (UseLibcall) { 427 428 SmallVector<QualType, 5> Params; 429 CallArgList Args; 430 // Size is always the first parameter 431 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 432 getContext().getSizeType()); 433 // Atomic address is always the second parameter 434 Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), 435 getContext().VoidPtrTy); 436 437 const char* LibCallName; 438 QualType RetTy = getContext().VoidTy; 439 switch (E->getOp()) { 440 // There is only one libcall for compare an exchange, because there is no 441 // optimisation benefit possible from a libcall version of a weak compare 442 // and exchange. 443 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 444 // void *desired, int success, int failure) 445 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 446 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 447 case AtomicExpr::AO__atomic_compare_exchange: 448 case AtomicExpr::AO__atomic_compare_exchange_n: 449 LibCallName = "__atomic_compare_exchange"; 450 RetTy = getContext().BoolTy; 451 Args.add(RValue::get(EmitCastToVoidPtr(Val1)), 452 getContext().VoidPtrTy); 453 Args.add(RValue::get(EmitCastToVoidPtr(Val2)), 454 getContext().VoidPtrTy); 455 Args.add(RValue::get(Order), 456 getContext().IntTy); 457 Order = OrderFail; 458 break; 459 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 460 // int order) 461 case AtomicExpr::AO__c11_atomic_exchange: 462 case AtomicExpr::AO__atomic_exchange_n: 463 case AtomicExpr::AO__atomic_exchange: 464 LibCallName = "__atomic_exchange"; 465 Args.add(RValue::get(EmitCastToVoidPtr(Val1)), 466 getContext().VoidPtrTy); 467 Args.add(RValue::get(EmitCastToVoidPtr(Dest)), 468 getContext().VoidPtrTy); 469 break; 470 // void __atomic_store(size_t size, void *mem, void *val, int order) 471 case AtomicExpr::AO__c11_atomic_store: 472 case AtomicExpr::AO__atomic_store: 473 case AtomicExpr::AO__atomic_store_n: 474 LibCallName = "__atomic_store"; 475 Args.add(RValue::get(EmitCastToVoidPtr(Val1)), 476 getContext().VoidPtrTy); 477 break; 478 // void __atomic_load(size_t size, void *mem, void *return, int order) 479 case AtomicExpr::AO__c11_atomic_load: 480 case AtomicExpr::AO__atomic_load: 481 case AtomicExpr::AO__atomic_load_n: 482 LibCallName = "__atomic_load"; 483 Args.add(RValue::get(EmitCastToVoidPtr(Dest)), 484 getContext().VoidPtrTy); 485 break; 486 #if 0 487 // These are only defined for 1-16 byte integers. It is not clear what 488 // their semantics would be on anything else... 489 case AtomicExpr::Add: LibCallName = "__atomic_fetch_add_generic"; break; 490 case AtomicExpr::Sub: LibCallName = "__atomic_fetch_sub_generic"; break; 491 case AtomicExpr::And: LibCallName = "__atomic_fetch_and_generic"; break; 492 case AtomicExpr::Or: LibCallName = "__atomic_fetch_or_generic"; break; 493 case AtomicExpr::Xor: LibCallName = "__atomic_fetch_xor_generic"; break; 494 #endif 495 default: return EmitUnsupportedRValue(E, "atomic library call"); 496 } 497 // order is always the last parameter 498 Args.add(RValue::get(Order), 499 getContext().IntTy); 500 501 const CGFunctionInfo &FuncInfo = 502 CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args, 503 FunctionType::ExtInfo(), RequiredArgs::All); 504 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 505 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 506 RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 507 if (E->isCmpXChg()) 508 return Res; 509 if (E->getType()->isVoidType()) 510 return RValue::get(0); 511 return convertTempToRValue(Dest, E->getType()); 512 } 513 514 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 515 E->getOp() == AtomicExpr::AO__atomic_store || 516 E->getOp() == AtomicExpr::AO__atomic_store_n; 517 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 518 E->getOp() == AtomicExpr::AO__atomic_load || 519 E->getOp() == AtomicExpr::AO__atomic_load_n; 520 521 llvm::Type *IPtrTy = 522 llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo(); 523 llvm::Value *OrigDest = Dest; 524 Ptr = Builder.CreateBitCast(Ptr, IPtrTy); 525 if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy); 526 if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy); 527 if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy); 528 529 if (isa<llvm::ConstantInt>(Order)) { 530 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 531 switch (ord) { 532 case AO_ABI_memory_order_relaxed: 533 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 534 llvm::Monotonic); 535 break; 536 case AO_ABI_memory_order_consume: 537 case AO_ABI_memory_order_acquire: 538 if (IsStore) 539 break; // Avoid crashing on code with undefined behavior 540 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 541 llvm::Acquire); 542 break; 543 case AO_ABI_memory_order_release: 544 if (IsLoad) 545 break; // Avoid crashing on code with undefined behavior 546 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 547 llvm::Release); 548 break; 549 case AO_ABI_memory_order_acq_rel: 550 if (IsLoad || IsStore) 551 break; // Avoid crashing on code with undefined behavior 552 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 553 llvm::AcquireRelease); 554 break; 555 case AO_ABI_memory_order_seq_cst: 556 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 557 llvm::SequentiallyConsistent); 558 break; 559 default: // invalid order 560 // We should not ever get here normally, but it's hard to 561 // enforce that in general. 562 break; 563 } 564 if (E->getType()->isVoidType()) 565 return RValue::get(0); 566 return convertTempToRValue(OrigDest, E->getType()); 567 } 568 569 // Long case, when Order isn't obviously constant. 570 571 // Create all the relevant BB's 572 llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0, 573 *AcqRelBB = 0, *SeqCstBB = 0; 574 MonotonicBB = createBasicBlock("monotonic", CurFn); 575 if (!IsStore) 576 AcquireBB = createBasicBlock("acquire", CurFn); 577 if (!IsLoad) 578 ReleaseBB = createBasicBlock("release", CurFn); 579 if (!IsLoad && !IsStore) 580 AcqRelBB = createBasicBlock("acqrel", CurFn); 581 SeqCstBB = createBasicBlock("seqcst", CurFn); 582 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 583 584 // Create the switch for the split 585 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 586 // doesn't matter unless someone is crazy enough to use something that 587 // doesn't fold to a constant for the ordering. 588 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 589 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 590 591 // Emit all the different atomics 592 Builder.SetInsertPoint(MonotonicBB); 593 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 594 llvm::Monotonic); 595 Builder.CreateBr(ContBB); 596 if (!IsStore) { 597 Builder.SetInsertPoint(AcquireBB); 598 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 599 llvm::Acquire); 600 Builder.CreateBr(ContBB); 601 SI->addCase(Builder.getInt32(1), AcquireBB); 602 SI->addCase(Builder.getInt32(2), AcquireBB); 603 } 604 if (!IsLoad) { 605 Builder.SetInsertPoint(ReleaseBB); 606 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 607 llvm::Release); 608 Builder.CreateBr(ContBB); 609 SI->addCase(Builder.getInt32(3), ReleaseBB); 610 } 611 if (!IsLoad && !IsStore) { 612 Builder.SetInsertPoint(AcqRelBB); 613 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 614 llvm::AcquireRelease); 615 Builder.CreateBr(ContBB); 616 SI->addCase(Builder.getInt32(4), AcqRelBB); 617 } 618 Builder.SetInsertPoint(SeqCstBB); 619 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 620 llvm::SequentiallyConsistent); 621 Builder.CreateBr(ContBB); 622 SI->addCase(Builder.getInt32(5), SeqCstBB); 623 624 // Cleanup and return 625 Builder.SetInsertPoint(ContBB); 626 if (E->getType()->isVoidType()) 627 return RValue::get(0); 628 return convertTempToRValue(OrigDest, E->getType()); 629 } 630 631 llvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { 632 unsigned addrspace = 633 cast<llvm::PointerType>(addr->getType())->getAddressSpace(); 634 llvm::IntegerType *ty = 635 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 636 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 637 } 638 639 RValue AtomicInfo::convertTempToRValue(llvm::Value *addr, 640 AggValueSlot resultSlot) const { 641 if (EvaluationKind == TEK_Aggregate) { 642 // Nothing to do if the result is ignored. 643 if (resultSlot.isIgnored()) return resultSlot.asRValue(); 644 645 assert(resultSlot.getAddr() == addr || hasPadding()); 646 647 // In these cases, we should have emitted directly into the result slot. 648 if (!hasPadding() || resultSlot.isValueOfAtomic()) 649 return resultSlot.asRValue(); 650 651 // Otherwise, fall into the common path. 652 } 653 654 // Drill into the padding structure if we have one. 655 if (hasPadding()) 656 addr = CGF.Builder.CreateStructGEP(addr, 0); 657 658 // If we're emitting to an aggregate, copy into the result slot. 659 if (EvaluationKind == TEK_Aggregate) { 660 CGF.EmitAggregateCopy(resultSlot.getAddr(), addr, getValueType(), 661 resultSlot.isVolatile()); 662 return resultSlot.asRValue(); 663 } 664 665 // Otherwise, just convert the temporary to an r-value using the 666 // normal conversion routine. 667 return CGF.convertTempToRValue(addr, getValueType()); 668 } 669 670 /// Emit a load from an l-value of atomic type. Note that the r-value 671 /// we produce is an r-value of the atomic *value* type. 672 RValue CodeGenFunction::EmitAtomicLoad(LValue src, AggValueSlot resultSlot) { 673 AtomicInfo atomics(*this, src); 674 675 // Check whether we should use a library call. 676 if (atomics.shouldUseLibcall()) { 677 llvm::Value *tempAddr; 678 if (resultSlot.isValueOfAtomic()) { 679 assert(atomics.getEvaluationKind() == TEK_Aggregate); 680 tempAddr = resultSlot.getPaddedAtomicAddr(); 681 } else if (!resultSlot.isIgnored() && !atomics.hasPadding()) { 682 assert(atomics.getEvaluationKind() == TEK_Aggregate); 683 tempAddr = resultSlot.getAddr(); 684 } else { 685 tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); 686 } 687 688 // void __atomic_load(size_t size, void *mem, void *return, int order); 689 CallArgList args; 690 args.add(RValue::get(atomics.getAtomicSizeValue()), 691 getContext().getSizeType()); 692 args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())), 693 getContext().VoidPtrTy); 694 args.add(RValue::get(EmitCastToVoidPtr(tempAddr)), 695 getContext().VoidPtrTy); 696 args.add(RValue::get(llvm::ConstantInt::get(IntTy, 697 AO_ABI_memory_order_seq_cst)), 698 getContext().IntTy); 699 emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args); 700 701 // Produce the r-value. 702 return atomics.convertTempToRValue(tempAddr, resultSlot); 703 } 704 705 // Okay, we're doing this natively. 706 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress()); 707 llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load"); 708 load->setAtomic(llvm::SequentiallyConsistent); 709 710 // Other decoration. 711 load->setAlignment(src.getAlignment().getQuantity()); 712 if (src.isVolatileQualified()) 713 load->setVolatile(true); 714 if (src.getTBAAInfo()) 715 CGM.DecorateInstruction(load, src.getTBAAInfo()); 716 717 // Okay, turn that back into the original value type. 718 QualType valueType = atomics.getValueType(); 719 llvm::Value *result = load; 720 721 // If we're ignoring an aggregate return, don't do anything. 722 if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored()) 723 return RValue::getAggregate(0, false); 724 725 // The easiest way to do this this is to go through memory, but we 726 // try not to in some easy cases. 727 if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) { 728 llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType); 729 if (isa<llvm::IntegerType>(resultTy)) { 730 assert(result->getType() == resultTy); 731 result = EmitFromMemory(result, valueType); 732 } else if (isa<llvm::PointerType>(resultTy)) { 733 result = Builder.CreateIntToPtr(result, resultTy); 734 } else { 735 result = Builder.CreateBitCast(result, resultTy); 736 } 737 return RValue::get(result); 738 } 739 740 // Create a temporary. This needs to be big enough to hold the 741 // atomic integer. 742 llvm::Value *temp; 743 bool tempIsVolatile = false; 744 CharUnits tempAlignment; 745 if (atomics.getEvaluationKind() == TEK_Aggregate && 746 (!atomics.hasPadding() || resultSlot.isValueOfAtomic())) { 747 assert(!resultSlot.isIgnored()); 748 if (resultSlot.isValueOfAtomic()) { 749 temp = resultSlot.getPaddedAtomicAddr(); 750 tempAlignment = atomics.getAtomicAlignment(); 751 } else { 752 temp = resultSlot.getAddr(); 753 tempAlignment = atomics.getValueAlignment(); 754 } 755 tempIsVolatile = resultSlot.isVolatile(); 756 } else { 757 temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); 758 tempAlignment = atomics.getAtomicAlignment(); 759 } 760 761 // Slam the integer into the temporary. 762 llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp); 763 Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity()) 764 ->setVolatile(tempIsVolatile); 765 766 return atomics.convertTempToRValue(temp, resultSlot); 767 } 768 769 770 771 /// Copy an r-value into memory as part of storing to an atomic type. 772 /// This needs to create a bit-pattern suitable for atomic operations. 773 void AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const { 774 // If we have an r-value, the rvalue should be of the atomic type, 775 // which means that the caller is responsible for having zeroed 776 // any padding. Just do an aggregate copy of that type. 777 if (rvalue.isAggregate()) { 778 CGF.EmitAggregateCopy(dest.getAddress(), 779 rvalue.getAggregateAddr(), 780 getAtomicType(), 781 (rvalue.isVolatileQualified() 782 || dest.isVolatileQualified()), 783 dest.getAlignment()); 784 return; 785 } 786 787 // Okay, otherwise we're copying stuff. 788 789 // Zero out the buffer if necessary. 790 emitMemSetZeroIfNecessary(dest); 791 792 // Drill past the padding if present. 793 dest = projectValue(dest); 794 795 // Okay, store the rvalue in. 796 if (rvalue.isScalar()) { 797 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true); 798 } else { 799 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true); 800 } 801 } 802 803 804 /// Materialize an r-value into memory for the purposes of storing it 805 /// to an atomic type. 806 llvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { 807 // Aggregate r-values are already in memory, and EmitAtomicStore 808 // requires them to be values of the atomic type. 809 if (rvalue.isAggregate()) 810 return rvalue.getAggregateAddr(); 811 812 // Otherwise, make a temporary and materialize into it. 813 llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp"); 814 LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment()); 815 emitCopyIntoMemory(rvalue, tempLV); 816 return temp; 817 } 818 819 /// Emit a store to an l-value of atomic type. 820 /// 821 /// Note that the r-value is expected to be an r-value *of the atomic 822 /// type*; this means that for aggregate r-values, it should include 823 /// storage for any padding that was necessary. 824 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 825 bool isInit) { 826 // If this is an aggregate r-value, it should agree in type except 827 // maybe for address-space qualification. 828 assert(!rvalue.isAggregate() || 829 rvalue.getAggregateAddr()->getType()->getPointerElementType() 830 == dest.getAddress()->getType()->getPointerElementType()); 831 832 AtomicInfo atomics(*this, dest); 833 834 // If this is an initialization, just put the value there normally. 835 if (isInit) { 836 atomics.emitCopyIntoMemory(rvalue, dest); 837 return; 838 } 839 840 // Check whether we should use a library call. 841 if (atomics.shouldUseLibcall()) { 842 // Produce a source address. 843 llvm::Value *srcAddr = atomics.materializeRValue(rvalue); 844 845 // void __atomic_store(size_t size, void *mem, void *val, int order) 846 CallArgList args; 847 args.add(RValue::get(atomics.getAtomicSizeValue()), 848 getContext().getSizeType()); 849 args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())), 850 getContext().VoidPtrTy); 851 args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), 852 getContext().VoidPtrTy); 853 args.add(RValue::get(llvm::ConstantInt::get(IntTy, 854 AO_ABI_memory_order_seq_cst)), 855 getContext().IntTy); 856 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 857 return; 858 } 859 860 // Okay, we're doing this natively. 861 llvm::Value *intValue; 862 863 // If we've got a scalar value of the right size, try to avoid going 864 // through memory. 865 if (rvalue.isScalar() && !atomics.hasPadding()) { 866 llvm::Value *value = rvalue.getScalarVal(); 867 if (isa<llvm::IntegerType>(value->getType())) { 868 intValue = value; 869 } else { 870 llvm::IntegerType *inputIntTy = 871 llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits()); 872 if (isa<llvm::PointerType>(value->getType())) { 873 intValue = Builder.CreatePtrToInt(value, inputIntTy); 874 } else { 875 intValue = Builder.CreateBitCast(value, inputIntTy); 876 } 877 } 878 879 // Otherwise, we need to go through memory. 880 } else { 881 // Put the r-value in memory. 882 llvm::Value *addr = atomics.materializeRValue(rvalue); 883 884 // Cast the temporary to the atomic int type and pull a value out. 885 addr = atomics.emitCastToAtomicIntPointer(addr); 886 intValue = Builder.CreateAlignedLoad(addr, 887 atomics.getAtomicAlignment().getQuantity()); 888 } 889 890 // Do the atomic store. 891 llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress()); 892 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 893 894 // Initializations don't need to be atomic. 895 if (!isInit) store->setAtomic(llvm::SequentiallyConsistent); 896 897 // Other decoration. 898 store->setAlignment(dest.getAlignment().getQuantity()); 899 if (dest.isVolatileQualified()) 900 store->setVolatile(true); 901 if (dest.getTBAAInfo()) 902 CGM.DecorateInstruction(store, dest.getTBAAInfo()); 903 } 904 905 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 906 AtomicInfo atomics(*this, dest); 907 908 switch (atomics.getEvaluationKind()) { 909 case TEK_Scalar: { 910 llvm::Value *value = EmitScalarExpr(init); 911 atomics.emitCopyIntoMemory(RValue::get(value), dest); 912 return; 913 } 914 915 case TEK_Complex: { 916 ComplexPairTy value = EmitComplexExpr(init); 917 atomics.emitCopyIntoMemory(RValue::getComplex(value), dest); 918 return; 919 } 920 921 case TEK_Aggregate: { 922 // Memset the buffer first if there's any possibility of 923 // uninitialized internal bits. 924 atomics.emitMemSetZeroIfNecessary(dest); 925 926 // HACK: whether the initializer actually has an atomic type 927 // doesn't really seem reliable right now. 928 if (!init->getType()->isAtomicType()) { 929 dest = atomics.projectValue(dest); 930 } 931 932 // Evaluate the expression directly into the destination. 933 AggValueSlot slot = AggValueSlot::forLValue(dest, 934 AggValueSlot::IsNotDestructed, 935 AggValueSlot::DoesNotNeedGCBarriers, 936 AggValueSlot::IsNotAliased); 937 EmitAggExpr(init, slot); 938 return; 939 } 940 } 941 llvm_unreachable("bad evaluation kind"); 942 } 943
