Home | History | Annotate | Download | only in IR
      1 //===- Metadata.cpp - Implement Metadata classes --------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file implements the Metadata classes.
     11 //
     12 //===----------------------------------------------------------------------===//
     13 
     14 #include "llvm/IR/Metadata.h"
     15 #include "LLVMContextImpl.h"
     16 #include "MetadataImpl.h"
     17 #include "SymbolTableListTraitsImpl.h"
     18 #include "llvm/ADT/STLExtras.h"
     19 #include "llvm/ADT/SmallSet.h"
     20 #include "llvm/ADT/StringMap.h"
     21 #include "llvm/IR/ConstantRange.h"
     22 #include "llvm/IR/DebugInfoMetadata.h"
     23 #include "llvm/IR/Instruction.h"
     24 #include "llvm/IR/LLVMContext.h"
     25 #include "llvm/IR/Module.h"
     26 #include "llvm/IR/ValueHandle.h"
     27 
     28 using namespace llvm;
     29 
     30 MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
     31     : Value(Ty, MetadataAsValueVal), MD(MD) {
     32   track();
     33 }
     34 
     35 MetadataAsValue::~MetadataAsValue() {
     36   getType()->getContext().pImpl->MetadataAsValues.erase(MD);
     37   untrack();
     38 }
     39 
     40 /// Canonicalize metadata arguments to intrinsics.
     41 ///
     42 /// To support bitcode upgrades (and assembly semantic sugar) for \a
     43 /// MetadataAsValue, we need to canonicalize certain metadata.
     44 ///
     45 ///   - nullptr is replaced by an empty MDNode.
     46 ///   - An MDNode with a single null operand is replaced by an empty MDNode.
     47 ///   - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
     48 ///
     49 /// This maintains readability of bitcode from when metadata was a type of
     50 /// value, and these bridges were unnecessary.
     51 static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
     52                                               Metadata *MD) {
     53   if (!MD)
     54     // !{}
     55     return MDNode::get(Context, None);
     56 
     57   // Return early if this isn't a single-operand MDNode.
     58   auto *N = dyn_cast<MDNode>(MD);
     59   if (!N || N->getNumOperands() != 1)
     60     return MD;
     61 
     62   if (!N->getOperand(0))
     63     // !{}
     64     return MDNode::get(Context, None);
     65 
     66   if (auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(0)))
     67     // Look through the MDNode.
     68     return C;
     69 
     70   return MD;
     71 }
     72 
     73 MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
     74   MD = canonicalizeMetadataForValue(Context, MD);
     75   auto *&Entry = Context.pImpl->MetadataAsValues[MD];
     76   if (!Entry)
     77     Entry = new MetadataAsValue(Type::getMetadataTy(Context), MD);
     78   return Entry;
     79 }
     80 
     81 MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
     82                                               Metadata *MD) {
     83   MD = canonicalizeMetadataForValue(Context, MD);
     84   auto &Store = Context.pImpl->MetadataAsValues;
     85   return Store.lookup(MD);
     86 }
     87 
     88 void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
     89   LLVMContext &Context = getContext();
     90   MD = canonicalizeMetadataForValue(Context, MD);
     91   auto &Store = Context.pImpl->MetadataAsValues;
     92 
     93   // Stop tracking the old metadata.
     94   Store.erase(this->MD);
     95   untrack();
     96   this->MD = nullptr;
     97 
     98   // Start tracking MD, or RAUW if necessary.
     99   auto *&Entry = Store[MD];
    100   if (Entry) {
    101     replaceAllUsesWith(Entry);
    102     delete this;
    103     return;
    104   }
    105 
    106   this->MD = MD;
    107   track();
    108   Entry = this;
    109 }
    110 
    111 void MetadataAsValue::track() {
    112   if (MD)
    113     MetadataTracking::track(&MD, *MD, *this);
    114 }
    115 
    116 void MetadataAsValue::untrack() {
    117   if (MD)
    118     MetadataTracking::untrack(MD);
    119 }
    120 
    121 bool MetadataTracking::track(void *Ref, Metadata &MD, OwnerTy Owner) {
    122   assert(Ref && "Expected live reference");
    123   assert((Owner || *static_cast<Metadata **>(Ref) == &MD) &&
    124          "Reference without owner must be direct");
    125   if (auto *R = ReplaceableMetadataImpl::getOrCreate(MD)) {
    126     R->addRef(Ref, Owner);
    127     return true;
    128   }
    129   if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD)) {
    130     assert(!PH->Use && "Placeholders can only be used once");
    131     assert(!Owner && "Unexpected callback to owner");
    132     PH->Use = static_cast<Metadata **>(Ref);
    133     return true;
    134   }
    135   return false;
    136 }
    137 
    138 void MetadataTracking::untrack(void *Ref, Metadata &MD) {
    139   assert(Ref && "Expected live reference");
    140   if (auto *R = ReplaceableMetadataImpl::getIfExists(MD))
    141     R->dropRef(Ref);
    142   else if (auto *PH = dyn_cast<DistinctMDOperandPlaceholder>(&MD))
    143     PH->Use = nullptr;
    144 }
    145 
    146 bool MetadataTracking::retrack(void *Ref, Metadata &MD, void *New) {
    147   assert(Ref && "Expected live reference");
    148   assert(New && "Expected live reference");
    149   assert(Ref != New && "Expected change");
    150   if (auto *R = ReplaceableMetadataImpl::getIfExists(MD)) {
    151     R->moveRef(Ref, New, MD);
    152     return true;
    153   }
    154   assert(!isa<DistinctMDOperandPlaceholder>(MD) &&
    155          "Unexpected move of an MDOperand");
    156   assert(!isReplaceable(MD) &&
    157          "Expected un-replaceable metadata, since we didn't move a reference");
    158   return false;
    159 }
    160 
    161 bool MetadataTracking::isReplaceable(const Metadata &MD) {
    162   return ReplaceableMetadataImpl::isReplaceable(MD);
    163 }
    164 
    165 void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) {
    166   bool WasInserted =
    167       UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex)))
    168           .second;
    169   (void)WasInserted;
    170   assert(WasInserted && "Expected to add a reference");
    171 
    172   ++NextIndex;
    173   assert(NextIndex != 0 && "Unexpected overflow");
    174 }
    175 
    176 void ReplaceableMetadataImpl::dropRef(void *Ref) {
    177   bool WasErased = UseMap.erase(Ref);
    178   (void)WasErased;
    179   assert(WasErased && "Expected to drop a reference");
    180 }
    181 
    182 void ReplaceableMetadataImpl::moveRef(void *Ref, void *New,
    183                                       const Metadata &MD) {
    184   auto I = UseMap.find(Ref);
    185   assert(I != UseMap.end() && "Expected to move a reference");
    186   auto OwnerAndIndex = I->second;
    187   UseMap.erase(I);
    188   bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second;
    189   (void)WasInserted;
    190   assert(WasInserted && "Expected to add a reference");
    191 
    192   // Check that the references are direct if there's no owner.
    193   (void)MD;
    194   assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
    195          "Reference without owner must be direct");
    196   assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
    197          "Reference without owner must be direct");
    198 }
    199 
    200 void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) {
    201   if (UseMap.empty())
    202     return;
    203 
    204   // Copy out uses since UseMap will get touched below.
    205   typedef std::pair<void *, std::pair<OwnerTy, uint64_t>> UseTy;
    206   SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
    207   std::sort(Uses.begin(), Uses.end(), [](const UseTy &L, const UseTy &R) {
    208     return L.second.second < R.second.second;
    209   });
    210   for (const auto &Pair : Uses) {
    211     // Check that this Ref hasn't disappeared after RAUW (when updating a
    212     // previous Ref).
    213     if (!UseMap.count(Pair.first))
    214       continue;
    215 
    216     OwnerTy Owner = Pair.second.first;
    217     if (!Owner) {
    218       // Update unowned tracking references directly.
    219       Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
    220       Ref = MD;
    221       if (MD)
    222         MetadataTracking::track(Ref);
    223       UseMap.erase(Pair.first);
    224       continue;
    225     }
    226 
    227     // Check for MetadataAsValue.
    228     if (Owner.is<MetadataAsValue *>()) {
    229       Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD);
    230       continue;
    231     }
    232 
    233     // There's a Metadata owner -- dispatch.
    234     Metadata *OwnerMD = Owner.get<Metadata *>();
    235     switch (OwnerMD->getMetadataID()) {
    236 #define HANDLE_METADATA_LEAF(CLASS)                                            \
    237   case Metadata::CLASS##Kind:                                                  \
    238     cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD);                \
    239     continue;
    240 #include "llvm/IR/Metadata.def"
    241     default:
    242       llvm_unreachable("Invalid metadata subclass");
    243     }
    244   }
    245   assert(UseMap.empty() && "Expected all uses to be replaced");
    246 }
    247 
    248 void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) {
    249   if (UseMap.empty())
    250     return;
    251 
    252   if (!ResolveUsers) {
    253     UseMap.clear();
    254     return;
    255   }
    256 
    257   // Copy out uses since UseMap could get touched below.
    258   typedef std::pair<void *, std::pair<OwnerTy, uint64_t>> UseTy;
    259   SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
    260   std::sort(Uses.begin(), Uses.end(), [](const UseTy &L, const UseTy &R) {
    261     return L.second.second < R.second.second;
    262   });
    263   UseMap.clear();
    264   for (const auto &Pair : Uses) {
    265     auto Owner = Pair.second.first;
    266     if (!Owner)
    267       continue;
    268     if (Owner.is<MetadataAsValue *>())
    269       continue;
    270 
    271     // Resolve MDNodes that point at this.
    272     auto *OwnerMD = dyn_cast<MDNode>(Owner.get<Metadata *>());
    273     if (!OwnerMD)
    274       continue;
    275     if (OwnerMD->isResolved())
    276       continue;
    277     OwnerMD->decrementUnresolvedOperandCount();
    278   }
    279 }
    280 
    281 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getOrCreate(Metadata &MD) {
    282   if (auto *N = dyn_cast<MDNode>(&MD))
    283     return N->isResolved() ? nullptr : N->Context.getOrCreateReplaceableUses();
    284   return dyn_cast<ValueAsMetadata>(&MD);
    285 }
    286 
    287 ReplaceableMetadataImpl *ReplaceableMetadataImpl::getIfExists(Metadata &MD) {
    288   if (auto *N = dyn_cast<MDNode>(&MD))
    289     return N->isResolved() ? nullptr : N->Context.getReplaceableUses();
    290   return dyn_cast<ValueAsMetadata>(&MD);
    291 }
    292 
    293 bool ReplaceableMetadataImpl::isReplaceable(const Metadata &MD) {
    294   if (auto *N = dyn_cast<MDNode>(&MD))
    295     return !N->isResolved();
    296   return dyn_cast<ValueAsMetadata>(&MD);
    297 }
    298 
    299 static Function *getLocalFunction(Value *V) {
    300   assert(V && "Expected value");
    301   if (auto *A = dyn_cast<Argument>(V))
    302     return A->getParent();
    303   if (BasicBlock *BB = cast<Instruction>(V)->getParent())
    304     return BB->getParent();
    305   return nullptr;
    306 }
    307 
    308 ValueAsMetadata *ValueAsMetadata::get(Value *V) {
    309   assert(V && "Unexpected null Value");
    310 
    311   auto &Context = V->getContext();
    312   auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
    313   if (!Entry) {
    314     assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
    315            "Expected constant or function-local value");
    316     assert(!V->IsUsedByMD &&
    317            "Expected this to be the only metadata use");
    318     V->IsUsedByMD = true;
    319     if (auto *C = dyn_cast<Constant>(V))
    320       Entry = new ConstantAsMetadata(C);
    321     else
    322       Entry = new LocalAsMetadata(V);
    323   }
    324 
    325   return Entry;
    326 }
    327 
    328 ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
    329   assert(V && "Unexpected null Value");
    330   return V->getContext().pImpl->ValuesAsMetadata.lookup(V);
    331 }
    332 
    333 void ValueAsMetadata::handleDeletion(Value *V) {
    334   assert(V && "Expected valid value");
    335 
    336   auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
    337   auto I = Store.find(V);
    338   if (I == Store.end())
    339     return;
    340 
    341   // Remove old entry from the map.
    342   ValueAsMetadata *MD = I->second;
    343   assert(MD && "Expected valid metadata");
    344   assert(MD->getValue() == V && "Expected valid mapping");
    345   Store.erase(I);
    346 
    347   // Delete the metadata.
    348   MD->replaceAllUsesWith(nullptr);
    349   delete MD;
    350 }
    351 
    352 void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
    353   assert(From && "Expected valid value");
    354   assert(To && "Expected valid value");
    355   assert(From != To && "Expected changed value");
    356   assert(From->getType() == To->getType() && "Unexpected type change");
    357 
    358   LLVMContext &Context = From->getType()->getContext();
    359   auto &Store = Context.pImpl->ValuesAsMetadata;
    360   auto I = Store.find(From);
    361   if (I == Store.end()) {
    362     assert(!From->IsUsedByMD &&
    363            "Expected From not to be used by metadata");
    364     return;
    365   }
    366 
    367   // Remove old entry from the map.
    368   assert(From->IsUsedByMD &&
    369          "Expected From to be used by metadata");
    370   From->IsUsedByMD = false;
    371   ValueAsMetadata *MD = I->second;
    372   assert(MD && "Expected valid metadata");
    373   assert(MD->getValue() == From && "Expected valid mapping");
    374   Store.erase(I);
    375 
    376   if (isa<LocalAsMetadata>(MD)) {
    377     if (auto *C = dyn_cast<Constant>(To)) {
    378       // Local became a constant.
    379       MD->replaceAllUsesWith(ConstantAsMetadata::get(C));
    380       delete MD;
    381       return;
    382     }
    383     if (getLocalFunction(From) && getLocalFunction(To) &&
    384         getLocalFunction(From) != getLocalFunction(To)) {
    385       // Function changed.
    386       MD->replaceAllUsesWith(nullptr);
    387       delete MD;
    388       return;
    389     }
    390   } else if (!isa<Constant>(To)) {
    391     // Changed to function-local value.
    392     MD->replaceAllUsesWith(nullptr);
    393     delete MD;
    394     return;
    395   }
    396 
    397   auto *&Entry = Store[To];
    398   if (Entry) {
    399     // The target already exists.
    400     MD->replaceAllUsesWith(Entry);
    401     delete MD;
    402     return;
    403   }
    404 
    405   // Update MD in place (and update the map entry).
    406   assert(!To->IsUsedByMD &&
    407          "Expected this to be the only metadata use");
    408   To->IsUsedByMD = true;
    409   MD->V = To;
    410   Entry = MD;
    411 }
    412 
    413 //===----------------------------------------------------------------------===//
    414 // MDString implementation.
    415 //
    416 
    417 MDString *MDString::get(LLVMContext &Context, StringRef Str) {
    418   auto &Store = Context.pImpl->MDStringCache;
    419   auto I = Store.emplace_second(Str);
    420   auto &MapEntry = I.first->getValue();
    421   if (!I.second)
    422     return &MapEntry;
    423   MapEntry.Entry = &*I.first;
    424   return &MapEntry;
    425 }
    426 
    427 StringRef MDString::getString() const {
    428   assert(Entry && "Expected to find string map entry");
    429   return Entry->first();
    430 }
    431 
    432 //===----------------------------------------------------------------------===//
    433 // MDNode implementation.
    434 //
    435 
    436 // Assert that the MDNode types will not be unaligned by the objects
    437 // prepended to them.
    438 #define HANDLE_MDNODE_LEAF(CLASS)                                              \
    439   static_assert(                                                               \
    440       llvm::AlignOf<uint64_t>::Alignment >= llvm::AlignOf<CLASS>::Alignment,   \
    441       "Alignment is insufficient after objects prepended to " #CLASS);
    442 #include "llvm/IR/Metadata.def"
    443 
    444 void *MDNode::operator new(size_t Size, unsigned NumOps) {
    445   size_t OpSize = NumOps * sizeof(MDOperand);
    446   // uint64_t is the most aligned type we need support (ensured by static_assert
    447   // above)
    448   OpSize = alignTo(OpSize, llvm::alignOf<uint64_t>());
    449   void *Ptr = reinterpret_cast<char *>(::operator new(OpSize + Size)) + OpSize;
    450   MDOperand *O = static_cast<MDOperand *>(Ptr);
    451   for (MDOperand *E = O - NumOps; O != E; --O)
    452     (void)new (O - 1) MDOperand;
    453   return Ptr;
    454 }
    455 
    456 void MDNode::operator delete(void *Mem) {
    457   MDNode *N = static_cast<MDNode *>(Mem);
    458   size_t OpSize = N->NumOperands * sizeof(MDOperand);
    459   OpSize = alignTo(OpSize, llvm::alignOf<uint64_t>());
    460 
    461   MDOperand *O = static_cast<MDOperand *>(Mem);
    462   for (MDOperand *E = O - N->NumOperands; O != E; --O)
    463     (O - 1)->~MDOperand();
    464   ::operator delete(reinterpret_cast<char *>(Mem) - OpSize);
    465 }
    466 
    467 MDNode::MDNode(LLVMContext &Context, unsigned ID, StorageType Storage,
    468                ArrayRef<Metadata *> Ops1, ArrayRef<Metadata *> Ops2)
    469     : Metadata(ID, Storage), NumOperands(Ops1.size() + Ops2.size()),
    470       NumUnresolved(0), Context(Context) {
    471   unsigned Op = 0;
    472   for (Metadata *MD : Ops1)
    473     setOperand(Op++, MD);
    474   for (Metadata *MD : Ops2)
    475     setOperand(Op++, MD);
    476 
    477   if (!isUniqued())
    478     return;
    479 
    480   // Count the unresolved operands.  If there are any, RAUW support will be
    481   // added lazily on first reference.
    482   countUnresolvedOperands();
    483 }
    484 
    485 TempMDNode MDNode::clone() const {
    486   switch (getMetadataID()) {
    487   default:
    488     llvm_unreachable("Invalid MDNode subclass");
    489 #define HANDLE_MDNODE_LEAF(CLASS)                                              \
    490   case CLASS##Kind:                                                            \
    491     return cast<CLASS>(this)->cloneImpl();
    492 #include "llvm/IR/Metadata.def"
    493   }
    494 }
    495 
    496 static bool isOperandUnresolved(Metadata *Op) {
    497   if (auto *N = dyn_cast_or_null<MDNode>(Op))
    498     return !N->isResolved();
    499   return false;
    500 }
    501 
    502 void MDNode::countUnresolvedOperands() {
    503   assert(NumUnresolved == 0 && "Expected unresolved ops to be uncounted");
    504   assert(isUniqued() && "Expected this to be uniqued");
    505   NumUnresolved = count_if(operands(), isOperandUnresolved);
    506 }
    507 
    508 void MDNode::makeUniqued() {
    509   assert(isTemporary() && "Expected this to be temporary");
    510   assert(!isResolved() && "Expected this to be unresolved");
    511 
    512   // Enable uniquing callbacks.
    513   for (auto &Op : mutable_operands())
    514     Op.reset(Op.get(), this);
    515 
    516   // Make this 'uniqued'.
    517   Storage = Uniqued;
    518   countUnresolvedOperands();
    519   if (!NumUnresolved) {
    520     dropReplaceableUses();
    521     assert(isResolved() && "Expected this to be resolved");
    522   }
    523 
    524   assert(isUniqued() && "Expected this to be uniqued");
    525 }
    526 
    527 void MDNode::makeDistinct() {
    528   assert(isTemporary() && "Expected this to be temporary");
    529   assert(!isResolved() && "Expected this to be unresolved");
    530 
    531   // Drop RAUW support and store as a distinct node.
    532   dropReplaceableUses();
    533   storeDistinctInContext();
    534 
    535   assert(isDistinct() && "Expected this to be distinct");
    536   assert(isResolved() && "Expected this to be resolved");
    537 }
    538 
    539 void MDNode::resolve() {
    540   assert(isUniqued() && "Expected this to be uniqued");
    541   assert(!isResolved() && "Expected this to be unresolved");
    542 
    543   NumUnresolved = 0;
    544   dropReplaceableUses();
    545 
    546   assert(isResolved() && "Expected this to be resolved");
    547 }
    548 
    549 void MDNode::dropReplaceableUses() {
    550   assert(!NumUnresolved && "Unexpected unresolved operand");
    551 
    552   // Drop any RAUW support.
    553   if (Context.hasReplaceableUses())
    554     Context.takeReplaceableUses()->resolveAllUses();
    555 }
    556 
    557 void MDNode::resolveAfterOperandChange(Metadata *Old, Metadata *New) {
    558   assert(isUniqued() && "Expected this to be uniqued");
    559   assert(NumUnresolved != 0 && "Expected unresolved operands");
    560 
    561   // Check if an operand was resolved.
    562   if (!isOperandUnresolved(Old)) {
    563     if (isOperandUnresolved(New))
    564       // An operand was un-resolved!
    565       ++NumUnresolved;
    566   } else if (!isOperandUnresolved(New))
    567     decrementUnresolvedOperandCount();
    568 }
    569 
    570 void MDNode::decrementUnresolvedOperandCount() {
    571   assert(!isResolved() && "Expected this to be unresolved");
    572   if (isTemporary())
    573     return;
    574 
    575   assert(isUniqued() && "Expected this to be uniqued");
    576   if (--NumUnresolved)
    577     return;
    578 
    579   // Last unresolved operand has just been resolved.
    580   dropReplaceableUses();
    581   assert(isResolved() && "Expected this to become resolved");
    582 }
    583 
    584 void MDNode::resolveCycles() {
    585   if (isResolved())
    586     return;
    587 
    588   // Resolve this node immediately.
    589   resolve();
    590 
    591   // Resolve all operands.
    592   for (const auto &Op : operands()) {
    593     auto *N = dyn_cast_or_null<MDNode>(Op);
    594     if (!N)
    595       continue;
    596 
    597     assert(!N->isTemporary() &&
    598            "Expected all forward declarations to be resolved");
    599     if (!N->isResolved())
    600       N->resolveCycles();
    601   }
    602 }
    603 
    604 static bool hasSelfReference(MDNode *N) {
    605   for (Metadata *MD : N->operands())
    606     if (MD == N)
    607       return true;
    608   return false;
    609 }
    610 
    611 MDNode *MDNode::replaceWithPermanentImpl() {
    612   switch (getMetadataID()) {
    613   default:
    614     // If this type isn't uniquable, replace with a distinct node.
    615     return replaceWithDistinctImpl();
    616 
    617 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS)                                    \
    618   case CLASS##Kind:                                                            \
    619     break;
    620 #include "llvm/IR/Metadata.def"
    621   }
    622 
    623   // Even if this type is uniquable, self-references have to be distinct.
    624   if (hasSelfReference(this))
    625     return replaceWithDistinctImpl();
    626   return replaceWithUniquedImpl();
    627 }
    628 
    629 MDNode *MDNode::replaceWithUniquedImpl() {
    630   // Try to uniquify in place.
    631   MDNode *UniquedNode = uniquify();
    632 
    633   if (UniquedNode == this) {
    634     makeUniqued();
    635     return this;
    636   }
    637 
    638   // Collision, so RAUW instead.
    639   replaceAllUsesWith(UniquedNode);
    640   deleteAsSubclass();
    641   return UniquedNode;
    642 }
    643 
    644 MDNode *MDNode::replaceWithDistinctImpl() {
    645   makeDistinct();
    646   return this;
    647 }
    648 
    649 void MDTuple::recalculateHash() {
    650   setHash(MDTupleInfo::KeyTy::calculateHash(this));
    651 }
    652 
    653 void MDNode::dropAllReferences() {
    654   for (unsigned I = 0, E = NumOperands; I != E; ++I)
    655     setOperand(I, nullptr);
    656   if (Context.hasReplaceableUses()) {
    657     Context.getReplaceableUses()->resolveAllUses(/* ResolveUsers */ false);
    658     (void)Context.takeReplaceableUses();
    659   }
    660 }
    661 
    662 void MDNode::handleChangedOperand(void *Ref, Metadata *New) {
    663   unsigned Op = static_cast<MDOperand *>(Ref) - op_begin();
    664   assert(Op < getNumOperands() && "Expected valid operand");
    665 
    666   if (!isUniqued()) {
    667     // This node is not uniqued.  Just set the operand and be done with it.
    668     setOperand(Op, New);
    669     return;
    670   }
    671 
    672   // This node is uniqued.
    673   eraseFromStore();
    674 
    675   Metadata *Old = getOperand(Op);
    676   setOperand(Op, New);
    677 
    678   // Drop uniquing for self-reference cycles.
    679   if (New == this) {
    680     if (!isResolved())
    681       resolve();
    682     storeDistinctInContext();
    683     return;
    684   }
    685 
    686   // Re-unique the node.
    687   auto *Uniqued = uniquify();
    688   if (Uniqued == this) {
    689     if (!isResolved())
    690       resolveAfterOperandChange(Old, New);
    691     return;
    692   }
    693 
    694   // Collision.
    695   if (!isResolved()) {
    696     // Still unresolved, so RAUW.
    697     //
    698     // First, clear out all operands to prevent any recursion (similar to
    699     // dropAllReferences(), but we still need the use-list).
    700     for (unsigned O = 0, E = getNumOperands(); O != E; ++O)
    701       setOperand(O, nullptr);
    702     if (Context.hasReplaceableUses())
    703       Context.getReplaceableUses()->replaceAllUsesWith(Uniqued);
    704     deleteAsSubclass();
    705     return;
    706   }
    707 
    708   // Store in non-uniqued form if RAUW isn't possible.
    709   storeDistinctInContext();
    710 }
    711 
    712 void MDNode::deleteAsSubclass() {
    713   switch (getMetadataID()) {
    714   default:
    715     llvm_unreachable("Invalid subclass of MDNode");
    716 #define HANDLE_MDNODE_LEAF(CLASS)                                              \
    717   case CLASS##Kind:                                                            \
    718     delete cast<CLASS>(this);                                                  \
    719     break;
    720 #include "llvm/IR/Metadata.def"
    721   }
    722 }
    723 
    724 template <class T, class InfoT>
    725 static T *uniquifyImpl(T *N, DenseSet<T *, InfoT> &Store) {
    726   if (T *U = getUniqued(Store, N))
    727     return U;
    728 
    729   Store.insert(N);
    730   return N;
    731 }
    732 
    733 template <class NodeTy> struct MDNode::HasCachedHash {
    734   typedef char Yes[1];
    735   typedef char No[2];
    736   template <class U, U Val> struct SFINAE {};
    737 
    738   template <class U>
    739   static Yes &check(SFINAE<void (U::*)(unsigned), &U::setHash> *);
    740   template <class U> static No &check(...);
    741 
    742   static const bool value = sizeof(check<NodeTy>(nullptr)) == sizeof(Yes);
    743 };
    744 
    745 MDNode *MDNode::uniquify() {
    746   assert(!hasSelfReference(this) && "Cannot uniquify a self-referencing node");
    747 
    748   // Try to insert into uniquing store.
    749   switch (getMetadataID()) {
    750   default:
    751     llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
    752 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS)                                    \
    753   case CLASS##Kind: {                                                          \
    754     CLASS *SubclassThis = cast<CLASS>(this);                                   \
    755     std::integral_constant<bool, HasCachedHash<CLASS>::value>                  \
    756         ShouldRecalculateHash;                                                 \
    757     dispatchRecalculateHash(SubclassThis, ShouldRecalculateHash);              \
    758     return uniquifyImpl(SubclassThis, getContext().pImpl->CLASS##s);           \
    759   }
    760 #include "llvm/IR/Metadata.def"
    761   }
    762 }
    763 
    764 void MDNode::eraseFromStore() {
    765   switch (getMetadataID()) {
    766   default:
    767     llvm_unreachable("Invalid or non-uniquable subclass of MDNode");
    768 #define HANDLE_MDNODE_LEAF_UNIQUABLE(CLASS)                                    \
    769   case CLASS##Kind:                                                            \
    770     getContext().pImpl->CLASS##s.erase(cast<CLASS>(this));                     \
    771     break;
    772 #include "llvm/IR/Metadata.def"
    773   }
    774 }
    775 
    776 MDTuple *MDTuple::getImpl(LLVMContext &Context, ArrayRef<Metadata *> MDs,
    777                           StorageType Storage, bool ShouldCreate) {
    778   unsigned Hash = 0;
    779   if (Storage == Uniqued) {
    780     MDTupleInfo::KeyTy Key(MDs);
    781     if (auto *N = getUniqued(Context.pImpl->MDTuples, Key))
    782       return N;
    783     if (!ShouldCreate)
    784       return nullptr;
    785     Hash = Key.getHash();
    786   } else {
    787     assert(ShouldCreate && "Expected non-uniqued nodes to always be created");
    788   }
    789 
    790   return storeImpl(new (MDs.size()) MDTuple(Context, Storage, Hash, MDs),
    791                    Storage, Context.pImpl->MDTuples);
    792 }
    793 
    794 void MDNode::deleteTemporary(MDNode *N) {
    795   assert(N->isTemporary() && "Expected temporary node");
    796   N->replaceAllUsesWith(nullptr);
    797   N->deleteAsSubclass();
    798 }
    799 
    800 void MDNode::storeDistinctInContext() {
    801   assert(!Context.hasReplaceableUses() && "Unexpected replaceable uses");
    802   assert(!NumUnresolved && "Unexpected unresolved nodes");
    803   Storage = Distinct;
    804   assert(isResolved() && "Expected this to be resolved");
    805 
    806   // Reset the hash.
    807   switch (getMetadataID()) {
    808   default:
    809     llvm_unreachable("Invalid subclass of MDNode");
    810 #define HANDLE_MDNODE_LEAF(CLASS)                                              \
    811   case CLASS##Kind: {                                                          \
    812     std::integral_constant<bool, HasCachedHash<CLASS>::value> ShouldResetHash; \
    813     dispatchResetHash(cast<CLASS>(this), ShouldResetHash);                     \
    814     break;                                                                     \
    815   }
    816 #include "llvm/IR/Metadata.def"
    817   }
    818 
    819   getContext().pImpl->DistinctMDNodes.push_back(this);
    820 }
    821 
    822 void MDNode::replaceOperandWith(unsigned I, Metadata *New) {
    823   if (getOperand(I) == New)
    824     return;
    825 
    826   if (!isUniqued()) {
    827     setOperand(I, New);
    828     return;
    829   }
    830 
    831   handleChangedOperand(mutable_begin() + I, New);
    832 }
    833 
    834 void MDNode::setOperand(unsigned I, Metadata *New) {
    835   assert(I < NumOperands);
    836   mutable_begin()[I].reset(New, isUniqued() ? this : nullptr);
    837 }
    838 
    839 /// Get a node or a self-reference that looks like it.
    840 ///
    841 /// Special handling for finding self-references, for use by \a
    842 /// MDNode::concatenate() and \a MDNode::intersect() to maintain behaviour from
    843 /// when self-referencing nodes were still uniqued.  If the first operand has
    844 /// the same operands as \c Ops, return the first operand instead.
    845 static MDNode *getOrSelfReference(LLVMContext &Context,
    846                                   ArrayRef<Metadata *> Ops) {
    847   if (!Ops.empty())
    848     if (MDNode *N = dyn_cast_or_null<MDNode>(Ops[0]))
    849       if (N->getNumOperands() == Ops.size() && N == N->getOperand(0)) {
    850         for (unsigned I = 1, E = Ops.size(); I != E; ++I)
    851           if (Ops[I] != N->getOperand(I))
    852             return MDNode::get(Context, Ops);
    853         return N;
    854       }
    855 
    856   return MDNode::get(Context, Ops);
    857 }
    858 
    859 MDNode *MDNode::concatenate(MDNode *A, MDNode *B) {
    860   if (!A)
    861     return B;
    862   if (!B)
    863     return A;
    864 
    865   SmallVector<Metadata *, 4> MDs;
    866   MDs.reserve(A->getNumOperands() + B->getNumOperands());
    867   MDs.append(A->op_begin(), A->op_end());
    868   MDs.append(B->op_begin(), B->op_end());
    869 
    870   // FIXME: This preserves long-standing behaviour, but is it really the right
    871   // behaviour?  Or was that an unintended side-effect of node uniquing?
    872   return getOrSelfReference(A->getContext(), MDs);
    873 }
    874 
    875 MDNode *MDNode::intersect(MDNode *A, MDNode *B) {
    876   if (!A || !B)
    877     return nullptr;
    878 
    879   SmallVector<Metadata *, 4> MDs;
    880   for (Metadata *MD : A->operands())
    881     if (std::find(B->op_begin(), B->op_end(), MD) != B->op_end())
    882       MDs.push_back(MD);
    883 
    884   // FIXME: This preserves long-standing behaviour, but is it really the right
    885   // behaviour?  Or was that an unintended side-effect of node uniquing?
    886   return getOrSelfReference(A->getContext(), MDs);
    887 }
    888 
    889 MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
    890   if (!A || !B)
    891     return nullptr;
    892 
    893   SmallVector<Metadata *, 4> MDs(B->op_begin(), B->op_end());
    894   for (Metadata *MD : A->operands())
    895     if (std::find(B->op_begin(), B->op_end(), MD) == B->op_end())
    896       MDs.push_back(MD);
    897 
    898   // FIXME: This preserves long-standing behaviour, but is it really the right
    899   // behaviour?  Or was that an unintended side-effect of node uniquing?
    900   return getOrSelfReference(A->getContext(), MDs);
    901 }
    902 
    903 MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
    904   if (!A || !B)
    905     return nullptr;
    906 
    907   APFloat AVal = mdconst::extract<ConstantFP>(A->getOperand(0))->getValueAPF();
    908   APFloat BVal = mdconst::extract<ConstantFP>(B->getOperand(0))->getValueAPF();
    909   if (AVal.compare(BVal) == APFloat::cmpLessThan)
    910     return A;
    911   return B;
    912 }
    913 
    914 static bool isContiguous(const ConstantRange &A, const ConstantRange &B) {
    915   return A.getUpper() == B.getLower() || A.getLower() == B.getUpper();
    916 }
    917 
    918 static bool canBeMerged(const ConstantRange &A, const ConstantRange &B) {
    919   return !A.intersectWith(B).isEmptySet() || isContiguous(A, B);
    920 }
    921 
    922 static bool tryMergeRange(SmallVectorImpl<ConstantInt *> &EndPoints,
    923                           ConstantInt *Low, ConstantInt *High) {
    924   ConstantRange NewRange(Low->getValue(), High->getValue());
    925   unsigned Size = EndPoints.size();
    926   APInt LB = EndPoints[Size - 2]->getValue();
    927   APInt LE = EndPoints[Size - 1]->getValue();
    928   ConstantRange LastRange(LB, LE);
    929   if (canBeMerged(NewRange, LastRange)) {
    930     ConstantRange Union = LastRange.unionWith(NewRange);
    931     Type *Ty = High->getType();
    932     EndPoints[Size - 2] =
    933         cast<ConstantInt>(ConstantInt::get(Ty, Union.getLower()));
    934     EndPoints[Size - 1] =
    935         cast<ConstantInt>(ConstantInt::get(Ty, Union.getUpper()));
    936     return true;
    937   }
    938   return false;
    939 }
    940 
    941 static void addRange(SmallVectorImpl<ConstantInt *> &EndPoints,
    942                      ConstantInt *Low, ConstantInt *High) {
    943   if (!EndPoints.empty())
    944     if (tryMergeRange(EndPoints, Low, High))
    945       return;
    946 
    947   EndPoints.push_back(Low);
    948   EndPoints.push_back(High);
    949 }
    950 
    951 MDNode *MDNode::getMostGenericRange(MDNode *A, MDNode *B) {
    952   // Given two ranges, we want to compute the union of the ranges. This
    953   // is slightly complitade by having to combine the intervals and merge
    954   // the ones that overlap.
    955 
    956   if (!A || !B)
    957     return nullptr;
    958 
    959   if (A == B)
    960     return A;
    961 
    962   // First, walk both lists in older of the lower boundary of each interval.
    963   // At each step, try to merge the new interval to the last one we adedd.
    964   SmallVector<ConstantInt *, 4> EndPoints;
    965   int AI = 0;
    966   int BI = 0;
    967   int AN = A->getNumOperands() / 2;
    968   int BN = B->getNumOperands() / 2;
    969   while (AI < AN && BI < BN) {
    970     ConstantInt *ALow = mdconst::extract<ConstantInt>(A->getOperand(2 * AI));
    971     ConstantInt *BLow = mdconst::extract<ConstantInt>(B->getOperand(2 * BI));
    972 
    973     if (ALow->getValue().slt(BLow->getValue())) {
    974       addRange(EndPoints, ALow,
    975                mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1)));
    976       ++AI;
    977     } else {
    978       addRange(EndPoints, BLow,
    979                mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1)));
    980       ++BI;
    981     }
    982   }
    983   while (AI < AN) {
    984     addRange(EndPoints, mdconst::extract<ConstantInt>(A->getOperand(2 * AI)),
    985              mdconst::extract<ConstantInt>(A->getOperand(2 * AI + 1)));
    986     ++AI;
    987   }
    988   while (BI < BN) {
    989     addRange(EndPoints, mdconst::extract<ConstantInt>(B->getOperand(2 * BI)),
    990              mdconst::extract<ConstantInt>(B->getOperand(2 * BI + 1)));
    991     ++BI;
    992   }
    993 
    994   // If we have more than 2 ranges (4 endpoints) we have to try to merge
    995   // the last and first ones.
    996   unsigned Size = EndPoints.size();
    997   if (Size > 4) {
    998     ConstantInt *FB = EndPoints[0];
    999     ConstantInt *FE = EndPoints[1];
   1000     if (tryMergeRange(EndPoints, FB, FE)) {
   1001       for (unsigned i = 0; i < Size - 2; ++i) {
   1002         EndPoints[i] = EndPoints[i + 2];
   1003       }
   1004       EndPoints.resize(Size - 2);
   1005     }
   1006   }
   1007 
   1008   // If in the end we have a single range, it is possible that it is now the
   1009   // full range. Just drop the metadata in that case.
   1010   if (EndPoints.size() == 2) {
   1011     ConstantRange Range(EndPoints[0]->getValue(), EndPoints[1]->getValue());
   1012     if (Range.isFullSet())
   1013       return nullptr;
   1014   }
   1015 
   1016   SmallVector<Metadata *, 4> MDs;
   1017   MDs.reserve(EndPoints.size());
   1018   for (auto *I : EndPoints)
   1019     MDs.push_back(ConstantAsMetadata::get(I));
   1020   return MDNode::get(A->getContext(), MDs);
   1021 }
   1022 
   1023 MDNode *MDNode::getMostGenericAlignmentOrDereferenceable(MDNode *A, MDNode *B) {
   1024   if (!A || !B)
   1025     return nullptr;
   1026 
   1027   ConstantInt *AVal = mdconst::extract<ConstantInt>(A->getOperand(0));
   1028   ConstantInt *BVal = mdconst::extract<ConstantInt>(B->getOperand(0));
   1029   if (AVal->getZExtValue() < BVal->getZExtValue())
   1030     return A;
   1031   return B;
   1032 }
   1033 
   1034 //===----------------------------------------------------------------------===//
   1035 // NamedMDNode implementation.
   1036 //
   1037 
   1038 static SmallVector<TrackingMDRef, 4> &getNMDOps(void *Operands) {
   1039   return *(SmallVector<TrackingMDRef, 4> *)Operands;
   1040 }
   1041 
   1042 NamedMDNode::NamedMDNode(const Twine &N)
   1043     : Name(N.str()), Parent(nullptr),
   1044       Operands(new SmallVector<TrackingMDRef, 4>()) {}
   1045 
   1046 NamedMDNode::~NamedMDNode() {
   1047   dropAllReferences();
   1048   delete &getNMDOps(Operands);
   1049 }
   1050 
   1051 unsigned NamedMDNode::getNumOperands() const {
   1052   return (unsigned)getNMDOps(Operands).size();
   1053 }
   1054 
   1055 MDNode *NamedMDNode::getOperand(unsigned i) const {
   1056   assert(i < getNumOperands() && "Invalid Operand number!");
   1057   auto *N = getNMDOps(Operands)[i].get();
   1058   return cast_or_null<MDNode>(N);
   1059 }
   1060 
   1061 void NamedMDNode::addOperand(MDNode *M) { getNMDOps(Operands).emplace_back(M); }
   1062 
   1063 void NamedMDNode::setOperand(unsigned I, MDNode *New) {
   1064   assert(I < getNumOperands() && "Invalid operand number");
   1065   getNMDOps(Operands)[I].reset(New);
   1066 }
   1067 
   1068 void NamedMDNode::eraseFromParent() {
   1069   getParent()->eraseNamedMetadata(this);
   1070 }
   1071 
   1072 void NamedMDNode::dropAllReferences() {
   1073   getNMDOps(Operands).clear();
   1074 }
   1075 
   1076 StringRef NamedMDNode::getName() const {
   1077   return StringRef(Name);
   1078 }
   1079 
   1080 //===----------------------------------------------------------------------===//
   1081 // Instruction Metadata method implementations.
   1082 //
   1083 void MDAttachmentMap::set(unsigned ID, MDNode &MD) {
   1084   for (auto &I : Attachments)
   1085     if (I.first == ID) {
   1086       I.second.reset(&MD);
   1087       return;
   1088     }
   1089   Attachments.emplace_back(std::piecewise_construct, std::make_tuple(ID),
   1090                            std::make_tuple(&MD));
   1091 }
   1092 
   1093 void MDAttachmentMap::erase(unsigned ID) {
   1094   if (empty())
   1095     return;
   1096 
   1097   // Common case is one/last value.
   1098   if (Attachments.back().first == ID) {
   1099     Attachments.pop_back();
   1100     return;
   1101   }
   1102 
   1103   for (auto I = Attachments.begin(), E = std::prev(Attachments.end()); I != E;
   1104        ++I)
   1105     if (I->first == ID) {
   1106       *I = std::move(Attachments.back());
   1107       Attachments.pop_back();
   1108       return;
   1109     }
   1110 }
   1111 
   1112 MDNode *MDAttachmentMap::lookup(unsigned ID) const {
   1113   for (const auto &I : Attachments)
   1114     if (I.first == ID)
   1115       return I.second;
   1116   return nullptr;
   1117 }
   1118 
   1119 void MDAttachmentMap::getAll(
   1120     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
   1121   Result.append(Attachments.begin(), Attachments.end());
   1122 
   1123   // Sort the resulting array so it is stable.
   1124   if (Result.size() > 1)
   1125     array_pod_sort(Result.begin(), Result.end());
   1126 }
   1127 
   1128 void MDGlobalAttachmentMap::insert(unsigned ID, MDNode &MD) {
   1129   Attachments.push_back({ID, TrackingMDNodeRef(&MD)});
   1130 }
   1131 
   1132 void MDGlobalAttachmentMap::get(unsigned ID,
   1133                                 SmallVectorImpl<MDNode *> &Result) {
   1134   for (auto A : Attachments)
   1135     if (A.MDKind == ID)
   1136       Result.push_back(A.Node);
   1137 }
   1138 
   1139 void MDGlobalAttachmentMap::erase(unsigned ID) {
   1140   auto Follower = Attachments.begin();
   1141   for (auto Leader = Attachments.begin(), E = Attachments.end(); Leader != E;
   1142        ++Leader) {
   1143     if (Leader->MDKind != ID) {
   1144       if (Follower != Leader)
   1145         *Follower = std::move(*Leader);
   1146       ++Follower;
   1147     }
   1148   }
   1149   Attachments.resize(Follower - Attachments.begin());
   1150 }
   1151 
   1152 void MDGlobalAttachmentMap::getAll(
   1153     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
   1154   for (auto &A : Attachments)
   1155     Result.emplace_back(A.MDKind, A.Node);
   1156 
   1157   // Sort the resulting array so it is stable with respect to metadata IDs. We
   1158   // need to preserve the original insertion order though.
   1159   std::stable_sort(
   1160       Result.begin(), Result.end(),
   1161       [](const std::pair<unsigned, MDNode *> &A,
   1162          const std::pair<unsigned, MDNode *> &B) { return A.first < B.first; });
   1163 }
   1164 
   1165 void Instruction::setMetadata(StringRef Kind, MDNode *Node) {
   1166   if (!Node && !hasMetadata())
   1167     return;
   1168   setMetadata(getContext().getMDKindID(Kind), Node);
   1169 }
   1170 
   1171 MDNode *Instruction::getMetadataImpl(StringRef Kind) const {
   1172   return getMetadataImpl(getContext().getMDKindID(Kind));
   1173 }
   1174 
   1175 void Instruction::dropUnknownNonDebugMetadata(ArrayRef<unsigned> KnownIDs) {
   1176   SmallSet<unsigned, 5> KnownSet;
   1177   KnownSet.insert(KnownIDs.begin(), KnownIDs.end());
   1178 
   1179   if (!hasMetadataHashEntry())
   1180     return; // Nothing to remove!
   1181 
   1182   auto &InstructionMetadata = getContext().pImpl->InstructionMetadata;
   1183 
   1184   if (KnownSet.empty()) {
   1185     // Just drop our entry at the store.
   1186     InstructionMetadata.erase(this);
   1187     setHasMetadataHashEntry(false);
   1188     return;
   1189   }
   1190 
   1191   auto &Info = InstructionMetadata[this];
   1192   Info.remove_if([&KnownSet](const std::pair<unsigned, TrackingMDNodeRef> &I) {
   1193     return !KnownSet.count(I.first);
   1194   });
   1195 
   1196   if (Info.empty()) {
   1197     // Drop our entry at the store.
   1198     InstructionMetadata.erase(this);
   1199     setHasMetadataHashEntry(false);
   1200   }
   1201 }
   1202 
   1203 void Instruction::setMetadata(unsigned KindID, MDNode *Node) {
   1204   if (!Node && !hasMetadata())
   1205     return;
   1206 
   1207   // Handle 'dbg' as a special case since it is not stored in the hash table.
   1208   if (KindID == LLVMContext::MD_dbg) {
   1209     DbgLoc = DebugLoc(Node);
   1210     return;
   1211   }
   1212 
   1213   // Handle the case when we're adding/updating metadata on an instruction.
   1214   if (Node) {
   1215     auto &Info = getContext().pImpl->InstructionMetadata[this];
   1216     assert(!Info.empty() == hasMetadataHashEntry() &&
   1217            "HasMetadata bit is wonked");
   1218     if (Info.empty())
   1219       setHasMetadataHashEntry(true);
   1220     Info.set(KindID, *Node);
   1221     return;
   1222   }
   1223 
   1224   // Otherwise, we're removing metadata from an instruction.
   1225   assert((hasMetadataHashEntry() ==
   1226           (getContext().pImpl->InstructionMetadata.count(this) > 0)) &&
   1227          "HasMetadata bit out of date!");
   1228   if (!hasMetadataHashEntry())
   1229     return;  // Nothing to remove!
   1230   auto &Info = getContext().pImpl->InstructionMetadata[this];
   1231 
   1232   // Handle removal of an existing value.
   1233   Info.erase(KindID);
   1234 
   1235   if (!Info.empty())
   1236     return;
   1237 
   1238   getContext().pImpl->InstructionMetadata.erase(this);
   1239   setHasMetadataHashEntry(false);
   1240 }
   1241 
   1242 void Instruction::setAAMetadata(const AAMDNodes &N) {
   1243   setMetadata(LLVMContext::MD_tbaa, N.TBAA);
   1244   setMetadata(LLVMContext::MD_alias_scope, N.Scope);
   1245   setMetadata(LLVMContext::MD_noalias, N.NoAlias);
   1246 }
   1247 
   1248 MDNode *Instruction::getMetadataImpl(unsigned KindID) const {
   1249   // Handle 'dbg' as a special case since it is not stored in the hash table.
   1250   if (KindID == LLVMContext::MD_dbg)
   1251     return DbgLoc.getAsMDNode();
   1252 
   1253   if (!hasMetadataHashEntry())
   1254     return nullptr;
   1255   auto &Info = getContext().pImpl->InstructionMetadata[this];
   1256   assert(!Info.empty() && "bit out of sync with hash table");
   1257 
   1258   return Info.lookup(KindID);
   1259 }
   1260 
   1261 void Instruction::getAllMetadataImpl(
   1262     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
   1263   Result.clear();
   1264 
   1265   // Handle 'dbg' as a special case since it is not stored in the hash table.
   1266   if (DbgLoc) {
   1267     Result.push_back(
   1268         std::make_pair((unsigned)LLVMContext::MD_dbg, DbgLoc.getAsMDNode()));
   1269     if (!hasMetadataHashEntry()) return;
   1270   }
   1271 
   1272   assert(hasMetadataHashEntry() &&
   1273          getContext().pImpl->InstructionMetadata.count(this) &&
   1274          "Shouldn't have called this");
   1275   const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second;
   1276   assert(!Info.empty() && "Shouldn't have called this");
   1277   Info.getAll(Result);
   1278 }
   1279 
   1280 void Instruction::getAllMetadataOtherThanDebugLocImpl(
   1281     SmallVectorImpl<std::pair<unsigned, MDNode *>> &Result) const {
   1282   Result.clear();
   1283   assert(hasMetadataHashEntry() &&
   1284          getContext().pImpl->InstructionMetadata.count(this) &&
   1285          "Shouldn't have called this");
   1286   const auto &Info = getContext().pImpl->InstructionMetadata.find(this)->second;
   1287   assert(!Info.empty() && "Shouldn't have called this");
   1288   Info.getAll(Result);
   1289 }
   1290 
   1291 bool Instruction::extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) {
   1292   assert((getOpcode() == Instruction::Br ||
   1293           getOpcode() == Instruction::Select) &&
   1294          "Looking for branch weights on something besides branch or select");
   1295 
   1296   auto *ProfileData = getMetadata(LLVMContext::MD_prof);
   1297   if (!ProfileData || ProfileData->getNumOperands() != 3)
   1298     return false;
   1299 
   1300   auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));
   1301   if (!ProfDataName || !ProfDataName->getString().equals("branch_weights"))
   1302     return false;
   1303 
   1304   auto *CITrue = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(1));
   1305   auto *CIFalse = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(2));
   1306   if (!CITrue || !CIFalse)
   1307     return false;
   1308 
   1309   TrueVal = CITrue->getValue().getZExtValue();
   1310   FalseVal = CIFalse->getValue().getZExtValue();
   1311 
   1312   return true;
   1313 }
   1314 
   1315 bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) {
   1316   assert((getOpcode() == Instruction::Br ||
   1317           getOpcode() == Instruction::Select ||
   1318           getOpcode() == Instruction::Call ||
   1319           getOpcode() == Instruction::Invoke) &&
   1320          "Looking for branch weights on something besides branch");
   1321 
   1322   TotalVal = 0;
   1323   auto *ProfileData = getMetadata(LLVMContext::MD_prof);
   1324   if (!ProfileData)
   1325     return false;
   1326 
   1327   auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0));
   1328   if (!ProfDataName || !ProfDataName->getString().equals("branch_weights"))
   1329     return false;
   1330 
   1331   TotalVal = 0;
   1332   for (unsigned i = 1; i < ProfileData->getNumOperands(); i++) {
   1333     auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i));
   1334     if (!V)
   1335       return false;
   1336     TotalVal += V->getValue().getZExtValue();
   1337   }
   1338   return true;
   1339 }
   1340 
   1341 void Instruction::clearMetadataHashEntries() {
   1342   assert(hasMetadataHashEntry() && "Caller should check");
   1343   getContext().pImpl->InstructionMetadata.erase(this);
   1344   setHasMetadataHashEntry(false);
   1345 }
   1346 
   1347 void GlobalObject::getMetadata(unsigned KindID,
   1348                                SmallVectorImpl<MDNode *> &MDs) const {
   1349   if (hasMetadata())
   1350     getContext().pImpl->GlobalObjectMetadata[this].get(KindID, MDs);
   1351 }
   1352 
   1353 void GlobalObject::getMetadata(StringRef Kind,
   1354                                SmallVectorImpl<MDNode *> &MDs) const {
   1355   if (hasMetadata())
   1356     getMetadata(getContext().getMDKindID(Kind), MDs);
   1357 }
   1358 
   1359 void GlobalObject::addMetadata(unsigned KindID, MDNode &MD) {
   1360   if (!hasMetadata())
   1361     setHasMetadataHashEntry(true);
   1362 
   1363   getContext().pImpl->GlobalObjectMetadata[this].insert(KindID, MD);
   1364 }
   1365 
   1366 void GlobalObject::addMetadata(StringRef Kind, MDNode &MD) {
   1367   addMetadata(getContext().getMDKindID(Kind), MD);
   1368 }
   1369 
   1370 void GlobalObject::eraseMetadata(unsigned KindID) {
   1371   // Nothing to unset.
   1372   if (!hasMetadata())
   1373     return;
   1374 
   1375   auto &Store = getContext().pImpl->GlobalObjectMetadata[this];
   1376   Store.erase(KindID);
   1377   if (Store.empty())
   1378     clearMetadata();
   1379 }
   1380 
   1381 void GlobalObject::getAllMetadata(
   1382     SmallVectorImpl<std::pair<unsigned, MDNode *>> &MDs) const {
   1383   MDs.clear();
   1384 
   1385   if (!hasMetadata())
   1386     return;
   1387 
   1388   getContext().pImpl->GlobalObjectMetadata[this].getAll(MDs);
   1389 }
   1390 
   1391 void GlobalObject::clearMetadata() {
   1392   if (!hasMetadata())
   1393     return;
   1394   getContext().pImpl->GlobalObjectMetadata.erase(this);
   1395   setHasMetadataHashEntry(false);
   1396 }
   1397 
   1398 void GlobalObject::setMetadata(unsigned KindID, MDNode *N) {
   1399   eraseMetadata(KindID);
   1400   if (N)
   1401     addMetadata(KindID, *N);
   1402 }
   1403 
   1404 void GlobalObject::setMetadata(StringRef Kind, MDNode *N) {
   1405   setMetadata(getContext().getMDKindID(Kind), N);
   1406 }
   1407 
   1408 MDNode *GlobalObject::getMetadata(unsigned KindID) const {
   1409   SmallVector<MDNode *, 1> MDs;
   1410   getMetadata(KindID, MDs);
   1411   assert(MDs.size() <= 1 && "Expected at most one metadata attachment");
   1412   if (MDs.empty())
   1413     return nullptr;
   1414   return MDs[0];
   1415 }
   1416 
   1417 MDNode *GlobalObject::getMetadata(StringRef Kind) const {
   1418   return getMetadata(getContext().getMDKindID(Kind));
   1419 }
   1420 
   1421 void GlobalObject::copyMetadata(const GlobalObject *Other, unsigned Offset) {
   1422   SmallVector<std::pair<unsigned, MDNode *>, 8> MDs;
   1423   Other->getAllMetadata(MDs);
   1424   for (auto &MD : MDs) {
   1425     // We need to adjust the type metadata offset.
   1426     if (Offset != 0 && MD.first == LLVMContext::MD_type) {
   1427       auto *OffsetConst = cast<ConstantInt>(
   1428           cast<ConstantAsMetadata>(MD.second->getOperand(0))->getValue());
   1429       Metadata *TypeId = MD.second->getOperand(1);
   1430       auto *NewOffsetMD = ConstantAsMetadata::get(ConstantInt::get(
   1431           OffsetConst->getType(), OffsetConst->getValue() + Offset));
   1432       addMetadata(LLVMContext::MD_type,
   1433                   *MDNode::get(getContext(), {NewOffsetMD, TypeId}));
   1434       continue;
   1435     }
   1436     addMetadata(MD.first, *MD.second);
   1437   }
   1438 }
   1439 
   1440 void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
   1441   addMetadata(
   1442       LLVMContext::MD_type,
   1443       *MDTuple::get(getContext(),
   1444                     {llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
   1445                          Type::getInt64Ty(getContext()), Offset)),
   1446                      TypeID}));
   1447 }
   1448 
   1449 void Function::setSubprogram(DISubprogram *SP) {
   1450   setMetadata(LLVMContext::MD_dbg, SP);
   1451 }
   1452 
   1453 DISubprogram *Function::getSubprogram() const {
   1454   return cast_or_null<DISubprogram>(getMetadata(LLVMContext::MD_dbg));
   1455 }
   1456