Home | History | Annotate | Download | only in ProfileData 
      1 //=-- InstrProf.cpp - Instrumented profiling format support -----------------=//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // This file contains support for clang's instrumentation based PGO and
     11 // coverage.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "llvm/IR/Constants.h"
     16 #include "llvm/IR/Function.h"
     17 #include "llvm/IR/Module.h"
     18 #include "llvm/IR/GlobalVariable.h"
     19 #include "llvm/ProfileData/InstrProf.h"
     20 #include "llvm/Support/ErrorHandling.h"
     21 #include "llvm/Support/ManagedStatic.h"
     22 
     23 using namespace llvm;
     24 
     25 namespace {
     26 class InstrProfErrorCategoryType : public std::error_category {
     27   const char *name() const LLVM_NOEXCEPT override { return "llvm.instrprof"; }
     28   std::string message(int IE) const override {
     29     instrprof_error E = static_cast<instrprof_error>(IE);
     30     switch (E) {
     31     case instrprof_error::success:
     32       return "Success";
     33     case instrprof_error::eof:
     34       return "End of File";
     35     case instrprof_error::unrecognized_format:
     36       return "Unrecognized instrumentation profile encoding format";
     37     case instrprof_error::bad_magic:
     38       return "Invalid instrumentation profile data (bad magic)";
     39     case instrprof_error::bad_header:
     40       return "Invalid instrumentation profile data (file header is corrupt)";
     41     case instrprof_error::unsupported_version:
     42       return "Unsupported instrumentation profile format version";
     43     case instrprof_error::unsupported_hash_type:
     44       return "Unsupported instrumentation profile hash type";
     45     case instrprof_error::too_large:
     46       return "Too much profile data";
     47     case instrprof_error::truncated:
     48       return "Truncated profile data";
     49     case instrprof_error::malformed:
     50       return "Malformed instrumentation profile data";
     51     case instrprof_error::unknown_function:
     52       return "No profile data available for function";
     53     case instrprof_error::hash_mismatch:
     54       return "Function control flow change detected (hash mismatch)";
     55     case instrprof_error::count_mismatch:
     56       return "Function basic block count change detected (counter mismatch)";
     57     case instrprof_error::counter_overflow:
     58       return "Counter overflow";
     59     case instrprof_error::value_site_count_mismatch:
     60       return "Function value site count change detected (counter mismatch)";
     61     }
     62     llvm_unreachable("A value of instrprof_error has no message.");
     63   }
     64 };
     65 }
     66 
     67 static ManagedStatic<InstrProfErrorCategoryType> ErrorCategory;
     68 
     69 const std::error_category &llvm::instrprof_category() {
     70   return *ErrorCategory;
     71 }
     72 
     73 namespace llvm {
     74 
     75 std::string getPGOFuncName(StringRef RawFuncName,
     76                            GlobalValue::LinkageTypes Linkage,
     77                            StringRef FileName,
     78                            uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
     79 
     80   // Function names may be prefixed with a binary '1' to indicate
     81   // that the backend should not modify the symbols due to any platform
     82   // naming convention. Do not include that '1' in the PGO profile name.
     83   if (RawFuncName[0] == '\1')
     84     RawFuncName = RawFuncName.substr(1);
     85 
     86   std::string FuncName = RawFuncName;
     87   if (llvm::GlobalValue::isLocalLinkage(Linkage)) {
     88     // For local symbols, prepend the main file name to distinguish them.
     89     // Do not include the full path in the file name since there's no guarantee
     90     // that it will stay the same, e.g., if the files are checked out from
     91     // version control in different locations.
     92     if (FileName.empty())
     93       FuncName = FuncName.insert(0, "<unknown>:");
     94     else
     95       FuncName = FuncName.insert(0, FileName.str() + ":");
     96   }
     97   return FuncName;
     98 }
     99 
    100 std::string getPGOFuncName(const Function &F, uint64_t Version) {
    101   return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(),
    102                         Version);
    103 }
    104 
    105 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
    106   if (FileName.empty())
    107     return PGOFuncName;
    108   // Drop the file name including ':'. See also getPGOFuncName.
    109   if (PGOFuncName.startswith(FileName))
    110     PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
    111   return PGOFuncName;
    112 }
    113 
    114 // \p FuncName is the string used as profile lookup key for the function. A
    115 // symbol is created to hold the name. Return the legalized symbol name.
    116 static std::string getPGOFuncNameVarName(StringRef FuncName,
    117                                          GlobalValue::LinkageTypes Linkage) {
    118   std::string VarName = getInstrProfNameVarPrefix();
    119   VarName += FuncName;
    120 
    121   if (!GlobalValue::isLocalLinkage(Linkage))
    122     return VarName;
    123 
    124   // Now fix up illegal chars in local VarName that may upset the assembler.
    125   const char *InvalidChars = "-:<>\"'";
    126   size_t found = VarName.find_first_of(InvalidChars);
    127   while (found != std::string::npos) {
    128     VarName[found] = '_';
    129     found = VarName.find_first_of(InvalidChars, found + 1);
    130   }
    131   return VarName;
    132 }
    133 
    134 GlobalVariable *createPGOFuncNameVar(Module &M,
    135                                      GlobalValue::LinkageTypes Linkage,
    136                                      StringRef FuncName) {
    137 
    138   // We generally want to match the function's linkage, but available_externally
    139   // and extern_weak both have the wrong semantics, and anything that doesn't
    140   // need to link across compilation units doesn't need to be visible at all.
    141   if (Linkage == GlobalValue::ExternalWeakLinkage)
    142     Linkage = GlobalValue::LinkOnceAnyLinkage;
    143   else if (Linkage == GlobalValue::AvailableExternallyLinkage)
    144     Linkage = GlobalValue::LinkOnceODRLinkage;
    145   else if (Linkage == GlobalValue::InternalLinkage ||
    146            Linkage == GlobalValue::ExternalLinkage)
    147     Linkage = GlobalValue::PrivateLinkage;
    148 
    149   auto *Value = ConstantDataArray::getString(M.getContext(), FuncName, false);
    150   auto FuncNameVar =
    151       new GlobalVariable(M, Value->getType(), true, Linkage, Value,
    152                          getPGOFuncNameVarName(FuncName, Linkage));
    153 
    154   // Hide the symbol so that we correctly get a copy for each executable.
    155   if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
    156     FuncNameVar->setVisibility(GlobalValue::HiddenVisibility);
    157 
    158   return FuncNameVar;
    159 }
    160 
    161 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) {
    162   return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName);
    163 }
    164 
    165 instrprof_error
    166 InstrProfValueSiteRecord::mergeValueData(InstrProfValueSiteRecord &Input,
    167                                          uint64_t Weight) {
    168   this->sortByTargetValues();
    169   Input.sortByTargetValues();
    170   auto I = ValueData.begin();
    171   auto IE = ValueData.end();
    172   instrprof_error Result = instrprof_error::success;
    173   for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
    174        ++J) {
    175     while (I != IE && I->Value < J->Value)
    176       ++I;
    177     if (I != IE && I->Value == J->Value) {
    178       uint64_t JCount = J->Count;
    179       bool Overflowed;
    180       if (Weight > 1) {
    181         JCount = SaturatingMultiply(JCount, Weight, &Overflowed);
    182         if (Overflowed)
    183           Result = instrprof_error::counter_overflow;
    184       }
    185       I->Count = SaturatingAdd(I->Count, JCount, &Overflowed);
    186       if (Overflowed)
    187         Result = instrprof_error::counter_overflow;
    188       ++I;
    189       continue;
    190     }
    191     ValueData.insert(I, *J);
    192   }
    193   return Result;
    194 }
    195 
    196 // Merge Value Profile data from Src record to this record for ValueKind.
    197 // Scale merged value counts by \p Weight.
    198 instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
    199                                                     InstrProfRecord &Src,
    200                                                     uint64_t Weight) {
    201   uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
    202   uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
    203   if (ThisNumValueSites != OtherNumValueSites)
    204     return instrprof_error::value_site_count_mismatch;
    205   std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
    206       getValueSitesForKind(ValueKind);
    207   std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
    208       Src.getValueSitesForKind(ValueKind);
    209   instrprof_error Result = instrprof_error::success;
    210   for (uint32_t I = 0; I < ThisNumValueSites; I++)
    211     MergeResult(Result,
    212                 ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I], Weight));
    213   return Result;
    214 }
    215 
    216 instrprof_error InstrProfRecord::merge(InstrProfRecord &Other,
    217                                        uint64_t Weight) {
    218   // If the number of counters doesn't match we either have bad data
    219   // or a hash collision.
    220   if (Counts.size() != Other.Counts.size())
    221     return instrprof_error::count_mismatch;
    222 
    223   instrprof_error Result = instrprof_error::success;
    224 
    225   for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
    226     bool Overflowed;
    227     uint64_t OtherCount = Other.Counts[I];
    228     if (Weight > 1) {
    229       OtherCount = SaturatingMultiply(OtherCount, Weight, &Overflowed);
    230       if (Overflowed)
    231         Result = instrprof_error::counter_overflow;
    232     }
    233     Counts[I] = SaturatingAdd(Counts[I], OtherCount, &Overflowed);
    234     if (Overflowed)
    235       Result = instrprof_error::counter_overflow;
    236   }
    237 
    238   for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
    239     MergeResult(Result, mergeValueProfData(Kind, Other, Weight));
    240 
    241   return Result;
    242 }
    243 
    244 // Map indirect call target name hash to name string.
    245 uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
    246                                      ValueMapType *ValueMap) {
    247   if (!ValueMap)
    248     return Value;
    249   switch (ValueKind) {
    250   case IPVK_IndirectCallTarget: {
    251     auto Result =
    252         std::lower_bound(ValueMap->begin(), ValueMap->end(), Value,
    253                          [](const std::pair<uint64_t, uint64_t> &LHS,
    254                             uint64_t RHS) { return LHS.first < RHS; });
    255     if (Result != ValueMap->end())
    256       Value = (uint64_t)Result->second;
    257     break;
    258   }
    259   }
    260   return Value;
    261 }
    262 
    263 void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
    264                                    InstrProfValueData *VData, uint32_t N,
    265                                    ValueMapType *ValueMap) {
    266   for (uint32_t I = 0; I < N; I++) {
    267     VData[I].Value = remapValue(VData[I].Value, ValueKind, ValueMap);
    268   }
    269   std::vector<InstrProfValueSiteRecord> &ValueSites =
    270       getValueSitesForKind(ValueKind);
    271   if (N == 0)
    272     ValueSites.push_back(InstrProfValueSiteRecord());
    273   else
    274     ValueSites.emplace_back(VData, VData + N);
    275 }
    276 
    277 #define INSTR_PROF_COMMON_API_IMPL
    278 #include "llvm/ProfileData/InstrProfData.inc"
    279 
    280 /*!
    281  * \brief ValueProfRecordClosure Interface implementation for  InstrProfRecord
    282  *  class. These C wrappers are used as adaptors so that C++ code can be
    283  *  invoked as callbacks.
    284  */
    285 uint32_t getNumValueKindsInstrProf(const void *Record) {
    286   return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
    287 }
    288 
    289 uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) {
    290   return reinterpret_cast<const InstrProfRecord *>(Record)
    291       ->getNumValueSites(VKind);
    292 }
    293 
    294 uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) {
    295   return reinterpret_cast<const InstrProfRecord *>(Record)
    296       ->getNumValueData(VKind);
    297 }
    298 
    299 uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK,
    300                                          uint32_t S) {
    301   return reinterpret_cast<const InstrProfRecord *>(R)
    302       ->getNumValueDataForSite(VK, S);
    303 }
    304 
    305 void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst,
    306                               uint32_t K, uint32_t S,
    307                               uint64_t (*Mapper)(uint32_t, uint64_t)) {
    308   return reinterpret_cast<const InstrProfRecord *>(R)->getValueForSite(
    309       Dst, K, S, Mapper);
    310 }
    311 
    312 ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
    313   ValueProfData *VD =
    314       (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
    315   memset(VD, 0, TotalSizeInBytes);
    316   return VD;
    317 }
    318 
    319 static ValueProfRecordClosure InstrProfRecordClosure = {
    320     0,
    321     getNumValueKindsInstrProf,
    322     getNumValueSitesInstrProf,
    323     getNumValueDataInstrProf,
    324     getNumValueDataForSiteInstrProf,
    325     0,
    326     getValueForSiteInstrProf,
    327     allocValueProfDataInstrProf};
    328 
    329 // Wrapper implementation using the closure mechanism.
    330 uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
    331   InstrProfRecordClosure.Record = &Record;
    332   return getValueProfDataSize(&InstrProfRecordClosure);
    333 }
    334 
    335 // Wrapper implementation using the closure mechanism.
    336 std::unique_ptr<ValueProfData>
    337 ValueProfData::serializeFrom(const InstrProfRecord &Record) {
    338   InstrProfRecordClosure.Record = &Record;
    339 
    340   std::unique_ptr<ValueProfData> VPD(
    341       serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
    342   return VPD;
    343 }
    344 
    345 void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
    346                                     InstrProfRecord::ValueMapType *VMap) {
    347   Record.reserveSites(Kind, NumValueSites);
    348 
    349   InstrProfValueData *ValueData = getValueProfRecordValueData(this);
    350   for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
    351     uint8_t ValueDataCount = this->SiteCountArray[VSite];
    352     Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap);
    353     ValueData += ValueDataCount;
    354   }
    355 }
    356 
    357 // For writing/serializing,  Old is the host endianness, and  New is
    358 // byte order intended on disk. For Reading/deserialization, Old
    359 // is the on-disk source endianness, and New is the host endianness.
    360 void ValueProfRecord::swapBytes(support::endianness Old,
    361                                 support::endianness New) {
    362   using namespace support;
    363   if (Old == New)
    364     return;
    365 
    366   if (getHostEndianness() != Old) {
    367     sys::swapByteOrder<uint32_t>(NumValueSites);
    368     sys::swapByteOrder<uint32_t>(Kind);
    369   }
    370   uint32_t ND = getValueProfRecordNumValueData(this);
    371   InstrProfValueData *VD = getValueProfRecordValueData(this);
    372 
    373   // No need to swap byte array: SiteCountArrray.
    374   for (uint32_t I = 0; I < ND; I++) {
    375     sys::swapByteOrder<uint64_t>(VD[I].Value);
    376     sys::swapByteOrder<uint64_t>(VD[I].Count);
    377   }
    378   if (getHostEndianness() == Old) {
    379     sys::swapByteOrder<uint32_t>(NumValueSites);
    380     sys::swapByteOrder<uint32_t>(Kind);
    381   }
    382 }
    383 
    384 void ValueProfData::deserializeTo(InstrProfRecord &Record,
    385                                   InstrProfRecord::ValueMapType *VMap) {
    386   if (NumValueKinds == 0)
    387     return;
    388 
    389   ValueProfRecord *VR = getFirstValueProfRecord(this);
    390   for (uint32_t K = 0; K < NumValueKinds; K++) {
    391     VR->deserializeTo(Record, VMap);
    392     VR = getValueProfRecordNext(VR);
    393   }
    394 }
    395 
    396 template <class T>
    397 static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) {
    398   using namespace support;
    399   if (Orig == little)
    400     return endian::readNext<T, little, unaligned>(D);
    401   else
    402     return endian::readNext<T, big, unaligned>(D);
    403 }
    404 
    405 static std::unique_ptr<ValueProfData> allocValueProfData(uint32_t TotalSize) {
    406   return std::unique_ptr<ValueProfData>(new (::operator new(TotalSize))
    407                                             ValueProfData());
    408 }
    409 
    410 instrprof_error ValueProfData::checkIntegrity() {
    411   if (NumValueKinds > IPVK_Last + 1)
    412     return instrprof_error::malformed;
    413   // Total size needs to be mulltiple of quadword size.
    414   if (TotalSize % sizeof(uint64_t))
    415     return instrprof_error::malformed;
    416 
    417   ValueProfRecord *VR = getFirstValueProfRecord(this);
    418   for (uint32_t K = 0; K < this->NumValueKinds; K++) {
    419     if (VR->Kind > IPVK_Last)
    420       return instrprof_error::malformed;
    421     VR = getValueProfRecordNext(VR);
    422     if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
    423       return instrprof_error::malformed;
    424   }
    425   return instrprof_error::success;
    426 }
    427 
    428 ErrorOr<std::unique_ptr<ValueProfData>>
    429 ValueProfData::getValueProfData(const unsigned char *D,
    430                                 const unsigned char *const BufferEnd,
    431                                 support::endianness Endianness) {
    432   using namespace support;
    433   if (D + sizeof(ValueProfData) > BufferEnd)
    434     return instrprof_error::truncated;
    435 
    436   const unsigned char *Header = D;
    437   uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
    438   if (D + TotalSize > BufferEnd)
    439     return instrprof_error::too_large;
    440 
    441   std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
    442   memcpy(VPD.get(), D, TotalSize);
    443   // Byte swap.
    444   VPD->swapBytesToHost(Endianness);
    445 
    446   instrprof_error EC = VPD->checkIntegrity();
    447   if (EC != instrprof_error::success)
    448     return EC;
    449 
    450   return std::move(VPD);
    451 }
    452 
    453 void ValueProfData::swapBytesToHost(support::endianness Endianness) {
    454   using namespace support;
    455   if (Endianness == getHostEndianness())
    456     return;
    457 
    458   sys::swapByteOrder<uint32_t>(TotalSize);
    459   sys::swapByteOrder<uint32_t>(NumValueKinds);
    460 
    461   ValueProfRecord *VR = getFirstValueProfRecord(this);
    462   for (uint32_t K = 0; K < NumValueKinds; K++) {
    463     VR->swapBytes(Endianness, getHostEndianness());
    464     VR = getValueProfRecordNext(VR);
    465   }
    466 }
    467 
    468 void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
    469   using namespace support;
    470   if (Endianness == getHostEndianness())
    471     return;
    472 
    473   ValueProfRecord *VR = getFirstValueProfRecord(this);
    474   for (uint32_t K = 0; K < NumValueKinds; K++) {
    475     ValueProfRecord *NVR = getValueProfRecordNext(VR);
    476     VR->swapBytes(getHostEndianness(), Endianness);
    477     VR = NVR;
    478   }
    479   sys::swapByteOrder<uint32_t>(TotalSize);
    480   sys::swapByteOrder<uint32_t>(NumValueKinds);
    481 }
    482 
    483 }
    484