1 //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // Instrumentation-based profiling data is generated by instrumented 11 // binaries through library functions in compiler-rt, and read by the clang 12 // frontend to feed PGO. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #ifndef LLVM_PROFILEDATA_INSTRPROF_H 17 #define LLVM_PROFILEDATA_INSTRPROF_H 18 19 #include "llvm/ADT/ArrayRef.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/StringRef.h" 22 #include "llvm/ADT/StringSet.h" 23 #include "llvm/ADT/Triple.h" 24 #include "llvm/IR/GlobalValue.h" 25 #include "llvm/IR/ProfileSummary.h" 26 #include "llvm/ProfileData/InstrProfData.inc" 27 #include "llvm/Support/Compiler.h" 28 #include "llvm/Support/Endian.h" 29 #include "llvm/Support/Error.h" 30 #include "llvm/Support/ErrorHandling.h" 31 #include "llvm/Support/Host.h" 32 #include "llvm/Support/MD5.h" 33 #include "llvm/Support/MathExtras.h" 34 #include "llvm/Support/raw_ostream.h" 35 #include <algorithm> 36 #include <cassert> 37 #include <cstddef> 38 #include <cstdint> 39 #include <cstring> 40 #include <list> 41 #include <memory> 42 #include <string> 43 #include <system_error> 44 #include <utility> 45 #include <vector> 46 47 namespace llvm { 48 49 class Function; 50 class GlobalVariable; 51 struct InstrProfRecord; 52 class InstrProfSymtab; 53 class Instruction; 54 class MDNode; 55 class Module; 56 57 enum InstrProfSectKind { 58 #define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind, 59 #include "llvm/ProfileData/InstrProfData.inc" 60 }; 61 62 /// Return the name of the profile section corresponding to \p IPSK. 63 /// 64 /// The name of the section depends on the object format type \p OF. If 65 /// \p AddSegmentInfo is true, a segment prefix and additional linker hints may 66 /// be added to the section name (this is the default). 67 std::string getInstrProfSectionName(InstrProfSectKind IPSK, 68 Triple::ObjectFormatType OF, 69 bool AddSegmentInfo = true); 70 71 /// Return the name profile runtime entry point to do value profiling 72 /// for a given site. 73 inline StringRef getInstrProfValueProfFuncName() { 74 return INSTR_PROF_VALUE_PROF_FUNC_STR; 75 } 76 77 /// Return the name profile runtime entry point to do value range profiling. 78 inline StringRef getInstrProfValueRangeProfFuncName() { 79 return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR; 80 } 81 82 /// Return the name prefix of variables containing instrumented function names. 83 inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } 84 85 /// Return the name prefix of variables containing per-function control data. 86 inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } 87 88 /// Return the name prefix of profile counter variables. 89 inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } 90 91 /// Return the name prefix of value profile variables. 92 inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } 93 94 /// Return the name of value profile node array variables: 95 inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } 96 97 /// Return the name prefix of the COMDAT group for instrumentation variables 98 /// associated with a COMDAT function. 99 inline StringRef getInstrProfComdatPrefix() { return "__profv_"; } 100 101 /// Return the name of the variable holding the strings (possibly compressed) 102 /// of all function's PGO names. 103 inline StringRef getInstrProfNamesVarName() { 104 return "__llvm_prf_nm"; 105 } 106 107 /// Return the name of a covarage mapping variable (internal linkage) 108 /// for each instrumented source module. Such variables are allocated 109 /// in the __llvm_covmap section. 110 inline StringRef getCoverageMappingVarName() { 111 return "__llvm_coverage_mapping"; 112 } 113 114 /// Return the name of the internal variable recording the array 115 /// of PGO name vars referenced by the coverage mapping. The owning 116 /// functions of those names are not emitted by FE (e.g, unused inline 117 /// functions.) 118 inline StringRef getCoverageUnusedNamesVarName() { 119 return "__llvm_coverage_names"; 120 } 121 122 /// Return the name of function that registers all the per-function control 123 /// data at program startup time by calling __llvm_register_function. This 124 /// function has internal linkage and is called by __llvm_profile_init 125 /// runtime method. This function is not generated for these platforms: 126 /// Darwin, Linux, and FreeBSD. 127 inline StringRef getInstrProfRegFuncsName() { 128 return "__llvm_profile_register_functions"; 129 } 130 131 /// Return the name of the runtime interface that registers per-function control 132 /// data for one instrumented function. 133 inline StringRef getInstrProfRegFuncName() { 134 return "__llvm_profile_register_function"; 135 } 136 137 /// Return the name of the runtime interface that registers the PGO name strings. 138 inline StringRef getInstrProfNamesRegFuncName() { 139 return "__llvm_profile_register_names_function"; 140 } 141 142 /// Return the name of the runtime initialization method that is generated by 143 /// the compiler. The function calls __llvm_profile_register_functions and 144 /// __llvm_profile_override_default_filename functions if needed. This function 145 /// has internal linkage and invoked at startup time via init_array. 146 inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; } 147 148 /// Return the name of the hook variable defined in profile runtime library. 149 /// A reference to the variable causes the linker to link in the runtime 150 /// initialization module (which defines the hook variable). 151 inline StringRef getInstrProfRuntimeHookVarName() { 152 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR); 153 } 154 155 /// Return the name of the compiler generated function that references the 156 /// runtime hook variable. The function is a weak global. 157 inline StringRef getInstrProfRuntimeHookVarUseFuncName() { 158 return "__llvm_profile_runtime_user"; 159 } 160 161 /// Return the marker used to separate PGO names during serialization. 162 inline StringRef getInstrProfNameSeparator() { return "\01"; } 163 164 /// Return the modified name for function \c F suitable to be 165 /// used the key for profile lookup. Variable \c InLTO indicates if this 166 /// is called in LTO optimization passes. 167 std::string getPGOFuncName(const Function &F, bool InLTO = false, 168 uint64_t Version = INSTR_PROF_INDEX_VERSION); 169 170 /// Return the modified name for a function suitable to be 171 /// used the key for profile lookup. The function's original 172 /// name is \c RawFuncName and has linkage of type \c Linkage. 173 /// The function is defined in module \c FileName. 174 std::string getPGOFuncName(StringRef RawFuncName, 175 GlobalValue::LinkageTypes Linkage, 176 StringRef FileName, 177 uint64_t Version = INSTR_PROF_INDEX_VERSION); 178 179 /// Return the name of the global variable used to store a function 180 /// name in PGO instrumentation. \c FuncName is the name of the function 181 /// returned by the \c getPGOFuncName call. 182 std::string getPGOFuncNameVarName(StringRef FuncName, 183 GlobalValue::LinkageTypes Linkage); 184 185 /// Create and return the global variable for function name used in PGO 186 /// instrumentation. \c FuncName is the name of the function returned 187 /// by \c getPGOFuncName call. 188 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); 189 190 /// Create and return the global variable for function name used in PGO 191 /// instrumentation. /// \c FuncName is the name of the function 192 /// returned by \c getPGOFuncName call, \c M is the owning module, 193 /// and \c Linkage is the linkage of the instrumented function. 194 GlobalVariable *createPGOFuncNameVar(Module &M, 195 GlobalValue::LinkageTypes Linkage, 196 StringRef PGOFuncName); 197 198 /// Return the initializer in string of the PGO name var \c NameVar. 199 StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar); 200 201 /// Given a PGO function name, remove the filename prefix and return 202 /// the original (static) function name. 203 StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, 204 StringRef FileName = "<unknown>"); 205 206 /// Given a vector of strings (function PGO names) \c NameStrs, the 207 /// method generates a combined string \c Result thatis ready to be 208 /// serialized. The \c Result string is comprised of three fields: 209 /// The first field is the legnth of the uncompressed strings, and the 210 /// the second field is the length of the zlib-compressed string. 211 /// Both fields are encoded in ULEB128. If \c doCompress is false, the 212 /// third field is the uncompressed strings; otherwise it is the 213 /// compressed string. When the string compression is off, the 214 /// second field will have value zero. 215 Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs, 216 bool doCompression, std::string &Result); 217 218 /// Produce \c Result string with the same format described above. The input 219 /// is vector of PGO function name variables that are referenced. 220 Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, 221 std::string &Result, bool doCompression = true); 222 223 /// \c NameStrings is a string composed of one of more sub-strings encoded in 224 /// the format described above. The substrings are separated by 0 or more zero 225 /// bytes. This method decodes the string and populates the \c Symtab. 226 Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab); 227 228 /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being 229 /// set in IR PGO compilation. 230 bool isIRPGOFlagSet(const Module *M); 231 232 /// Check if we can safely rename this Comdat function. Instances of the same 233 /// comdat function may have different control flows thus can not share the 234 /// same counter variable. 235 bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false); 236 237 enum InstrProfValueKind : uint32_t { 238 #define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, 239 #include "llvm/ProfileData/InstrProfData.inc" 240 }; 241 242 /// Get the value profile data for value site \p SiteIdx from \p InstrProfR 243 /// and annotate the instruction \p Inst with the value profile meta data. 244 /// Annotate up to \p MaxMDCount (default 3) number of records per value site. 245 void annotateValueSite(Module &M, Instruction &Inst, 246 const InstrProfRecord &InstrProfR, 247 InstrProfValueKind ValueKind, uint32_t SiteIndx, 248 uint32_t MaxMDCount = 3); 249 250 /// Same as the above interface but using an ArrayRef, as well as \p Sum. 251 void annotateValueSite(Module &M, Instruction &Inst, 252 ArrayRef<InstrProfValueData> VDs, uint64_t Sum, 253 InstrProfValueKind ValueKind, uint32_t MaxMDCount); 254 255 /// Extract the value profile data from \p Inst which is annotated with 256 /// value profile meta data. Return false if there is no value data annotated, 257 /// otherwise return true. 258 bool getValueProfDataFromInst(const Instruction &Inst, 259 InstrProfValueKind ValueKind, 260 uint32_t MaxNumValueData, 261 InstrProfValueData ValueData[], 262 uint32_t &ActualNumValueData, uint64_t &TotalC); 263 264 inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } 265 266 /// Return the PGOFuncName meta data associated with a function. 267 MDNode *getPGOFuncNameMetadata(const Function &F); 268 269 /// Create the PGOFuncName meta data if PGOFuncName is different from 270 /// function's raw name. This should only apply to internal linkage functions 271 /// declared by users only. 272 void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); 273 274 /// Check if we can use Comdat for profile variables. This will eliminate 275 /// the duplicated profile variables for Comdat functions. 276 bool needsComdatForCounter(const Function &F, const Module &M); 277 278 const std::error_category &instrprof_category(); 279 280 enum class instrprof_error { 281 success = 0, 282 eof, 283 unrecognized_format, 284 bad_magic, 285 bad_header, 286 unsupported_version, 287 unsupported_hash_type, 288 too_large, 289 truncated, 290 malformed, 291 unknown_function, 292 hash_mismatch, 293 count_mismatch, 294 counter_overflow, 295 value_site_count_mismatch, 296 compress_failed, 297 uncompress_failed, 298 empty_raw_profile, 299 zlib_unavailable 300 }; 301 302 inline std::error_code make_error_code(instrprof_error E) { 303 return std::error_code(static_cast<int>(E), instrprof_category()); 304 } 305 306 class InstrProfError : public ErrorInfo<InstrProfError> { 307 public: 308 InstrProfError(instrprof_error Err) : Err(Err) { 309 assert(Err != instrprof_error::success && "Not an error"); 310 } 311 312 std::string message() const override; 313 314 void log(raw_ostream &OS) const override { OS << message(); } 315 316 std::error_code convertToErrorCode() const override { 317 return make_error_code(Err); 318 } 319 320 instrprof_error get() const { return Err; } 321 322 /// Consume an Error and return the raw enum value contained within it. The 323 /// Error must either be a success value, or contain a single InstrProfError. 324 static instrprof_error take(Error E) { 325 auto Err = instrprof_error::success; 326 handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) { 327 assert(Err == instrprof_error::success && "Multiple errors encountered"); 328 Err = IPE.get(); 329 }); 330 return Err; 331 } 332 333 static char ID; 334 335 private: 336 instrprof_error Err; 337 }; 338 339 class SoftInstrProfErrors { 340 /// Count the number of soft instrprof_errors encountered and keep track of 341 /// the first such error for reporting purposes. 342 343 /// The first soft error encountered. 344 instrprof_error FirstError = instrprof_error::success; 345 346 /// The number of hash mismatches. 347 unsigned NumHashMismatches = 0; 348 349 /// The number of count mismatches. 350 unsigned NumCountMismatches = 0; 351 352 /// The number of counter overflows. 353 unsigned NumCounterOverflows = 0; 354 355 /// The number of value site count mismatches. 356 unsigned NumValueSiteCountMismatches = 0; 357 358 public: 359 SoftInstrProfErrors() = default; 360 361 ~SoftInstrProfErrors() { 362 assert(FirstError == instrprof_error::success && 363 "Unchecked soft error encountered"); 364 } 365 366 /// Track a soft error (\p IE) and increment its associated counter. 367 void addError(instrprof_error IE); 368 369 /// Get the number of hash mismatches. 370 unsigned getNumHashMismatches() const { return NumHashMismatches; } 371 372 /// Get the number of count mismatches. 373 unsigned getNumCountMismatches() const { return NumCountMismatches; } 374 375 /// Get the number of counter overflows. 376 unsigned getNumCounterOverflows() const { return NumCounterOverflows; } 377 378 /// Get the number of value site count mismatches. 379 unsigned getNumValueSiteCountMismatches() const { 380 return NumValueSiteCountMismatches; 381 } 382 383 /// Return the first encountered error and reset FirstError to a success 384 /// value. 385 Error takeError() { 386 if (FirstError == instrprof_error::success) 387 return Error::success(); 388 auto E = make_error<InstrProfError>(FirstError); 389 FirstError = instrprof_error::success; 390 return E; 391 } 392 }; 393 394 namespace object { 395 396 class SectionRef; 397 398 } // end namespace object 399 400 namespace IndexedInstrProf { 401 402 uint64_t ComputeHash(StringRef K); 403 404 } // end namespace IndexedInstrProf 405 406 /// A symbol table used for function PGO name look-up with keys 407 /// (such as pointers, md5hash values) to the function. A function's 408 /// PGO name or name's md5hash are used in retrieving the profile 409 /// data of the function. See \c getPGOFuncName() method for details 410 /// on how PGO name is formed. 411 class InstrProfSymtab { 412 public: 413 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>; 414 415 private: 416 StringRef Data; 417 uint64_t Address = 0; 418 // Unique name strings. 419 StringSet<> NameTab; 420 // A map from MD5 keys to function name strings. 421 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap; 422 // A map from MD5 keys to function define. We only populate this map 423 // when build the Symtab from a Module. 424 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap; 425 // A map from function runtime address to function name MD5 hash. 426 // This map is only populated and used by raw instr profile reader. 427 AddrHashMap AddrToMD5Map; 428 429 public: 430 InstrProfSymtab() = default; 431 432 /// Create InstrProfSymtab from an object file section which 433 /// contains function PGO names. When section may contain raw 434 /// string data or string data in compressed form. This method 435 /// only initialize the symtab with reference to the data and 436 /// the section base address. The decompression will be delayed 437 /// until before it is used. See also \c create(StringRef) method. 438 Error create(object::SectionRef &Section); 439 440 /// This interface is used by reader of CoverageMapping test 441 /// format. 442 inline Error create(StringRef D, uint64_t BaseAddr); 443 444 /// \c NameStrings is a string composed of one of more sub-strings 445 /// encoded in the format described in \c collectPGOFuncNameStrings. 446 /// This method is a wrapper to \c readPGOFuncNameStrings method. 447 inline Error create(StringRef NameStrings); 448 449 /// A wrapper interface to populate the PGO symtab with functions 450 /// decls from module \c M. This interface is used by transformation 451 /// passes such as indirect function call promotion. Variable \c InLTO 452 /// indicates if this is called from LTO optimization passes. 453 Error create(Module &M, bool InLTO = false); 454 455 /// Create InstrProfSymtab from a set of names iteratable from 456 /// \p IterRange. This interface is used by IndexedProfReader. 457 template <typename NameIterRange> Error create(const NameIterRange &IterRange); 458 459 // If the symtab is created by a series of calls to \c addFuncName, \c 460 // finalizeSymtab needs to be called before looking up function names. 461 // This is required because the underlying map is a vector (for space 462 // efficiency) which needs to be sorted. 463 inline void finalizeSymtab(); 464 465 /// Update the symtab by adding \p FuncName to the table. This interface 466 /// is used by the raw and text profile readers. 467 Error addFuncName(StringRef FuncName) { 468 if (FuncName.empty()) 469 return make_error<InstrProfError>(instrprof_error::malformed); 470 auto Ins = NameTab.insert(FuncName); 471 if (Ins.second) 472 MD5NameMap.push_back(std::make_pair( 473 IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); 474 return Error::success(); 475 } 476 477 /// Map a function address to its name's MD5 hash. This interface 478 /// is only used by the raw profiler reader. 479 void mapAddress(uint64_t Addr, uint64_t MD5Val) { 480 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); 481 } 482 483 AddrHashMap &getAddrHashMap() { return AddrToMD5Map; } 484 485 /// Return function's PGO name from the function name's symbol 486 /// address in the object file. If an error occurs, return 487 /// an empty string. 488 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize); 489 490 /// Return function's PGO name from the name's md5 hash value. 491 /// If not found, return an empty string. 492 inline StringRef getFuncName(uint64_t FuncMD5Hash); 493 494 /// Return function from the name's md5 hash. Return nullptr if not found. 495 inline Function *getFunction(uint64_t FuncMD5Hash); 496 497 /// Return the function's original assembly name by stripping off 498 /// the prefix attached (to symbols with priviate linkage). For 499 /// global functions, it returns the same string as getFuncName. 500 inline StringRef getOrigFuncName(uint64_t FuncMD5Hash); 501 502 /// Return the name section data. 503 inline StringRef getNameData() const { return Data; } 504 }; 505 506 Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { 507 Data = D; 508 Address = BaseAddr; 509 return Error::success(); 510 } 511 512 Error InstrProfSymtab::create(StringRef NameStrings) { 513 return readPGOFuncNameStrings(NameStrings, *this); 514 } 515 516 template <typename NameIterRange> 517 Error InstrProfSymtab::create(const NameIterRange &IterRange) { 518 for (auto Name : IterRange) 519 if (Error E = addFuncName(Name)) 520 return E; 521 522 finalizeSymtab(); 523 return Error::success(); 524 } 525 526 void InstrProfSymtab::finalizeSymtab() { 527 std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first()); 528 std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first()); 529 std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first()); 530 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), 531 AddrToMD5Map.end()); 532 } 533 534 StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) { 535 auto Result = 536 std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash, 537 [](const std::pair<uint64_t, std::string> &LHS, 538 uint64_t RHS) { return LHS.first < RHS; }); 539 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash) 540 return Result->second; 541 return StringRef(); 542 } 543 544 Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { 545 auto Result = 546 std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash, 547 [](const std::pair<uint64_t, Function*> &LHS, 548 uint64_t RHS) { return LHS.first < RHS; }); 549 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) 550 return Result->second; 551 return nullptr; 552 } 553 554 // See also getPGOFuncName implementation. These two need to be 555 // matched. 556 StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { 557 StringRef PGOName = getFuncName(FuncMD5Hash); 558 size_t S = PGOName.find_first_of(':'); 559 if (S == StringRef::npos) 560 return PGOName; 561 return PGOName.drop_front(S + 1); 562 } 563 564 struct InstrProfValueSiteRecord { 565 /// Value profiling data pairs at a given value site. 566 std::list<InstrProfValueData> ValueData; 567 568 InstrProfValueSiteRecord() { ValueData.clear(); } 569 template <class InputIterator> 570 InstrProfValueSiteRecord(InputIterator F, InputIterator L) 571 : ValueData(F, L) {} 572 573 /// Sort ValueData ascending by Value 574 void sortByTargetValues() { 575 ValueData.sort( 576 [](const InstrProfValueData &left, const InstrProfValueData &right) { 577 return left.Value < right.Value; 578 }); 579 } 580 /// Sort ValueData Descending by Count 581 inline void sortByCount(); 582 583 /// Merge data from another InstrProfValueSiteRecord 584 /// Optionally scale merged counts by \p Weight. 585 void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, 586 function_ref<void(instrprof_error)> Warn); 587 /// Scale up value profile data counts. 588 void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn); 589 }; 590 591 /// Profiling information for a single function. 592 struct InstrProfRecord { 593 std::vector<uint64_t> Counts; 594 595 InstrProfRecord() = default; 596 InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {} 597 InstrProfRecord(InstrProfRecord &&) = default; 598 InstrProfRecord(const InstrProfRecord &RHS) 599 : Counts(RHS.Counts), 600 ValueData(RHS.ValueData 601 ? llvm::make_unique<ValueProfData>(*RHS.ValueData) 602 : nullptr) {} 603 InstrProfRecord &operator=(InstrProfRecord &&) = default; 604 InstrProfRecord &operator=(const InstrProfRecord &RHS) { 605 Counts = RHS.Counts; 606 if (!RHS.ValueData) { 607 ValueData = nullptr; 608 return *this; 609 } 610 if (!ValueData) 611 ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData); 612 else 613 *ValueData = *RHS.ValueData; 614 return *this; 615 } 616 617 using ValueMapType = std::vector<std::pair<uint64_t, uint64_t>>; 618 619 /// Return the number of value profile kinds with non-zero number 620 /// of profile sites. 621 inline uint32_t getNumValueKinds() const; 622 /// Return the number of instrumented sites for ValueKind. 623 inline uint32_t getNumValueSites(uint32_t ValueKind) const; 624 625 /// Return the total number of ValueData for ValueKind. 626 inline uint32_t getNumValueData(uint32_t ValueKind) const; 627 628 /// Return the number of value data collected for ValueKind at profiling 629 /// site: Site. 630 inline uint32_t getNumValueDataForSite(uint32_t ValueKind, 631 uint32_t Site) const; 632 633 /// Return the array of profiled values at \p Site. If \p TotalC 634 /// is not null, the total count of all target values at this site 635 /// will be stored in \c *TotalC. 636 inline std::unique_ptr<InstrProfValueData[]> 637 getValueForSite(uint32_t ValueKind, uint32_t Site, 638 uint64_t *TotalC = nullptr) const; 639 640 /// Get the target value/counts of kind \p ValueKind collected at site 641 /// \p Site and store the result in array \p Dest. Return the total 642 /// counts of all target values at this site. 643 inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, 644 uint32_t Site) const; 645 646 /// Reserve space for NumValueSites sites. 647 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); 648 649 /// Add ValueData for ValueKind at value Site. 650 void addValueData(uint32_t ValueKind, uint32_t Site, 651 InstrProfValueData *VData, uint32_t N, 652 ValueMapType *ValueMap); 653 654 /// Merge the counts in \p Other into this one. 655 /// Optionally scale merged counts by \p Weight. 656 void merge(InstrProfRecord &Other, uint64_t Weight, 657 function_ref<void(instrprof_error)> Warn); 658 659 /// Scale up profile counts (including value profile data) by 660 /// \p Weight. 661 void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn); 662 663 /// Sort value profile data (per site) by count. 664 void sortValueData() { 665 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 666 for (auto &SR : getValueSitesForKind(Kind)) 667 SR.sortByCount(); 668 } 669 670 /// Clear value data entries and edge counters. 671 void Clear() { 672 Counts.clear(); 673 clearValueData(); 674 } 675 676 /// Clear value data entries 677 void clearValueData() { ValueData = nullptr; } 678 679 private: 680 struct ValueProfData { 681 std::vector<InstrProfValueSiteRecord> IndirectCallSites; 682 std::vector<InstrProfValueSiteRecord> MemOPSizes; 683 }; 684 std::unique_ptr<ValueProfData> ValueData; 685 686 MutableArrayRef<InstrProfValueSiteRecord> 687 getValueSitesForKind(uint32_t ValueKind) { 688 // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever 689 // implemented in LLVM) to call the const overload of this function, then 690 // cast away the constness from the result. 691 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind( 692 ValueKind); 693 return makeMutableArrayRef( 694 const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size()); 695 } 696 ArrayRef<InstrProfValueSiteRecord> 697 getValueSitesForKind(uint32_t ValueKind) const { 698 if (!ValueData) 699 return None; 700 switch (ValueKind) { 701 case IPVK_IndirectCallTarget: 702 return ValueData->IndirectCallSites; 703 case IPVK_MemOPSize: 704 return ValueData->MemOPSizes; 705 default: 706 llvm_unreachable("Unknown value kind!"); 707 } 708 } 709 710 std::vector<InstrProfValueSiteRecord> & 711 getOrCreateValueSitesForKind(uint32_t ValueKind) { 712 if (!ValueData) 713 ValueData = llvm::make_unique<ValueProfData>(); 714 switch (ValueKind) { 715 case IPVK_IndirectCallTarget: 716 return ValueData->IndirectCallSites; 717 case IPVK_MemOPSize: 718 return ValueData->MemOPSizes; 719 default: 720 llvm_unreachable("Unknown value kind!"); 721 } 722 } 723 724 // Map indirect call target name hash to name string. 725 uint64_t remapValue(uint64_t Value, uint32_t ValueKind, 726 ValueMapType *HashKeys); 727 728 // Merge Value Profile data from Src record to this record for ValueKind. 729 // Scale merged value counts by \p Weight. 730 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src, 731 uint64_t Weight, 732 function_ref<void(instrprof_error)> Warn); 733 734 // Scale up value profile data count. 735 void scaleValueProfData(uint32_t ValueKind, uint64_t Weight, 736 function_ref<void(instrprof_error)> Warn); 737 }; 738 739 struct NamedInstrProfRecord : InstrProfRecord { 740 StringRef Name; 741 uint64_t Hash; 742 743 NamedInstrProfRecord() = default; 744 NamedInstrProfRecord(StringRef Name, uint64_t Hash, 745 std::vector<uint64_t> Counts) 746 : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} 747 }; 748 749 uint32_t InstrProfRecord::getNumValueKinds() const { 750 uint32_t NumValueKinds = 0; 751 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 752 NumValueKinds += !(getValueSitesForKind(Kind).empty()); 753 return NumValueKinds; 754 } 755 756 uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { 757 uint32_t N = 0; 758 for (auto &SR : getValueSitesForKind(ValueKind)) 759 N += SR.ValueData.size(); 760 return N; 761 } 762 763 uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { 764 return getValueSitesForKind(ValueKind).size(); 765 } 766 767 uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, 768 uint32_t Site) const { 769 return getValueSitesForKind(ValueKind)[Site].ValueData.size(); 770 } 771 772 std::unique_ptr<InstrProfValueData[]> 773 InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, 774 uint64_t *TotalC) const { 775 uint64_t Dummy; 776 uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC); 777 uint32_t N = getNumValueDataForSite(ValueKind, Site); 778 if (N == 0) { 779 TotalCount = 0; 780 return std::unique_ptr<InstrProfValueData[]>(nullptr); 781 } 782 783 auto VD = llvm::make_unique<InstrProfValueData[]>(N); 784 TotalCount = getValueForSite(VD.get(), ValueKind, Site); 785 786 return VD; 787 } 788 789 uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], 790 uint32_t ValueKind, 791 uint32_t Site) const { 792 uint32_t I = 0; 793 uint64_t TotalCount = 0; 794 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { 795 Dest[I].Value = V.Value; 796 Dest[I].Count = V.Count; 797 TotalCount = SaturatingAdd(TotalCount, V.Count); 798 I++; 799 } 800 return TotalCount; 801 } 802 803 void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { 804 if (!NumValueSites) 805 return; 806 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites); 807 } 808 809 inline support::endianness getHostEndianness() { 810 return sys::IsLittleEndianHost ? support::little : support::big; 811 } 812 813 // Include definitions for value profile data 814 #define INSTR_PROF_VALUE_PROF_DATA 815 #include "llvm/ProfileData/InstrProfData.inc" 816 817 void InstrProfValueSiteRecord::sortByCount() { 818 ValueData.sort( 819 [](const InstrProfValueData &left, const InstrProfValueData &right) { 820 return left.Count > right.Count; 821 }); 822 // Now truncate 823 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE; 824 if (ValueData.size() > max_s) 825 ValueData.resize(max_s); 826 } 827 828 namespace IndexedInstrProf { 829 830 enum class HashT : uint32_t { 831 MD5, 832 Last = MD5 833 }; 834 835 inline uint64_t ComputeHash(HashT Type, StringRef K) { 836 switch (Type) { 837 case HashT::MD5: 838 return MD5Hash(K); 839 } 840 llvm_unreachable("Unhandled hash type"); 841 } 842 843 const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81" 844 845 enum ProfVersion { 846 // Version 1 is the first version. In this version, the value of 847 // a key/value pair can only include profile data of a single function. 848 // Due to this restriction, the number of block counters for a given 849 // function is not recorded but derived from the length of the value. 850 Version1 = 1, 851 // The version 2 format supports recording profile data of multiple 852 // functions which share the same key in one value field. To support this, 853 // the number block counters is recorded as an uint64_t field right after the 854 // function structural hash. 855 Version2 = 2, 856 // Version 3 supports value profile data. The value profile data is expected 857 // to follow the block counter profile data. 858 Version3 = 3, 859 // In this version, profile summary data \c IndexedInstrProf::Summary is 860 // stored after the profile header. 861 Version4 = 4, 862 // The current version is 4. 863 CurrentVersion = INSTR_PROF_INDEX_VERSION 864 }; 865 const uint64_t Version = ProfVersion::CurrentVersion; 866 867 const HashT HashType = HashT::MD5; 868 869 inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } 870 871 // This structure defines the file header of the LLVM profile 872 // data file in indexed-format. 873 struct Header { 874 uint64_t Magic; 875 uint64_t Version; 876 uint64_t Unused; // Becomes unused since version 4 877 uint64_t HashType; 878 uint64_t HashOffset; 879 }; 880 881 // Profile summary data recorded in the profile data file in indexed 882 // format. It is introduced in version 4. The summary data follows 883 // right after the profile file header. 884 struct Summary { 885 struct Entry { 886 uint64_t Cutoff; ///< The required percentile of total execution count. 887 uint64_t 888 MinBlockCount; ///< The minimum execution count for this percentile. 889 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count. 890 }; 891 // The field kind enumerator to assigned value mapping should remain 892 // unchanged when a new kind is added or an old kind gets deleted in 893 // the future. 894 enum SummaryFieldKind { 895 /// The total number of functions instrumented. 896 TotalNumFunctions = 0, 897 /// Total number of instrumented blocks/edges. 898 TotalNumBlocks = 1, 899 /// The maximal execution count among all functions. 900 /// This field does not exist for profile data from IR based 901 /// instrumentation. 902 MaxFunctionCount = 2, 903 /// Max block count of the program. 904 MaxBlockCount = 3, 905 /// Max internal block count of the program (excluding entry blocks). 906 MaxInternalBlockCount = 4, 907 /// The sum of all instrumented block counts. 908 TotalBlockCount = 5, 909 NumKinds = TotalBlockCount + 1 910 }; 911 912 // The number of summmary fields following the summary header. 913 uint64_t NumSummaryFields; 914 // The number of Cutoff Entries (Summary::Entry) following summary fields. 915 uint64_t NumCutoffEntries; 916 917 Summary() = delete; 918 Summary(uint32_t Size) { memset(this, 0, Size); } 919 920 void operator delete(void *ptr) { ::operator delete(ptr); } 921 922 static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) { 923 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) + 924 NumSumFields * sizeof(uint64_t); 925 } 926 927 const uint64_t *getSummaryDataBase() const { 928 return reinterpret_cast<const uint64_t *>(this + 1); 929 } 930 931 uint64_t *getSummaryDataBase() { 932 return reinterpret_cast<uint64_t *>(this + 1); 933 } 934 935 const Entry *getCutoffEntryBase() const { 936 return reinterpret_cast<const Entry *>( 937 &getSummaryDataBase()[NumSummaryFields]); 938 } 939 940 Entry *getCutoffEntryBase() { 941 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]); 942 } 943 944 uint64_t get(SummaryFieldKind K) const { 945 return getSummaryDataBase()[K]; 946 } 947 948 void set(SummaryFieldKind K, uint64_t V) { 949 getSummaryDataBase()[K] = V; 950 } 951 952 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; } 953 954 void setEntry(uint32_t I, const ProfileSummaryEntry &E) { 955 Entry &ER = getCutoffEntryBase()[I]; 956 ER.Cutoff = E.Cutoff; 957 ER.MinBlockCount = E.MinCount; 958 ER.NumBlocks = E.NumCounts; 959 } 960 }; 961 962 inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) { 963 return std::unique_ptr<Summary>(new (::operator new(TotalSize)) 964 Summary(TotalSize)); 965 } 966 967 } // end namespace IndexedInstrProf 968 969 namespace RawInstrProf { 970 971 // Version 1: First version 972 // Version 2: Added value profile data section. Per-function control data 973 // struct has more fields to describe value profile information. 974 // Version 3: Compressed name section support. Function PGO name reference 975 // from control data struct is changed from raw pointer to Name's MD5 value. 976 // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the 977 // raw header. 978 const uint64_t Version = INSTR_PROF_RAW_VERSION; 979 980 template <class IntPtrT> inline uint64_t getMagic(); 981 template <> inline uint64_t getMagic<uint64_t>() { 982 return INSTR_PROF_RAW_MAGIC_64; 983 } 984 985 template <> inline uint64_t getMagic<uint32_t>() { 986 return INSTR_PROF_RAW_MAGIC_32; 987 } 988 989 // Per-function profile data header/control structure. 990 // The definition should match the structure defined in 991 // compiler-rt/lib/profile/InstrProfiling.h. 992 // It should also match the synthesized type in 993 // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. 994 template <class IntPtrT> struct LLVM_ALIGNAS(8) ProfileData { 995 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; 996 #include "llvm/ProfileData/InstrProfData.inc" 997 }; 998 999 // File header structure of the LLVM profile data in raw format. 1000 // The definition should match the header referenced in 1001 // compiler-rt/lib/profile/InstrProfilingFile.c and 1002 // InstrProfilingBuffer.c. 1003 struct Header { 1004 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name; 1005 #include "llvm/ProfileData/InstrProfData.inc" 1006 }; 1007 1008 } // end namespace RawInstrProf 1009 1010 // Parse MemOP Size range option. 1011 void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, 1012 int64_t &RangeLast); 1013 1014 } // end namespace llvm 1015 1016 #endif // LLVM_PROFILEDATA_INSTRPROF_H 1017