1 //===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // Instrumentation-based profile-guided optimization 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef CLANG_CODEGEN_CODEGENPGO_H 15 #define CLANG_CODEGEN_CODEGENPGO_H 16 17 #include "CGBuilder.h" 18 #include "CodeGenModule.h" 19 #include "CodeGenTypes.h" 20 #include "clang/Frontend/CodeGenOptions.h" 21 #include "llvm/ADT/StringMap.h" 22 #include "llvm/Support/MemoryBuffer.h" 23 #include <memory> 24 25 namespace clang { 26 namespace CodeGen { 27 class RegionCounter; 28 29 /// Per-function PGO state. This class should generally not be used directly, 30 /// but instead through the CodeGenFunction and RegionCounter types. 31 class CodeGenPGO { 32 private: 33 CodeGenModule &CGM; 34 std::unique_ptr<std::string> PrefixedFuncName; 35 StringRef RawFuncName; 36 llvm::GlobalValue::LinkageTypes VarLinkage; 37 38 unsigned NumRegionCounters; 39 uint64_t FunctionHash; 40 llvm::GlobalVariable *RegionCounters; 41 std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap; 42 std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap; 43 std::unique_ptr<std::vector<uint64_t>> RegionCounts; 44 uint64_t CurrentRegionCount; 45 46 public: 47 CodeGenPGO(CodeGenModule &CGM) 48 : CGM(CGM), NumRegionCounters(0), FunctionHash(0), 49 RegionCounters(nullptr), CurrentRegionCount(0) {} 50 51 /// Whether or not we have PGO region data for the current function. This is 52 /// false both when we have no data at all and when our data has been 53 /// discarded. 54 bool haveRegionCounts() const { return RegionCounts != nullptr; } 55 56 /// Get the string used to identify this function in the profile data. 57 /// For functions with local linkage, this includes the main file name. 58 StringRef getFuncName() const { return StringRef(*PrefixedFuncName); } 59 std::string getFuncVarName(StringRef VarName) const { 60 return ("__llvm_profile_" + VarName + "_" + RawFuncName).str(); 61 } 62 63 /// Return the counter value of the current region. 64 uint64_t getCurrentRegionCount() const { return CurrentRegionCount; } 65 66 /// Set the counter value for the current region. This is used to keep track 67 /// of changes to the most recent counter from control flow and non-local 68 /// exits. 69 void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } 70 71 /// Indicate that the current region is never reached, and thus should have a 72 /// counter value of zero. This is important so that subsequent regions can 73 /// correctly track their parent counts. 74 void setCurrentRegionUnreachable() { setCurrentRegionCount(0); } 75 76 /// Check if an execution count is known for a given statement. If so, return 77 /// true and put the value in Count; else return false. 78 bool getStmtCount(const Stmt *S, uint64_t &Count) { 79 if (!StmtCountMap) 80 return false; 81 llvm::DenseMap<const Stmt*, uint64_t>::const_iterator 82 I = StmtCountMap->find(S); 83 if (I == StmtCountMap->end()) 84 return false; 85 Count = I->second; 86 return true; 87 } 88 89 /// If the execution count for the current statement is known, record that 90 /// as the current count. 91 void setCurrentStmt(const Stmt *S) { 92 uint64_t Count; 93 if (getStmtCount(S, Count)) 94 setCurrentRegionCount(Count); 95 } 96 97 /// Calculate branch weights appropriate for PGO data 98 llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount); 99 llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights); 100 llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt); 101 102 /// Assign counters to regions and configure them for PGO of a given 103 /// function. Does nothing if instrumentation is not enabled and either 104 /// generates global variables or associates PGO data with each of the 105 /// counters depending on whether we are generating or using instrumentation. 106 void assignRegionCounters(const Decl *D, llvm::Function *Fn); 107 /// Emit static data structures for instrumentation data. 108 void emitInstrumentationData(); 109 /// Clean up region counter state. Must be called if assignRegionCounters is 110 /// used. 111 void destroyRegionCounters(); 112 /// Emit static initialization code, if any. 113 static llvm::Function *emitInitialization(CodeGenModule &CGM); 114 115 private: 116 void setFuncName(llvm::Function *Fn); 117 void mapRegionCounters(const Decl *D); 118 void computeRegionCounts(const Decl *D); 119 void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, 120 llvm::Function *Fn); 121 void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, 122 bool IsInMainFile); 123 void emitCounterVariables(); 124 llvm::GlobalVariable *buildDataVar(); 125 126 /// Emit code to increment the counter at the given index 127 void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter); 128 129 /// Return the region counter for the given statement. This should only be 130 /// called on statements that have a dedicated counter. 131 unsigned getRegionCounter(const Stmt *S) { 132 if (!RegionCounterMap) 133 return 0; 134 return (*RegionCounterMap)[S]; 135 } 136 137 /// Return the region count for the counter at the given index. 138 uint64_t getRegionCount(unsigned Counter) { 139 if (!haveRegionCounts()) 140 return 0; 141 return (*RegionCounts)[Counter]; 142 } 143 144 friend class RegionCounter; 145 }; 146 147 /// A counter for a particular region. This is the primary interface through 148 /// which clients manage PGO counters and their values. 149 class RegionCounter { 150 CodeGenPGO *PGO; 151 unsigned Counter; 152 uint64_t Count; 153 uint64_t ParentCount; 154 uint64_t RegionCount; 155 int64_t Adjust; 156 157 RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex) 158 : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)), 159 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} 160 161 public: 162 RegionCounter(CodeGenPGO &PGO, const Stmt *S) 163 : PGO(&PGO), Counter(PGO.getRegionCounter(S)), 164 Count(PGO.getRegionCount(Counter)), 165 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} 166 167 /// Get the value of the counter. In most cases this is the number of times 168 /// the region of the counter was entered, but for switch labels it's the 169 /// number of direct jumps to that label. 170 uint64_t getCount() const { return Count; } 171 172 /// Get the value of the counter with adjustments applied. Adjustments occur 173 /// when control enters or leaves the region abnormally; i.e., if there is a 174 /// jump to a label within the region, or if the function can return from 175 /// within the region. The adjusted count, then, is the value of the counter 176 /// at the end of the region. 177 uint64_t getAdjustedCount() const { 178 return Count + Adjust; 179 } 180 181 /// Get the value of the counter in this region's parent, i.e., the region 182 /// that was active when this region began. This is useful for deriving 183 /// counts in implicitly counted regions, like the false case of a condition 184 /// or the normal exits of a loop. 185 uint64_t getParentCount() const { return ParentCount; } 186 187 /// Activate the counter by emitting an increment and starting to track 188 /// adjustments. If AddIncomingFallThrough is true, the current region count 189 /// will be added to the counter for the purposes of tracking the region. 190 void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) { 191 beginRegion(AddIncomingFallThrough); 192 PGO->emitCounterIncrement(Builder, Counter); 193 } 194 void beginRegion(bool AddIncomingFallThrough=false) { 195 RegionCount = Count; 196 if (AddIncomingFallThrough) 197 RegionCount += PGO->getCurrentRegionCount(); 198 PGO->setCurrentRegionCount(RegionCount); 199 } 200 201 /// For counters on boolean branches, begins tracking adjustments for the 202 /// uncounted path. 203 void beginElseRegion() { 204 RegionCount = ParentCount - Count; 205 PGO->setCurrentRegionCount(RegionCount); 206 } 207 208 /// Reset the current region count. 209 void setCurrentRegionCount(uint64_t CurrentCount) { 210 RegionCount = CurrentCount; 211 PGO->setCurrentRegionCount(RegionCount); 212 } 213 214 /// Adjust for non-local control flow after emitting a subexpression or 215 /// substatement. This must be called to account for constructs such as gotos, 216 /// labels, and returns, so that we can ensure that our region's count is 217 /// correct in the code that follows. 218 void adjustForControlFlow() { 219 Adjust += PGO->getCurrentRegionCount() - RegionCount; 220 // Reset the region count in case this is called again later. 221 RegionCount = PGO->getCurrentRegionCount(); 222 } 223 224 /// Commit all adjustments to the current region. If the region is a loop, 225 /// the LoopAdjust value should be the count of all the breaks and continues 226 /// from the loop, to compensate for those counts being deducted from the 227 /// adjustments for the body of the loop. 228 void applyAdjustmentsToRegion(uint64_t LoopAdjust) { 229 PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust); 230 } 231 }; 232 233 } // end namespace CodeGen 234 } // end namespace clang 235 236 #endif 237
