1 //===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // Instrumentation-based profile-guided optimization 11 // 12 //===----------------------------------------------------------------------===// 13 14 #ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H 15 #define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H 16 17 #include "CGBuilder.h" 18 #include "CodeGenModule.h" 19 #include "CodeGenTypes.h" 20 #include "clang/Frontend/CodeGenOptions.h" 21 #include "llvm/ADT/StringMap.h" 22 #include "llvm/Support/MemoryBuffer.h" 23 #include <memory> 24 25 namespace clang { 26 namespace CodeGen { 27 class RegionCounter; 28 29 /// Per-function PGO state. This class should generally not be used directly, 30 /// but instead through the CodeGenFunction and RegionCounter types. 31 class CodeGenPGO { 32 private: 33 CodeGenModule &CGM; 34 std::string FuncName; 35 llvm::GlobalVariable *FuncNameVar; 36 37 unsigned NumRegionCounters; 38 uint64_t FunctionHash; 39 std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap; 40 std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap; 41 std::vector<uint64_t> RegionCounts; 42 uint64_t CurrentRegionCount; 43 /// \brief A flag that is set to true when this function doesn't need 44 /// to have coverage mapping data. 45 bool SkipCoverageMapping; 46 47 public: 48 CodeGenPGO(CodeGenModule &CGM) 49 : CGM(CGM), NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0), 50 SkipCoverageMapping(false) {} 51 52 /// Whether or not we have PGO region data for the current function. This is 53 /// false both when we have no data at all and when our data has been 54 /// discarded. 55 bool haveRegionCounts() const { return !RegionCounts.empty(); } 56 57 /// Return the counter value of the current region. 58 uint64_t getCurrentRegionCount() const { return CurrentRegionCount; } 59 60 /// Set the counter value for the current region. This is used to keep track 61 /// of changes to the most recent counter from control flow and non-local 62 /// exits. 63 void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } 64 65 /// Indicate that the current region is never reached, and thus should have a 66 /// counter value of zero. This is important so that subsequent regions can 67 /// correctly track their parent counts. 68 void setCurrentRegionUnreachable() { setCurrentRegionCount(0); } 69 70 /// Check if an execution count is known for a given statement. If so, return 71 /// true and put the value in Count; else return false. 72 Optional<uint64_t> getStmtCount(const Stmt *S) { 73 if (!StmtCountMap) 74 return None; 75 auto I = StmtCountMap->find(S); 76 if (I == StmtCountMap->end()) 77 return None; 78 return I->second; 79 } 80 81 /// If the execution count for the current statement is known, record that 82 /// as the current count. 83 void setCurrentStmt(const Stmt *S) { 84 if (auto Count = getStmtCount(S)) 85 setCurrentRegionCount(*Count); 86 } 87 88 /// Calculate branch weights appropriate for PGO data 89 llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount); 90 llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights); 91 llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt); 92 93 /// Check if we need to emit coverage mapping for a given declaration 94 void checkGlobalDecl(GlobalDecl GD); 95 /// Assign counters to regions and configure them for PGO of a given 96 /// function. Does nothing if instrumentation is not enabled and either 97 /// generates global variables or associates PGO data with each of the 98 /// counters depending on whether we are generating or using instrumentation. 99 void assignRegionCounters(const Decl *D, llvm::Function *Fn); 100 /// Emit a coverage mapping range with a counter zero 101 /// for an unused declaration. 102 void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, 103 llvm::GlobalValue::LinkageTypes Linkage); 104 private: 105 void setFuncName(llvm::Function *Fn); 106 void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage); 107 void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage); 108 void mapRegionCounters(const Decl *D); 109 void computeRegionCounts(const Decl *D); 110 void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, 111 llvm::Function *Fn); 112 void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, 113 bool IsInMainFile); 114 void emitCounterVariables(); 115 void emitCounterRegionMapping(const Decl *D); 116 117 /// Emit code to increment the counter at the given index 118 void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter); 119 120 /// Return the region counter for the given statement. This should only be 121 /// called on statements that have a dedicated counter. 122 unsigned getRegionCounter(const Stmt *S) { 123 if (!RegionCounterMap) 124 return 0; 125 return (*RegionCounterMap)[S]; 126 } 127 128 /// Return the region count for the counter at the given index. 129 uint64_t getRegionCount(unsigned Counter) { 130 if (!haveRegionCounts()) 131 return 0; 132 return RegionCounts[Counter]; 133 } 134 135 friend class RegionCounter; 136 }; 137 138 /// A counter for a particular region. This is the primary interface through 139 /// which clients manage PGO counters and their values. 140 class RegionCounter { 141 CodeGenPGO *PGO; 142 unsigned Counter; 143 uint64_t Count; 144 uint64_t ParentCount; 145 uint64_t RegionCount; 146 int64_t Adjust; 147 148 RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex) 149 : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)), 150 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} 151 152 public: 153 RegionCounter(CodeGenPGO &PGO, const Stmt *S) 154 : PGO(&PGO), Counter(PGO.getRegionCounter(S)), 155 Count(PGO.getRegionCount(Counter)), 156 ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {} 157 158 /// Get the value of the counter. In most cases this is the number of times 159 /// the region of the counter was entered, but for switch labels it's the 160 /// number of direct jumps to that label. 161 uint64_t getCount() const { return Count; } 162 163 /// Get the value of the counter with adjustments applied. Adjustments occur 164 /// when control enters or leaves the region abnormally; i.e., if there is a 165 /// jump to a label within the region, or if the function can return from 166 /// within the region. The adjusted count, then, is the value of the counter 167 /// at the end of the region. 168 uint64_t getAdjustedCount() const { 169 return Count + Adjust; 170 } 171 172 /// Get the value of the counter in this region's parent, i.e., the region 173 /// that was active when this region began. This is useful for deriving 174 /// counts in implicitly counted regions, like the false case of a condition 175 /// or the normal exits of a loop. 176 uint64_t getParentCount() const { return ParentCount; } 177 178 /// Activate the counter by emitting an increment and starting to track 179 /// adjustments. If AddIncomingFallThrough is true, the current region count 180 /// will be added to the counter for the purposes of tracking the region. 181 void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) { 182 beginRegion(AddIncomingFallThrough); 183 PGO->emitCounterIncrement(Builder, Counter); 184 } 185 void beginRegion(bool AddIncomingFallThrough=false) { 186 RegionCount = Count; 187 if (AddIncomingFallThrough) 188 RegionCount += PGO->getCurrentRegionCount(); 189 PGO->setCurrentRegionCount(RegionCount); 190 } 191 192 /// For counters on boolean branches, begins tracking adjustments for the 193 /// uncounted path. 194 void beginElseRegion() { 195 RegionCount = ParentCount - Count; 196 PGO->setCurrentRegionCount(RegionCount); 197 } 198 199 /// Reset the current region count. 200 void setCurrentRegionCount(uint64_t CurrentCount) { 201 RegionCount = CurrentCount; 202 PGO->setCurrentRegionCount(RegionCount); 203 } 204 205 /// Adjust for non-local control flow after emitting a subexpression or 206 /// substatement. This must be called to account for constructs such as gotos, 207 /// labels, and returns, so that we can ensure that our region's count is 208 /// correct in the code that follows. 209 void adjustForControlFlow() { 210 Adjust += PGO->getCurrentRegionCount() - RegionCount; 211 // Reset the region count in case this is called again later. 212 RegionCount = PGO->getCurrentRegionCount(); 213 } 214 215 /// Commit all adjustments to the current region. If the region is a loop, 216 /// the LoopAdjust value should be the count of all the breaks and continues 217 /// from the loop, to compensate for those counts being deducted from the 218 /// adjustments for the body of the loop. 219 void applyAdjustmentsToRegion(uint64_t LoopAdjust) { 220 PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust); 221 } 222 }; 223 224 } // end namespace CodeGen 225 } // end namespace clang 226 227 #endif 228
