1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "CGLoopInfo.h" 11 #include "clang/AST/ASTContext.h" 12 #include "clang/AST/Attr.h" 13 #include "clang/Sema/LoopHint.h" 14 #include "llvm/IR/BasicBlock.h" 15 #include "llvm/IR/Constants.h" 16 #include "llvm/IR/InstrTypes.h" 17 #include "llvm/IR/Instructions.h" 18 #include "llvm/IR/Metadata.h" 19 using namespace clang::CodeGen; 20 using namespace llvm; 21 22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, 23 llvm::DebugLoc Location) { 24 25 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && 26 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && 27 Attrs.VectorizeEnable == LoopAttributes::Unspecified && 28 Attrs.UnrollEnable == LoopAttributes::Unspecified && 29 Attrs.DistributeEnable == LoopAttributes::Unspecified && 30 !Location) 31 return nullptr; 32 33 SmallVector<Metadata *, 4> Args; 34 // Reserve operand 0 for loop id self reference. 35 auto TempNode = MDNode::getTemporary(Ctx, None); 36 Args.push_back(TempNode.get()); 37 38 // If we have a valid debug location for the loop, add it. 39 if (Location) 40 Args.push_back(Location.getAsMDNode()); 41 42 // Setting vectorize.width 43 if (Attrs.VectorizeWidth > 0) { 44 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), 45 ConstantAsMetadata::get(ConstantInt::get( 46 Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; 47 Args.push_back(MDNode::get(Ctx, Vals)); 48 } 49 50 // Setting interleave.count 51 if (Attrs.InterleaveCount > 0) { 52 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), 53 ConstantAsMetadata::get(ConstantInt::get( 54 Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; 55 Args.push_back(MDNode::get(Ctx, Vals)); 56 } 57 58 // Setting interleave.count 59 if (Attrs.UnrollCount > 0) { 60 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), 61 ConstantAsMetadata::get(ConstantInt::get( 62 Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; 63 Args.push_back(MDNode::get(Ctx, Vals)); 64 } 65 66 // Setting vectorize.enable 67 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { 68 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), 69 ConstantAsMetadata::get(ConstantInt::get( 70 Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == 71 LoopAttributes::Enable)))}; 72 Args.push_back(MDNode::get(Ctx, Vals)); 73 } 74 75 // Setting unroll.full or unroll.disable 76 if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { 77 std::string Name; 78 if (Attrs.UnrollEnable == LoopAttributes::Enable) 79 Name = "llvm.loop.unroll.enable"; 80 else if (Attrs.UnrollEnable == LoopAttributes::Full) 81 Name = "llvm.loop.unroll.full"; 82 else 83 Name = "llvm.loop.unroll.disable"; 84 Metadata *Vals[] = {MDString::get(Ctx, Name)}; 85 Args.push_back(MDNode::get(Ctx, Vals)); 86 } 87 88 if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { 89 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), 90 ConstantAsMetadata::get(ConstantInt::get( 91 Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == 92 LoopAttributes::Enable)))}; 93 Args.push_back(MDNode::get(Ctx, Vals)); 94 } 95 96 // Set the first operand to itself. 97 MDNode *LoopID = MDNode::get(Ctx, Args); 98 LoopID->replaceOperandWith(0, LoopID); 99 return LoopID; 100 } 101 102 LoopAttributes::LoopAttributes(bool IsParallel) 103 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), 104 UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0), 105 InterleaveCount(0), UnrollCount(0), 106 DistributeEnable(LoopAttributes::Unspecified) {} 107 108 void LoopAttributes::clear() { 109 IsParallel = false; 110 VectorizeWidth = 0; 111 InterleaveCount = 0; 112 UnrollCount = 0; 113 VectorizeEnable = LoopAttributes::Unspecified; 114 UnrollEnable = LoopAttributes::Unspecified; 115 } 116 117 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, 118 llvm::DebugLoc Location) 119 : LoopID(nullptr), Header(Header), Attrs(Attrs) { 120 LoopID = createMetadata(Header->getContext(), Attrs, Location); 121 } 122 123 void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) { 124 Active.push_back(LoopInfo(Header, StagedAttrs, Location)); 125 // Clear the attributes so nested loops do not inherit them. 126 StagedAttrs.clear(); 127 } 128 129 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, 130 ArrayRef<const clang::Attr *> Attrs, 131 llvm::DebugLoc Location) { 132 133 // Identify loop hint attributes from Attrs. 134 for (const auto *Attr : Attrs) { 135 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); 136 const OpenCLUnrollHintAttr *OpenCLHint = 137 dyn_cast<OpenCLUnrollHintAttr>(Attr); 138 139 // Skip non loop hint attributes 140 if (!LH && !OpenCLHint) { 141 continue; 142 } 143 144 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; 145 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; 146 unsigned ValueInt = 1; 147 // Translate opencl_unroll_hint attribute argument to 148 // equivalent LoopHintAttr enums. 149 // OpenCL v2.0 s6.11.5: 150 // 0 - full unroll (no argument). 151 // 1 - disable unroll. 152 // other positive integer n - unroll by n. 153 if (OpenCLHint) { 154 ValueInt = OpenCLHint->getUnrollHint(); 155 if (ValueInt == 0) { 156 State = LoopHintAttr::Full; 157 } else if (ValueInt != 1) { 158 Option = LoopHintAttr::UnrollCount; 159 State = LoopHintAttr::Numeric; 160 } 161 } else if (LH) { 162 auto *ValueExpr = LH->getValue(); 163 if (ValueExpr) { 164 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); 165 ValueInt = ValueAPS.getSExtValue(); 166 } 167 168 Option = LH->getOption(); 169 State = LH->getState(); 170 } 171 switch (State) { 172 case LoopHintAttr::Disable: 173 switch (Option) { 174 case LoopHintAttr::Vectorize: 175 // Disable vectorization by specifying a width of 1. 176 setVectorizeWidth(1); 177 break; 178 case LoopHintAttr::Interleave: 179 // Disable interleaving by speciyfing a count of 1. 180 setInterleaveCount(1); 181 break; 182 case LoopHintAttr::Unroll: 183 setUnrollState(LoopAttributes::Disable); 184 break; 185 case LoopHintAttr::Distribute: 186 setDistributeState(false); 187 break; 188 case LoopHintAttr::UnrollCount: 189 case LoopHintAttr::VectorizeWidth: 190 case LoopHintAttr::InterleaveCount: 191 llvm_unreachable("Options cannot be disabled."); 192 break; 193 } 194 break; 195 case LoopHintAttr::Enable: 196 switch (Option) { 197 case LoopHintAttr::Vectorize: 198 case LoopHintAttr::Interleave: 199 setVectorizeEnable(true); 200 break; 201 case LoopHintAttr::Unroll: 202 setUnrollState(LoopAttributes::Enable); 203 break; 204 case LoopHintAttr::Distribute: 205 setDistributeState(true); 206 break; 207 case LoopHintAttr::UnrollCount: 208 case LoopHintAttr::VectorizeWidth: 209 case LoopHintAttr::InterleaveCount: 210 llvm_unreachable("Options cannot enabled."); 211 break; 212 } 213 break; 214 case LoopHintAttr::AssumeSafety: 215 switch (Option) { 216 case LoopHintAttr::Vectorize: 217 case LoopHintAttr::Interleave: 218 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. 219 setParallel(true); 220 setVectorizeEnable(true); 221 break; 222 case LoopHintAttr::Unroll: 223 case LoopHintAttr::UnrollCount: 224 case LoopHintAttr::VectorizeWidth: 225 case LoopHintAttr::InterleaveCount: 226 case LoopHintAttr::Distribute: 227 llvm_unreachable("Options cannot be used to assume mem safety."); 228 break; 229 } 230 break; 231 case LoopHintAttr::Full: 232 switch (Option) { 233 case LoopHintAttr::Unroll: 234 setUnrollState(LoopAttributes::Full); 235 break; 236 case LoopHintAttr::Vectorize: 237 case LoopHintAttr::Interleave: 238 case LoopHintAttr::UnrollCount: 239 case LoopHintAttr::VectorizeWidth: 240 case LoopHintAttr::InterleaveCount: 241 case LoopHintAttr::Distribute: 242 llvm_unreachable("Options cannot be used with 'full' hint."); 243 break; 244 } 245 break; 246 case LoopHintAttr::Numeric: 247 switch (Option) { 248 case LoopHintAttr::VectorizeWidth: 249 setVectorizeWidth(ValueInt); 250 break; 251 case LoopHintAttr::InterleaveCount: 252 setInterleaveCount(ValueInt); 253 break; 254 case LoopHintAttr::UnrollCount: 255 setUnrollCount(ValueInt); 256 break; 257 case LoopHintAttr::Unroll: 258 case LoopHintAttr::Vectorize: 259 case LoopHintAttr::Interleave: 260 case LoopHintAttr::Distribute: 261 llvm_unreachable("Options cannot be assigned a value."); 262 break; 263 } 264 break; 265 } 266 } 267 268 /// Stage the attributes. 269 push(Header, Location); 270 } 271 272 void LoopInfoStack::pop() { 273 assert(!Active.empty() && "No active loops to pop"); 274 Active.pop_back(); 275 } 276 277 void LoopInfoStack::InsertHelper(Instruction *I) const { 278 if (!hasInfo()) 279 return; 280 281 const LoopInfo &L = getInfo(); 282 if (!L.getLoopID()) 283 return; 284 285 if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) { 286 for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i) 287 if (TI->getSuccessor(i) == L.getHeader()) { 288 TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID()); 289 break; 290 } 291 return; 292 } 293 294 if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory()) 295 I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID()); 296 } 297
