1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "CGLoopInfo.h" 11 #include "clang/AST/ASTContext.h" 12 #include "clang/AST/Attr.h" 13 #include "clang/Sema/LoopHint.h" 14 #include "llvm/IR/BasicBlock.h" 15 #include "llvm/IR/Constants.h" 16 #include "llvm/IR/InstrTypes.h" 17 #include "llvm/IR/Instructions.h" 18 #include "llvm/IR/Metadata.h" 19 using namespace clang::CodeGen; 20 using namespace llvm; 21 22 static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs) { 23 24 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && 25 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && 26 Attrs.VectorizeEnable == LoopAttributes::Unspecified && 27 Attrs.UnrollEnable == LoopAttributes::Unspecified) 28 return nullptr; 29 30 SmallVector<Metadata *, 4> Args; 31 // Reserve operand 0 for loop id self reference. 32 auto TempNode = MDNode::getTemporary(Ctx, None); 33 Args.push_back(TempNode.get()); 34 35 // Setting vectorize.width 36 if (Attrs.VectorizeWidth > 0) { 37 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), 38 ConstantAsMetadata::get(ConstantInt::get( 39 Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; 40 Args.push_back(MDNode::get(Ctx, Vals)); 41 } 42 43 // Setting interleave.count 44 if (Attrs.InterleaveCount > 0) { 45 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), 46 ConstantAsMetadata::get(ConstantInt::get( 47 Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; 48 Args.push_back(MDNode::get(Ctx, Vals)); 49 } 50 51 // Setting interleave.count 52 if (Attrs.UnrollCount > 0) { 53 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), 54 ConstantAsMetadata::get(ConstantInt::get( 55 Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; 56 Args.push_back(MDNode::get(Ctx, Vals)); 57 } 58 59 // Setting vectorize.enable 60 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { 61 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), 62 ConstantAsMetadata::get(ConstantInt::get( 63 Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == 64 LoopAttributes::Enable)))}; 65 Args.push_back(MDNode::get(Ctx, Vals)); 66 } 67 68 // Setting unroll.full or unroll.disable 69 if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { 70 std::string Name; 71 if (Attrs.UnrollEnable == LoopAttributes::Enable) 72 Name = "llvm.loop.unroll.enable"; 73 else if (Attrs.UnrollEnable == LoopAttributes::Full) 74 Name = "llvm.loop.unroll.full"; 75 else 76 Name = "llvm.loop.unroll.disable"; 77 Metadata *Vals[] = {MDString::get(Ctx, Name)}; 78 Args.push_back(MDNode::get(Ctx, Vals)); 79 } 80 81 // Set the first operand to itself. 82 MDNode *LoopID = MDNode::get(Ctx, Args); 83 LoopID->replaceOperandWith(0, LoopID); 84 return LoopID; 85 } 86 87 LoopAttributes::LoopAttributes(bool IsParallel) 88 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), 89 UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0), 90 InterleaveCount(0), UnrollCount(0) {} 91 92 void LoopAttributes::clear() { 93 IsParallel = false; 94 VectorizeWidth = 0; 95 InterleaveCount = 0; 96 UnrollCount = 0; 97 VectorizeEnable = LoopAttributes::Unspecified; 98 UnrollEnable = LoopAttributes::Unspecified; 99 } 100 101 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs) 102 : LoopID(nullptr), Header(Header), Attrs(Attrs) { 103 LoopID = createMetadata(Header->getContext(), Attrs); 104 } 105 106 void LoopInfoStack::push(BasicBlock *Header) { 107 Active.push_back(LoopInfo(Header, StagedAttrs)); 108 // Clear the attributes so nested loops do not inherit them. 109 StagedAttrs.clear(); 110 } 111 112 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, 113 ArrayRef<const clang::Attr *> Attrs) { 114 115 // Identify loop hint attributes from Attrs. 116 for (const auto *Attr : Attrs) { 117 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); 118 119 // Skip non loop hint attributes 120 if (!LH) 121 continue; 122 123 auto *ValueExpr = LH->getValue(); 124 unsigned ValueInt = 1; 125 if (ValueExpr) { 126 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); 127 ValueInt = ValueAPS.getSExtValue(); 128 } 129 130 LoopHintAttr::OptionType Option = LH->getOption(); 131 LoopHintAttr::LoopHintState State = LH->getState(); 132 switch (State) { 133 case LoopHintAttr::Disable: 134 switch (Option) { 135 case LoopHintAttr::Vectorize: 136 // Disable vectorization by specifying a width of 1. 137 setVectorizeWidth(1); 138 break; 139 case LoopHintAttr::Interleave: 140 // Disable interleaving by speciyfing a count of 1. 141 setInterleaveCount(1); 142 break; 143 case LoopHintAttr::Unroll: 144 setUnrollState(LoopAttributes::Disable); 145 break; 146 case LoopHintAttr::UnrollCount: 147 case LoopHintAttr::VectorizeWidth: 148 case LoopHintAttr::InterleaveCount: 149 llvm_unreachable("Options cannot be disabled."); 150 break; 151 } 152 break; 153 case LoopHintAttr::Enable: 154 switch (Option) { 155 case LoopHintAttr::Vectorize: 156 case LoopHintAttr::Interleave: 157 setVectorizeEnable(true); 158 break; 159 case LoopHintAttr::Unroll: 160 setUnrollState(LoopAttributes::Enable); 161 break; 162 case LoopHintAttr::UnrollCount: 163 case LoopHintAttr::VectorizeWidth: 164 case LoopHintAttr::InterleaveCount: 165 llvm_unreachable("Options cannot enabled."); 166 break; 167 } 168 break; 169 case LoopHintAttr::AssumeSafety: 170 switch (Option) { 171 case LoopHintAttr::Vectorize: 172 case LoopHintAttr::Interleave: 173 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. 174 setParallel(true); 175 setVectorizeEnable(true); 176 break; 177 case LoopHintAttr::Unroll: 178 case LoopHintAttr::UnrollCount: 179 case LoopHintAttr::VectorizeWidth: 180 case LoopHintAttr::InterleaveCount: 181 llvm_unreachable("Options cannot be used to assume mem safety."); 182 break; 183 } 184 break; 185 case LoopHintAttr::Full: 186 switch (Option) { 187 case LoopHintAttr::Unroll: 188 setUnrollState(LoopAttributes::Full); 189 break; 190 case LoopHintAttr::Vectorize: 191 case LoopHintAttr::Interleave: 192 case LoopHintAttr::UnrollCount: 193 case LoopHintAttr::VectorizeWidth: 194 case LoopHintAttr::InterleaveCount: 195 llvm_unreachable("Options cannot be used with 'full' hint."); 196 break; 197 } 198 break; 199 case LoopHintAttr::Numeric: 200 switch (Option) { 201 case LoopHintAttr::VectorizeWidth: 202 setVectorizeWidth(ValueInt); 203 break; 204 case LoopHintAttr::InterleaveCount: 205 setInterleaveCount(ValueInt); 206 break; 207 case LoopHintAttr::UnrollCount: 208 setUnrollCount(ValueInt); 209 break; 210 case LoopHintAttr::Unroll: 211 case LoopHintAttr::Vectorize: 212 case LoopHintAttr::Interleave: 213 llvm_unreachable("Options cannot be assigned a value."); 214 break; 215 } 216 break; 217 } 218 } 219 220 /// Stage the attributes. 221 push(Header); 222 } 223 224 void LoopInfoStack::pop() { 225 assert(!Active.empty() && "No active loops to pop"); 226 Active.pop_back(); 227 } 228 229 void LoopInfoStack::InsertHelper(Instruction *I) const { 230 if (!hasInfo()) 231 return; 232 233 const LoopInfo &L = getInfo(); 234 if (!L.getLoopID()) 235 return; 236 237 if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) { 238 for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i) 239 if (TI->getSuccessor(i) == L.getHeader()) { 240 TI->setMetadata("llvm.loop", L.getLoopID()); 241 break; 242 } 243 return; 244 } 245 246 if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory()) 247 I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID()); 248 } 249
