1 //===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This pass lowers the 'expect' intrinsic to LLVM metadata. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Scalar.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/IR/BasicBlock.h" 17 #include "llvm/IR/Constants.h" 18 #include "llvm/IR/Function.h" 19 #include "llvm/IR/Instructions.h" 20 #include "llvm/IR/Intrinsics.h" 21 #include "llvm/IR/LLVMContext.h" 22 #include "llvm/IR/MDBuilder.h" 23 #include "llvm/IR/Metadata.h" 24 #include "llvm/Pass.h" 25 #include "llvm/Support/CommandLine.h" 26 #include "llvm/Support/Debug.h" 27 #include <vector> 28 29 using namespace llvm; 30 31 #define DEBUG_TYPE "lower-expect-intrinsic" 32 33 STATISTIC(IfHandled, "Number of 'expect' intrinsic instructions handled"); 34 35 static cl::opt<uint32_t> 36 LikelyBranchWeight("likely-branch-weight", cl::Hidden, cl::init(64), 37 cl::desc("Weight of the branch likely to be taken (default = 64)")); 38 static cl::opt<uint32_t> 39 UnlikelyBranchWeight("unlikely-branch-weight", cl::Hidden, cl::init(4), 40 cl::desc("Weight of the branch unlikely to be taken (default = 4)")); 41 42 namespace { 43 44 class LowerExpectIntrinsic : public FunctionPass { 45 46 bool HandleSwitchExpect(SwitchInst *SI); 47 48 bool HandleIfExpect(BranchInst *BI); 49 50 public: 51 static char ID; 52 LowerExpectIntrinsic() : FunctionPass(ID) { 53 initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); 54 } 55 56 bool runOnFunction(Function &F) override; 57 }; 58 } 59 60 61 bool LowerExpectIntrinsic::HandleSwitchExpect(SwitchInst *SI) { 62 CallInst *CI = dyn_cast<CallInst>(SI->getCondition()); 63 if (!CI) 64 return false; 65 66 Function *Fn = CI->getCalledFunction(); 67 if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 68 return false; 69 70 Value *ArgValue = CI->getArgOperand(0); 71 ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 72 if (!ExpectedValue) 73 return false; 74 75 SwitchInst::CaseIt Case = SI->findCaseValue(ExpectedValue); 76 unsigned n = SI->getNumCases(); // +1 for default case. 77 std::vector<uint32_t> Weights(n + 1); 78 79 Weights[0] = Case == SI->case_default() ? LikelyBranchWeight 80 : UnlikelyBranchWeight; 81 for (unsigned i = 0; i != n; ++i) 82 Weights[i + 1] = i == Case.getCaseIndex() ? LikelyBranchWeight 83 : UnlikelyBranchWeight; 84 85 SI->setMetadata(LLVMContext::MD_prof, 86 MDBuilder(CI->getContext()).createBranchWeights(Weights)); 87 88 SI->setCondition(ArgValue); 89 return true; 90 } 91 92 93 bool LowerExpectIntrinsic::HandleIfExpect(BranchInst *BI) { 94 if (BI->isUnconditional()) 95 return false; 96 97 // Handle non-optimized IR code like: 98 // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) 99 // %tobool = icmp ne i64 %expval, 0 100 // br i1 %tobool, label %if.then, label %if.end 101 // 102 // Or the following simpler case: 103 // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) 104 // br i1 %expval, label %if.then, label %if.end 105 106 CallInst *CI; 107 108 ICmpInst *CmpI = dyn_cast<ICmpInst>(BI->getCondition()); 109 if (!CmpI) { 110 CI = dyn_cast<CallInst>(BI->getCondition()); 111 } else { 112 if (CmpI->getPredicate() != CmpInst::ICMP_NE) 113 return false; 114 CI = dyn_cast<CallInst>(CmpI->getOperand(0)); 115 } 116 117 if (!CI) 118 return false; 119 120 Function *Fn = CI->getCalledFunction(); 121 if (!Fn || Fn->getIntrinsicID() != Intrinsic::expect) 122 return false; 123 124 Value *ArgValue = CI->getArgOperand(0); 125 ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); 126 if (!ExpectedValue) 127 return false; 128 129 MDBuilder MDB(CI->getContext()); 130 MDNode *Node; 131 132 // If expect value is equal to 1 it means that we are more likely to take 133 // branch 0, in other case more likely is branch 1. 134 if (ExpectedValue->isOne()) 135 Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); 136 else 137 Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); 138 139 BI->setMetadata(LLVMContext::MD_prof, Node); 140 141 if (CmpI) 142 CmpI->setOperand(0, ArgValue); 143 else 144 BI->setCondition(ArgValue); 145 return true; 146 } 147 148 149 bool LowerExpectIntrinsic::runOnFunction(Function &F) { 150 for (Function::iterator I = F.begin(), E = F.end(); I != E;) { 151 BasicBlock *BB = I++; 152 153 // Create "block_weights" metadata. 154 if (BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator())) { 155 if (HandleIfExpect(BI)) 156 IfHandled++; 157 } else if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator())) { 158 if (HandleSwitchExpect(SI)) 159 IfHandled++; 160 } 161 162 // remove llvm.expect intrinsics. 163 for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); 164 BI != BE; ) { 165 CallInst *CI = dyn_cast<CallInst>(BI++); 166 if (!CI) 167 continue; 168 169 Function *Fn = CI->getCalledFunction(); 170 if (Fn && Fn->getIntrinsicID() == Intrinsic::expect) { 171 Value *Exp = CI->getArgOperand(0); 172 CI->replaceAllUsesWith(Exp); 173 CI->eraseFromParent(); 174 } 175 } 176 } 177 178 return false; 179 } 180 181 182 char LowerExpectIntrinsic::ID = 0; 183 INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", "Lower 'expect' " 184 "Intrinsics", false, false) 185 186 FunctionPass *llvm::createLowerExpectIntrinsicPass() { 187 return new LowerExpectIntrinsic(); 188 } 189
