1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the auto-upgrade helper functions 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/AutoUpgrade.h" 15 #include "llvm/IR/Constants.h" 16 #include "llvm/IR/Function.h" 17 #include "llvm/IR/IRBuilder.h" 18 #include "llvm/IR/Instruction.h" 19 #include "llvm/IR/IntrinsicInst.h" 20 #include "llvm/IR/LLVMContext.h" 21 #include "llvm/IR/Module.h" 22 #include "llvm/Support/CFG.h" 23 #include "llvm/Support/CallSite.h" 24 #include "llvm/Support/ErrorHandling.h" 25 #include <cstring> 26 using namespace llvm; 27 28 // Upgrade the declarations of the SSE4.1 functions whose arguments have 29 // changed their type from v4f32 to v2i64. 30 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, 31 Function *&NewFn) { 32 // Check whether this is an old version of the function, which received 33 // v4f32 arguments. 34 Type *Arg0Type = F->getFunctionType()->getParamType(0); 35 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 36 return false; 37 38 // Yes, it's old, replace it with new version. 39 F->setName(F->getName() + ".old"); 40 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 41 return true; 42 } 43 44 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 45 assert(F && "Illegal to upgrade a non-existent Function."); 46 47 // Quickly eliminate it, if it's not a candidate. 48 StringRef Name = F->getName(); 49 if (Name.size() <= 8 || !Name.startswith("llvm.")) 50 return false; 51 Name = Name.substr(5); // Strip off "llvm." 52 53 switch (Name[0]) { 54 default: break; 55 case 'a': { 56 if (Name.startswith("arm.neon.vclz")) { 57 Type* args[2] = { 58 F->arg_begin()->getType(), 59 Type::getInt1Ty(F->getContext()) 60 }; 61 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 62 // the end of the name. Change name from llvm.arm.neon.vclz.* to 63 // llvm.ctlz.* 64 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 65 NewFn = Function::Create(fType, F->getLinkage(), 66 "llvm.ctlz." + Name.substr(14), F->getParent()); 67 return true; 68 } 69 if (Name.startswith("arm.neon.vcnt")) { 70 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 71 F->arg_begin()->getType()); 72 return true; 73 } 74 break; 75 } 76 case 'c': { 77 if (Name.startswith("ctlz.") && F->arg_size() == 1) { 78 F->setName(Name + ".old"); 79 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 80 F->arg_begin()->getType()); 81 return true; 82 } 83 if (Name.startswith("cttz.") && F->arg_size() == 1) { 84 F->setName(Name + ".old"); 85 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 86 F->arg_begin()->getType()); 87 return true; 88 } 89 break; 90 } 91 case 'x': { 92 if (Name.startswith("x86.sse2.pcmpeq.") || 93 Name.startswith("x86.sse2.pcmpgt.") || 94 Name.startswith("x86.avx2.pcmpeq.") || 95 Name.startswith("x86.avx2.pcmpgt.") || 96 Name.startswith("x86.avx.vpermil.") || 97 Name == "x86.avx.movnt.dq.256" || 98 Name == "x86.avx.movnt.pd.256" || 99 Name == "x86.avx.movnt.ps.256" || 100 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { 101 NewFn = 0; 102 return true; 103 } 104 // SSE4.1 ptest functions may have an old signature. 105 if (Name.startswith("x86.sse41.ptest")) { 106 if (Name == "x86.sse41.ptestc") 107 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); 108 if (Name == "x86.sse41.ptestz") 109 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn); 110 if (Name == "x86.sse41.ptestnzc") 111 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 112 } 113 // frcz.ss/sd may need to have an argument dropped 114 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) { 115 F->setName(Name + ".old"); 116 NewFn = Intrinsic::getDeclaration(F->getParent(), 117 Intrinsic::x86_xop_vfrcz_ss); 118 return true; 119 } 120 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) { 121 F->setName(Name + ".old"); 122 NewFn = Intrinsic::getDeclaration(F->getParent(), 123 Intrinsic::x86_xop_vfrcz_sd); 124 return true; 125 } 126 // Fix the FMA4 intrinsics to remove the 4 127 if (Name.startswith("x86.fma4.")) { 128 F->setName("llvm.x86.fma" + Name.substr(8)); 129 NewFn = F; 130 return true; 131 } 132 break; 133 } 134 } 135 136 // This may not belong here. This function is effectively being overloaded 137 // to both detect an intrinsic which needs upgrading, and to provide the 138 // upgraded form of the intrinsic. We should perhaps have two separate 139 // functions for this. 140 return false; 141 } 142 143 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 144 NewFn = 0; 145 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 146 147 // Upgrade intrinsic attributes. This does not change the function. 148 if (NewFn) 149 F = NewFn; 150 if (unsigned id = F->getIntrinsicID()) 151 F->setAttributes(Intrinsic::getAttributes(F->getContext(), 152 (Intrinsic::ID)id)); 153 return Upgraded; 154 } 155 156 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 157 // Nothing to do yet. 158 return false; 159 } 160 161 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the 162 // upgraded intrinsic. All argument and return casting must be provided in 163 // order to seamlessly integrate with existing context. 164 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 165 Function *F = CI->getCalledFunction(); 166 LLVMContext &C = CI->getContext(); 167 IRBuilder<> Builder(C); 168 Builder.SetInsertPoint(CI->getParent(), CI); 169 170 assert(F && "Intrinsic call is not direct?"); 171 172 if (!NewFn) { 173 // Get the Function's name. 174 StringRef Name = F->getName(); 175 176 Value *Rep; 177 // Upgrade packed integer vector compares intrinsics to compare instructions 178 if (Name.startswith("llvm.x86.sse2.pcmpeq.") || 179 Name.startswith("llvm.x86.avx2.pcmpeq.")) { 180 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), 181 "pcmpeq"); 182 // need to sign extend since icmp returns vector of i1 183 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 184 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") || 185 Name.startswith("llvm.x86.avx2.pcmpgt.")) { 186 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), 187 "pcmpgt"); 188 // need to sign extend since icmp returns vector of i1 189 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 190 } else if (Name == "llvm.x86.avx.movnt.dq.256" || 191 Name == "llvm.x86.avx.movnt.ps.256" || 192 Name == "llvm.x86.avx.movnt.pd.256") { 193 IRBuilder<> Builder(C); 194 Builder.SetInsertPoint(CI->getParent(), CI); 195 196 Module *M = F->getParent(); 197 SmallVector<Value *, 1> Elts; 198 Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1)); 199 MDNode *Node = MDNode::get(C, Elts); 200 201 Value *Arg0 = CI->getArgOperand(0); 202 Value *Arg1 = CI->getArgOperand(1); 203 204 // Convert the type of the pointer to a pointer to the stored type. 205 Value *BC = Builder.CreateBitCast(Arg0, 206 PointerType::getUnqual(Arg1->getType()), 207 "cast"); 208 StoreInst *SI = Builder.CreateStore(Arg1, BC); 209 SI->setMetadata(M->getMDKindID("nontemporal"), Node); 210 SI->setAlignment(16); 211 212 // Remove intrinsic. 213 CI->eraseFromParent(); 214 return; 215 } else if (Name.startswith("llvm.x86.xop.vpcom")) { 216 Intrinsic::ID intID; 217 if (Name.endswith("ub")) 218 intID = Intrinsic::x86_xop_vpcomub; 219 else if (Name.endswith("uw")) 220 intID = Intrinsic::x86_xop_vpcomuw; 221 else if (Name.endswith("ud")) 222 intID = Intrinsic::x86_xop_vpcomud; 223 else if (Name.endswith("uq")) 224 intID = Intrinsic::x86_xop_vpcomuq; 225 else if (Name.endswith("b")) 226 intID = Intrinsic::x86_xop_vpcomb; 227 else if (Name.endswith("w")) 228 intID = Intrinsic::x86_xop_vpcomw; 229 else if (Name.endswith("d")) 230 intID = Intrinsic::x86_xop_vpcomd; 231 else if (Name.endswith("q")) 232 intID = Intrinsic::x86_xop_vpcomq; 233 else 234 llvm_unreachable("Unknown suffix"); 235 236 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom" 237 unsigned Imm; 238 if (Name.startswith("lt")) 239 Imm = 0; 240 else if (Name.startswith("le")) 241 Imm = 1; 242 else if (Name.startswith("gt")) 243 Imm = 2; 244 else if (Name.startswith("ge")) 245 Imm = 3; 246 else if (Name.startswith("eq")) 247 Imm = 4; 248 else if (Name.startswith("ne")) 249 Imm = 5; 250 else if (Name.startswith("true")) 251 Imm = 6; 252 else if (Name.startswith("false")) 253 Imm = 7; 254 else 255 llvm_unreachable("Unknown condition"); 256 257 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID); 258 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0), 259 CI->getArgOperand(1), Builder.getInt8(Imm)); 260 } else { 261 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; 262 if (Name == "llvm.x86.avx.vpermil.pd.256") 263 PD256 = true; 264 else if (Name == "llvm.x86.avx.vpermil.pd") 265 PD128 = true; 266 else if (Name == "llvm.x86.avx.vpermil.ps.256") 267 PS256 = true; 268 else if (Name == "llvm.x86.avx.vpermil.ps") 269 PS128 = true; 270 271 if (PD256 || PD128 || PS256 || PS128) { 272 Value *Op0 = CI->getArgOperand(0); 273 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 274 SmallVector<Constant*, 8> Idxs; 275 276 if (PD128) 277 for (unsigned i = 0; i != 2; ++i) 278 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1)); 279 else if (PD256) 280 for (unsigned l = 0; l != 4; l+=2) 281 for (unsigned i = 0; i != 2; ++i) 282 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l)); 283 else if (PS128) 284 for (unsigned i = 0; i != 4; ++i) 285 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3)); 286 else if (PS256) 287 for (unsigned l = 0; l != 8; l+=4) 288 for (unsigned i = 0; i != 4; ++i) 289 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l)); 290 else 291 llvm_unreachable("Unexpected function"); 292 293 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs)); 294 } else { 295 llvm_unreachable("Unknown function for CallInst upgrade."); 296 } 297 } 298 299 CI->replaceAllUsesWith(Rep); 300 CI->eraseFromParent(); 301 return; 302 } 303 304 std::string Name = CI->getName().str(); 305 CI->setName(Name + ".old"); 306 307 switch (NewFn->getIntrinsicID()) { 308 default: 309 llvm_unreachable("Unknown function for CallInst upgrade."); 310 311 case Intrinsic::ctlz: 312 case Intrinsic::cttz: 313 assert(CI->getNumArgOperands() == 1 && 314 "Mismatch between function args and call args"); 315 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 316 Builder.getFalse(), Name)); 317 CI->eraseFromParent(); 318 return; 319 320 case Intrinsic::arm_neon_vclz: { 321 // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.* 322 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 323 Builder.getFalse(), 324 "llvm.ctlz." + Name.substr(14))); 325 CI->eraseFromParent(); 326 return; 327 } 328 case Intrinsic::ctpop: { 329 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0))); 330 CI->eraseFromParent(); 331 return; 332 } 333 334 case Intrinsic::x86_xop_vfrcz_ss: 335 case Intrinsic::x86_xop_vfrcz_sd: 336 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1), 337 Name)); 338 CI->eraseFromParent(); 339 return; 340 341 case Intrinsic::x86_sse41_ptestc: 342 case Intrinsic::x86_sse41_ptestz: 343 case Intrinsic::x86_sse41_ptestnzc: { 344 // The arguments for these intrinsics used to be v4f32, and changed 345 // to v2i64. This is purely a nop, since those are bitwise intrinsics. 346 // So, the only thing required is a bitcast for both arguments. 347 // First, check the arguments have the old type. 348 Value *Arg0 = CI->getArgOperand(0); 349 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 350 return; 351 352 // Old intrinsic, add bitcasts 353 Value *Arg1 = CI->getArgOperand(1); 354 355 Value *BC0 = 356 Builder.CreateBitCast(Arg0, 357 VectorType::get(Type::getInt64Ty(C), 2), 358 "cast"); 359 Value *BC1 = 360 Builder.CreateBitCast(Arg1, 361 VectorType::get(Type::getInt64Ty(C), 2), 362 "cast"); 363 364 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name); 365 CI->replaceAllUsesWith(NewCall); 366 CI->eraseFromParent(); 367 return; 368 } 369 } 370 } 371 372 // This tests each Function to determine if it needs upgrading. When we find 373 // one we are interested in, we then upgrade all calls to reflect the new 374 // function. 375 void llvm::UpgradeCallsToIntrinsic(Function* F) { 376 assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 377 378 // Upgrade the function and check if it is a totaly new function. 379 Function *NewFn; 380 if (UpgradeIntrinsicFunction(F, NewFn)) { 381 if (NewFn != F) { 382 // Replace all uses to the old function with the new one if necessary. 383 for (Value::use_iterator UI = F->use_begin(), UE = F->use_end(); 384 UI != UE; ) { 385 if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 386 UpgradeIntrinsicCall(CI, NewFn); 387 } 388 // Remove old function, no longer used, from the module. 389 F->eraseFromParent(); 390 } 391 } 392 } 393 394
