1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the auto-upgrade helper functions 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/AutoUpgrade.h" 15 #include "llvm/Constants.h" 16 #include "llvm/Function.h" 17 #include "llvm/IRBuilder.h" 18 #include "llvm/Instruction.h" 19 #include "llvm/IntrinsicInst.h" 20 #include "llvm/LLVMContext.h" 21 #include "llvm/Module.h" 22 #include "llvm/Support/CFG.h" 23 #include "llvm/Support/CallSite.h" 24 #include "llvm/Support/ErrorHandling.h" 25 #include <cstring> 26 using namespace llvm; 27 28 // Upgrade the declarations of the SSE4.1 functions whose arguments have 29 // changed their type from v4f32 to v2i64. 30 static bool UpgradeSSE41Function(Function* F, Intrinsic::ID IID, 31 Function *&NewFn) { 32 // Check whether this is an old version of the function, which received 33 // v4f32 arguments. 34 Type *Arg0Type = F->getFunctionType()->getParamType(0); 35 if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 36 return false; 37 38 // Yes, it's old, replace it with new version. 39 F->setName(F->getName() + ".old"); 40 NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 41 return true; 42 } 43 44 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 45 assert(F && "Illegal to upgrade a non-existent Function."); 46 47 // Quickly eliminate it, if it's not a candidate. 48 StringRef Name = F->getName(); 49 if (Name.size() <= 8 || !Name.startswith("llvm.")) 50 return false; 51 Name = Name.substr(5); // Strip off "llvm." 52 53 switch (Name[0]) { 54 default: break; 55 case 'a': { 56 if (Name.startswith("arm.neon.vclz")) { 57 Type* args[2] = { 58 F->arg_begin()->getType(), 59 Type::getInt1Ty(F->getContext()) 60 }; 61 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 62 // the end of the name. Change name from llvm.arm.neon.vclz.* to 63 // llvm.ctlz.* 64 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 65 NewFn = Function::Create(fType, F->getLinkage(), 66 "llvm.ctlz." + Name.substr(14), F->getParent()); 67 return true; 68 } 69 if (Name.startswith("arm.neon.vcnt")) { 70 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 71 F->arg_begin()->getType()); 72 return true; 73 } 74 break; 75 } 76 case 'c': { 77 if (Name.startswith("ctlz.") && F->arg_size() == 1) { 78 F->setName(Name + ".old"); 79 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 80 F->arg_begin()->getType()); 81 return true; 82 } 83 if (Name.startswith("cttz.") && F->arg_size() == 1) { 84 F->setName(Name + ".old"); 85 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 86 F->arg_begin()->getType()); 87 return true; 88 } 89 break; 90 } 91 case 'x': { 92 if (Name.startswith("x86.sse2.pcmpeq.") || 93 Name.startswith("x86.sse2.pcmpgt.") || 94 Name.startswith("x86.avx2.pcmpeq.") || 95 Name.startswith("x86.avx2.pcmpgt.") || 96 Name.startswith("x86.avx.vpermil.") || 97 Name == "x86.avx.movnt.dq.256" || 98 Name == "x86.avx.movnt.pd.256" || 99 Name == "x86.avx.movnt.ps.256" || 100 (Name.startswith("x86.xop.vpcom") && F->arg_size() == 2)) { 101 NewFn = 0; 102 return true; 103 } 104 // SSE4.1 ptest functions may have an old signature. 105 if (Name.startswith("x86.sse41.ptest")) { 106 if (Name == "x86.sse41.ptestc") 107 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestc, NewFn); 108 if (Name == "x86.sse41.ptestz") 109 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestz, NewFn); 110 if (Name == "x86.sse41.ptestnzc") 111 return UpgradeSSE41Function(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 112 } 113 // frcz.ss/sd may need to have an argument dropped 114 if (Name.startswith("x86.xop.vfrcz.ss") && F->arg_size() == 2) { 115 F->setName(Name + ".old"); 116 NewFn = Intrinsic::getDeclaration(F->getParent(), 117 Intrinsic::x86_xop_vfrcz_ss); 118 return true; 119 } 120 if (Name.startswith("x86.xop.vfrcz.sd") && F->arg_size() == 2) { 121 F->setName(Name + ".old"); 122 NewFn = Intrinsic::getDeclaration(F->getParent(), 123 Intrinsic::x86_xop_vfrcz_sd); 124 return true; 125 } 126 // Fix the FMA4 intrinsics to remove the 4 127 if (Name.startswith("x86.fma4.")) { 128 F->setName("llvm.x86.fma" + Name.substr(8)); 129 NewFn = F; 130 return true; 131 } 132 break; 133 } 134 } 135 136 // This may not belong here. This function is effectively being overloaded 137 // to both detect an intrinsic which needs upgrading, and to provide the 138 // upgraded form of the intrinsic. We should perhaps have two separate 139 // functions for this. 140 return false; 141 } 142 143 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 144 NewFn = 0; 145 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 146 147 // Upgrade intrinsic attributes. This does not change the function. 148 if (NewFn) 149 F = NewFn; 150 if (unsigned id = F->getIntrinsicID()) 151 F->setAttributes(Intrinsic::getAttributes((Intrinsic::ID)id)); 152 return Upgraded; 153 } 154 155 bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 156 // Nothing to do yet. 157 return false; 158 } 159 160 // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the 161 // upgraded intrinsic. All argument and return casting must be provided in 162 // order to seamlessly integrate with existing context. 163 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 164 Function *F = CI->getCalledFunction(); 165 LLVMContext &C = CI->getContext(); 166 IRBuilder<> Builder(C); 167 Builder.SetInsertPoint(CI->getParent(), CI); 168 169 assert(F && "Intrinsic call is not direct?"); 170 171 if (!NewFn) { 172 // Get the Function's name. 173 StringRef Name = F->getName(); 174 175 Value *Rep; 176 // Upgrade packed integer vector compares intrinsics to compare instructions 177 if (Name.startswith("llvm.x86.sse2.pcmpeq.") || 178 Name.startswith("llvm.x86.avx2.pcmpeq.")) { 179 Rep = Builder.CreateICmpEQ(CI->getArgOperand(0), CI->getArgOperand(1), 180 "pcmpeq"); 181 // need to sign extend since icmp returns vector of i1 182 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 183 } else if (Name.startswith("llvm.x86.sse2.pcmpgt.") || 184 Name.startswith("llvm.x86.avx2.pcmpgt.")) { 185 Rep = Builder.CreateICmpSGT(CI->getArgOperand(0), CI->getArgOperand(1), 186 "pcmpgt"); 187 // need to sign extend since icmp returns vector of i1 188 Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 189 } else if (Name == "llvm.x86.avx.movnt.dq.256" || 190 Name == "llvm.x86.avx.movnt.ps.256" || 191 Name == "llvm.x86.avx.movnt.pd.256") { 192 IRBuilder<> Builder(C); 193 Builder.SetInsertPoint(CI->getParent(), CI); 194 195 Module *M = F->getParent(); 196 SmallVector<Value *, 1> Elts; 197 Elts.push_back(ConstantInt::get(Type::getInt32Ty(C), 1)); 198 MDNode *Node = MDNode::get(C, Elts); 199 200 Value *Arg0 = CI->getArgOperand(0); 201 Value *Arg1 = CI->getArgOperand(1); 202 203 // Convert the type of the pointer to a pointer to the stored type. 204 Value *BC = Builder.CreateBitCast(Arg0, 205 PointerType::getUnqual(Arg1->getType()), 206 "cast"); 207 StoreInst *SI = Builder.CreateStore(Arg1, BC); 208 SI->setMetadata(M->getMDKindID("nontemporal"), Node); 209 SI->setAlignment(16); 210 211 // Remove intrinsic. 212 CI->eraseFromParent(); 213 return; 214 } else if (Name.startswith("llvm.x86.xop.vpcom")) { 215 Intrinsic::ID intID; 216 if (Name.endswith("ub")) 217 intID = Intrinsic::x86_xop_vpcomub; 218 else if (Name.endswith("uw")) 219 intID = Intrinsic::x86_xop_vpcomuw; 220 else if (Name.endswith("ud")) 221 intID = Intrinsic::x86_xop_vpcomud; 222 else if (Name.endswith("uq")) 223 intID = Intrinsic::x86_xop_vpcomuq; 224 else if (Name.endswith("b")) 225 intID = Intrinsic::x86_xop_vpcomb; 226 else if (Name.endswith("w")) 227 intID = Intrinsic::x86_xop_vpcomw; 228 else if (Name.endswith("d")) 229 intID = Intrinsic::x86_xop_vpcomd; 230 else if (Name.endswith("q")) 231 intID = Intrinsic::x86_xop_vpcomq; 232 else 233 llvm_unreachable("Unknown suffix"); 234 235 Name = Name.substr(18); // strip off "llvm.x86.xop.vpcom" 236 unsigned Imm; 237 if (Name.startswith("lt")) 238 Imm = 0; 239 else if (Name.startswith("le")) 240 Imm = 1; 241 else if (Name.startswith("gt")) 242 Imm = 2; 243 else if (Name.startswith("ge")) 244 Imm = 3; 245 else if (Name.startswith("eq")) 246 Imm = 4; 247 else if (Name.startswith("ne")) 248 Imm = 5; 249 else if (Name.startswith("true")) 250 Imm = 6; 251 else if (Name.startswith("false")) 252 Imm = 7; 253 else 254 llvm_unreachable("Unknown condition"); 255 256 Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID); 257 Rep = Builder.CreateCall3(VPCOM, CI->getArgOperand(0), 258 CI->getArgOperand(1), Builder.getInt8(Imm)); 259 } else { 260 bool PD128 = false, PD256 = false, PS128 = false, PS256 = false; 261 if (Name == "llvm.x86.avx.vpermil.pd.256") 262 PD256 = true; 263 else if (Name == "llvm.x86.avx.vpermil.pd") 264 PD128 = true; 265 else if (Name == "llvm.x86.avx.vpermil.ps.256") 266 PS256 = true; 267 else if (Name == "llvm.x86.avx.vpermil.ps") 268 PS128 = true; 269 270 if (PD256 || PD128 || PS256 || PS128) { 271 Value *Op0 = CI->getArgOperand(0); 272 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 273 SmallVector<Constant*, 8> Idxs; 274 275 if (PD128) 276 for (unsigned i = 0; i != 2; ++i) 277 Idxs.push_back(Builder.getInt32((Imm >> i) & 0x1)); 278 else if (PD256) 279 for (unsigned l = 0; l != 4; l+=2) 280 for (unsigned i = 0; i != 2; ++i) 281 Idxs.push_back(Builder.getInt32(((Imm >> (l+i)) & 0x1) + l)); 282 else if (PS128) 283 for (unsigned i = 0; i != 4; ++i) 284 Idxs.push_back(Builder.getInt32((Imm >> (2 * i)) & 0x3)); 285 else if (PS256) 286 for (unsigned l = 0; l != 8; l+=4) 287 for (unsigned i = 0; i != 4; ++i) 288 Idxs.push_back(Builder.getInt32(((Imm >> (2 * i)) & 0x3) + l)); 289 else 290 llvm_unreachable("Unexpected function"); 291 292 Rep = Builder.CreateShuffleVector(Op0, Op0, ConstantVector::get(Idxs)); 293 } else { 294 llvm_unreachable("Unknown function for CallInst upgrade."); 295 } 296 } 297 298 CI->replaceAllUsesWith(Rep); 299 CI->eraseFromParent(); 300 return; 301 } 302 303 std::string Name = CI->getName().str(); 304 CI->setName(Name + ".old"); 305 306 switch (NewFn->getIntrinsicID()) { 307 default: 308 llvm_unreachable("Unknown function for CallInst upgrade."); 309 310 case Intrinsic::ctlz: 311 case Intrinsic::cttz: 312 assert(CI->getNumArgOperands() == 1 && 313 "Mismatch between function args and call args"); 314 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 315 Builder.getFalse(), Name)); 316 CI->eraseFromParent(); 317 return; 318 319 case Intrinsic::arm_neon_vclz: { 320 // Change name from llvm.arm.neon.vclz.* to llvm.ctlz.* 321 CI->replaceAllUsesWith(Builder.CreateCall2(NewFn, CI->getArgOperand(0), 322 Builder.getFalse(), 323 "llvm.ctlz." + Name.substr(14))); 324 CI->eraseFromParent(); 325 return; 326 } 327 case Intrinsic::ctpop: { 328 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(0))); 329 CI->eraseFromParent(); 330 return; 331 } 332 333 case Intrinsic::x86_xop_vfrcz_ss: 334 case Intrinsic::x86_xop_vfrcz_sd: 335 CI->replaceAllUsesWith(Builder.CreateCall(NewFn, CI->getArgOperand(1), 336 Name)); 337 CI->eraseFromParent(); 338 return; 339 340 case Intrinsic::x86_sse41_ptestc: 341 case Intrinsic::x86_sse41_ptestz: 342 case Intrinsic::x86_sse41_ptestnzc: { 343 // The arguments for these intrinsics used to be v4f32, and changed 344 // to v2i64. This is purely a nop, since those are bitwise intrinsics. 345 // So, the only thing required is a bitcast for both arguments. 346 // First, check the arguments have the old type. 347 Value *Arg0 = CI->getArgOperand(0); 348 if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 349 return; 350 351 // Old intrinsic, add bitcasts 352 Value *Arg1 = CI->getArgOperand(1); 353 354 Value *BC0 = 355 Builder.CreateBitCast(Arg0, 356 VectorType::get(Type::getInt64Ty(C), 2), 357 "cast"); 358 Value *BC1 = 359 Builder.CreateBitCast(Arg1, 360 VectorType::get(Type::getInt64Ty(C), 2), 361 "cast"); 362 363 CallInst* NewCall = Builder.CreateCall2(NewFn, BC0, BC1, Name); 364 CI->replaceAllUsesWith(NewCall); 365 CI->eraseFromParent(); 366 return; 367 } 368 } 369 } 370 371 // This tests each Function to determine if it needs upgrading. When we find 372 // one we are interested in, we then upgrade all calls to reflect the new 373 // function. 374 void llvm::UpgradeCallsToIntrinsic(Function* F) { 375 assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 376 377 // Upgrade the function and check if it is a totaly new function. 378 Function *NewFn; 379 if (UpgradeIntrinsicFunction(F, NewFn)) { 380 if (NewFn != F) { 381 // Replace all uses to the old function with the new one if necessary. 382 for (Value::use_iterator UI = F->use_begin(), UE = F->use_end(); 383 UI != UE; ) { 384 if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 385 UpgradeIntrinsicCall(CI, NewFn); 386 } 387 // Remove old function, no longer used, from the module. 388 F->eraseFromParent(); 389 } 390 } 391 } 392 393
