1 //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This pass munges the code in the input function to better prepare it for 11 // SelectionDAG-based code generation. This works around limitations in it's 12 // basic-block-at-a-time approach. It should eventually be removed. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #include "llvm/ADT/APInt.h" 17 #include "llvm/ADT/ArrayRef.h" 18 #include "llvm/ADT/DenseMap.h" 19 #include "llvm/ADT/PointerIntPair.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/SmallPtrSet.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/ADT/Statistic.h" 24 #include "llvm/Analysis/BlockFrequencyInfo.h" 25 #include "llvm/Analysis/BranchProbabilityInfo.h" 26 #include "llvm/Analysis/ConstantFolding.h" 27 #include "llvm/Analysis/InstructionSimplify.h" 28 #include "llvm/Analysis/LoopInfo.h" 29 #include "llvm/Analysis/MemoryBuiltins.h" 30 #include "llvm/Analysis/ProfileSummaryInfo.h" 31 #include "llvm/Analysis/TargetLibraryInfo.h" 32 #include "llvm/Analysis/TargetTransformInfo.h" 33 #include "llvm/Transforms/Utils/Local.h" 34 #include "llvm/Analysis/ValueTracking.h" 35 #include "llvm/CodeGen/Analysis.h" 36 #include "llvm/CodeGen/ISDOpcodes.h" 37 #include "llvm/CodeGen/SelectionDAGNodes.h" 38 #include "llvm/CodeGen/TargetLowering.h" 39 #include "llvm/CodeGen/TargetPassConfig.h" 40 #include "llvm/CodeGen/TargetSubtargetInfo.h" 41 #include "llvm/CodeGen/ValueTypes.h" 42 #include "llvm/Config/llvm-config.h" 43 #include "llvm/IR/Argument.h" 44 #include "llvm/IR/Attributes.h" 45 #include "llvm/IR/BasicBlock.h" 46 #include "llvm/IR/CallSite.h" 47 #include "llvm/IR/Constant.h" 48 #include "llvm/IR/Constants.h" 49 #include "llvm/IR/DataLayout.h" 50 #include "llvm/IR/DerivedTypes.h" 51 #include "llvm/IR/Dominators.h" 52 #include "llvm/IR/Function.h" 53 #include "llvm/IR/GetElementPtrTypeIterator.h" 54 #include "llvm/IR/GlobalValue.h" 55 #include "llvm/IR/GlobalVariable.h" 56 #include "llvm/IR/IRBuilder.h" 57 #include "llvm/IR/InlineAsm.h" 58 #include "llvm/IR/InstrTypes.h" 59 #include "llvm/IR/Instruction.h" 60 #include "llvm/IR/Instructions.h" 61 #include "llvm/IR/IntrinsicInst.h" 62 #include "llvm/IR/Intrinsics.h" 63 #include "llvm/IR/LLVMContext.h" 64 #include "llvm/IR/MDBuilder.h" 65 #include "llvm/IR/Module.h" 66 #include "llvm/IR/Operator.h" 67 #include "llvm/IR/PatternMatch.h" 68 #include "llvm/IR/Statepoint.h" 69 #include "llvm/IR/Type.h" 70 #include "llvm/IR/Use.h" 71 #include "llvm/IR/User.h" 72 #include "llvm/IR/Value.h" 73 #include "llvm/IR/ValueHandle.h" 74 #include "llvm/IR/ValueMap.h" 75 #include "llvm/Pass.h" 76 #include "llvm/Support/BlockFrequency.h" 77 #include "llvm/Support/BranchProbability.h" 78 #include "llvm/Support/Casting.h" 79 #include "llvm/Support/CommandLine.h" 80 #include "llvm/Support/Compiler.h" 81 #include "llvm/Support/Debug.h" 82 #include "llvm/Support/ErrorHandling.h" 83 #include "llvm/Support/MachineValueType.h" 84 #include "llvm/Support/MathExtras.h" 85 #include "llvm/Support/raw_ostream.h" 86 #include "llvm/Target/TargetMachine.h" 87 #include "llvm/Target/TargetOptions.h" 88 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 89 #include "llvm/Transforms/Utils/BypassSlowDivision.h" 90 #include "llvm/Transforms/Utils/SimplifyLibCalls.h" 91 #include <algorithm> 92 #include <cassert> 93 #include <cstdint> 94 #include <iterator> 95 #include <limits> 96 #include <memory> 97 #include <utility> 98 #include <vector> 99 100 using namespace llvm; 101 using namespace llvm::PatternMatch; 102 103 #define DEBUG_TYPE "codegenprepare" 104 105 STATISTIC(NumBlocksElim, "Number of blocks eliminated"); 106 STATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated"); 107 STATISTIC(NumGEPsElim, "Number of GEPs converted to casts"); 108 STATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of " 109 "sunken Cmps"); 110 STATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses " 111 "of sunken Casts"); 112 STATISTIC(NumMemoryInsts, "Number of memory instructions whose address " 113 "computations were sunk"); 114 STATISTIC(NumMemoryInstsPhiCreated, 115 "Number of phis created when address " 116 "computations were sunk to memory instructions"); 117 STATISTIC(NumMemoryInstsSelectCreated, 118 "Number of select created when address " 119 "computations were sunk to memory instructions"); 120 STATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads"); 121 STATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized"); 122 STATISTIC(NumAndsAdded, 123 "Number of and mask instructions added to form ext loads"); 124 STATISTIC(NumAndUses, "Number of uses of and mask instructions optimized"); 125 STATISTIC(NumRetsDup, "Number of return instructions duplicated"); 126 STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved"); 127 STATISTIC(NumSelectsExpanded, "Number of selects turned into branches"); 128 STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed"); 129 130 static cl::opt<bool> DisableBranchOpts( 131 "disable-cgp-branch-opts", cl::Hidden, cl::init(false), 132 cl::desc("Disable branch optimizations in CodeGenPrepare")); 133 134 static cl::opt<bool> 135 DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false), 136 cl::desc("Disable GC optimizations in CodeGenPrepare")); 137 138 static cl::opt<bool> DisableSelectToBranch( 139 "disable-cgp-select2branch", cl::Hidden, cl::init(false), 140 cl::desc("Disable select to branch conversion.")); 141 142 static cl::opt<bool> AddrSinkUsingGEPs( 143 "addr-sink-using-gep", cl::Hidden, cl::init(true), 144 cl::desc("Address sinking in CGP using GEPs.")); 145 146 static cl::opt<bool> EnableAndCmpSinking( 147 "enable-andcmp-sinking", cl::Hidden, cl::init(true), 148 cl::desc("Enable sinkinig and/cmp into branches.")); 149 150 static cl::opt<bool> DisableStoreExtract( 151 "disable-cgp-store-extract", cl::Hidden, cl::init(false), 152 cl::desc("Disable store(extract) optimizations in CodeGenPrepare")); 153 154 static cl::opt<bool> StressStoreExtract( 155 "stress-cgp-store-extract", cl::Hidden, cl::init(false), 156 cl::desc("Stress test store(extract) optimizations in CodeGenPrepare")); 157 158 static cl::opt<bool> DisableExtLdPromotion( 159 "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false), 160 cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in " 161 "CodeGenPrepare")); 162 163 static cl::opt<bool> StressExtLdPromotion( 164 "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false), 165 cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) " 166 "optimization in CodeGenPrepare")); 167 168 static cl::opt<bool> DisablePreheaderProtect( 169 "disable-preheader-prot", cl::Hidden, cl::init(false), 170 cl::desc("Disable protection against removing loop preheaders")); 171 172 static cl::opt<bool> ProfileGuidedSectionPrefix( 173 "profile-guided-section-prefix", cl::Hidden, cl::init(true), cl::ZeroOrMore, 174 cl::desc("Use profile info to add section prefix for hot/cold functions")); 175 176 static cl::opt<unsigned> FreqRatioToSkipMerge( 177 "cgp-freq-ratio-to-skip-merge", cl::Hidden, cl::init(2), 178 cl::desc("Skip merging empty blocks if (frequency of empty block) / " 179 "(frequency of destination block) is greater than this ratio")); 180 181 static cl::opt<bool> ForceSplitStore( 182 "force-split-store", cl::Hidden, cl::init(false), 183 cl::desc("Force store splitting no matter what the target query says.")); 184 185 static cl::opt<bool> 186 EnableTypePromotionMerge("cgp-type-promotion-merge", cl::Hidden, 187 cl::desc("Enable merging of redundant sexts when one is dominating" 188 " the other."), cl::init(true)); 189 190 static cl::opt<bool> DisableComplexAddrModes( 191 "disable-complex-addr-modes", cl::Hidden, cl::init(false), 192 cl::desc("Disables combining addressing modes with different parts " 193 "in optimizeMemoryInst.")); 194 195 static cl::opt<bool> 196 AddrSinkNewPhis("addr-sink-new-phis", cl::Hidden, cl::init(false), 197 cl::desc("Allow creation of Phis in Address sinking.")); 198 199 static cl::opt<bool> 200 AddrSinkNewSelects("addr-sink-new-select", cl::Hidden, cl::init(true), 201 cl::desc("Allow creation of selects in Address sinking.")); 202 203 static cl::opt<bool> AddrSinkCombineBaseReg( 204 "addr-sink-combine-base-reg", cl::Hidden, cl::init(true), 205 cl::desc("Allow combining of BaseReg field in Address sinking.")); 206 207 static cl::opt<bool> AddrSinkCombineBaseGV( 208 "addr-sink-combine-base-gv", cl::Hidden, cl::init(true), 209 cl::desc("Allow combining of BaseGV field in Address sinking.")); 210 211 static cl::opt<bool> AddrSinkCombineBaseOffs( 212 "addr-sink-combine-base-offs", cl::Hidden, cl::init(true), 213 cl::desc("Allow combining of BaseOffs field in Address sinking.")); 214 215 static cl::opt<bool> AddrSinkCombineScaledReg( 216 "addr-sink-combine-scaled-reg", cl::Hidden, cl::init(true), 217 cl::desc("Allow combining of ScaledReg field in Address sinking.")); 218 219 static cl::opt<bool> 220 EnableGEPOffsetSplit("cgp-split-large-offset-gep", cl::Hidden, 221 cl::init(true), 222 cl::desc("Enable splitting large offset of GEP.")); 223 224 namespace { 225 226 enum ExtType { 227 ZeroExtension, // Zero extension has been seen. 228 SignExtension, // Sign extension has been seen. 229 BothExtension // This extension type is used if we saw sext after 230 // ZeroExtension had been set, or if we saw zext after 231 // SignExtension had been set. It makes the type 232 // information of a promoted instruction invalid. 233 }; 234 235 using SetOfInstrs = SmallPtrSet<Instruction *, 16>; 236 using TypeIsSExt = PointerIntPair<Type *, 2, ExtType>; 237 using InstrToOrigTy = DenseMap<Instruction *, TypeIsSExt>; 238 using SExts = SmallVector<Instruction *, 16>; 239 using ValueToSExts = DenseMap<Value *, SExts>; 240 241 class TypePromotionTransaction; 242 243 class CodeGenPrepare : public FunctionPass { 244 const TargetMachine *TM = nullptr; 245 const TargetSubtargetInfo *SubtargetInfo; 246 const TargetLowering *TLI = nullptr; 247 const TargetRegisterInfo *TRI; 248 const TargetTransformInfo *TTI = nullptr; 249 const TargetLibraryInfo *TLInfo; 250 const LoopInfo *LI; 251 std::unique_ptr<BlockFrequencyInfo> BFI; 252 std::unique_ptr<BranchProbabilityInfo> BPI; 253 254 /// As we scan instructions optimizing them, this is the next instruction 255 /// to optimize. Transforms that can invalidate this should update it. 256 BasicBlock::iterator CurInstIterator; 257 258 /// Keeps track of non-local addresses that have been sunk into a block. 259 /// This allows us to avoid inserting duplicate code for blocks with 260 /// multiple load/stores of the same address. The usage of WeakTrackingVH 261 /// enables SunkAddrs to be treated as a cache whose entries can be 262 /// invalidated if a sunken address computation has been erased. 263 ValueMap<Value*, WeakTrackingVH> SunkAddrs; 264 265 /// Keeps track of all instructions inserted for the current function. 266 SetOfInstrs InsertedInsts; 267 268 /// Keeps track of the type of the related instruction before their 269 /// promotion for the current function. 270 InstrToOrigTy PromotedInsts; 271 272 /// Keep track of instructions removed during promotion. 273 SetOfInstrs RemovedInsts; 274 275 /// Keep track of sext chains based on their initial value. 276 DenseMap<Value *, Instruction *> SeenChainsForSExt; 277 278 /// Keep track of GEPs accessing the same data structures such as structs or 279 /// arrays that are candidates to be split later because of their large 280 /// size. 281 DenseMap< 282 AssertingVH<Value>, 283 SmallVector<std::pair<AssertingVH<GetElementPtrInst>, int64_t>, 32>> 284 LargeOffsetGEPMap; 285 286 /// Keep track of new GEP base after splitting the GEPs having large offset. 287 SmallSet<AssertingVH<Value>, 2> NewGEPBases; 288 289 /// Map serial numbers to Large offset GEPs. 290 DenseMap<AssertingVH<GetElementPtrInst>, int> LargeOffsetGEPID; 291 292 /// Keep track of SExt promoted. 293 ValueToSExts ValToSExtendedUses; 294 295 /// True if CFG is modified in any way. 296 bool ModifiedDT; 297 298 /// True if optimizing for size. 299 bool OptSize; 300 301 /// DataLayout for the Function being processed. 302 const DataLayout *DL = nullptr; 303 304 public: 305 static char ID; // Pass identification, replacement for typeid 306 307 CodeGenPrepare() : FunctionPass(ID) { 308 initializeCodeGenPreparePass(*PassRegistry::getPassRegistry()); 309 } 310 311 bool runOnFunction(Function &F) override; 312 313 StringRef getPassName() const override { return "CodeGen Prepare"; } 314 315 void getAnalysisUsage(AnalysisUsage &AU) const override { 316 // FIXME: When we can selectively preserve passes, preserve the domtree. 317 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 318 AU.addRequired<TargetLibraryInfoWrapperPass>(); 319 AU.addRequired<TargetTransformInfoWrapperPass>(); 320 AU.addRequired<LoopInfoWrapperPass>(); 321 } 322 323 private: 324 bool eliminateFallThrough(Function &F); 325 bool eliminateMostlyEmptyBlocks(Function &F); 326 BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB); 327 bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const; 328 void eliminateMostlyEmptyBlock(BasicBlock *BB); 329 bool isMergingEmptyBlockProfitable(BasicBlock *BB, BasicBlock *DestBB, 330 bool isPreheader); 331 bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT); 332 bool optimizeInst(Instruction *I, bool &ModifiedDT); 333 bool optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, 334 Type *AccessTy, unsigned AddrSpace); 335 bool optimizeInlineAsmInst(CallInst *CS); 336 bool optimizeCallInst(CallInst *CI, bool &ModifiedDT); 337 bool optimizeExt(Instruction *&I); 338 bool optimizeExtUses(Instruction *I); 339 bool optimizeLoadExt(LoadInst *Load); 340 bool optimizeSelectInst(SelectInst *SI); 341 bool optimizeShuffleVectorInst(ShuffleVectorInst *SVI); 342 bool optimizeSwitchInst(SwitchInst *SI); 343 bool optimizeExtractElementInst(Instruction *Inst); 344 bool dupRetToEnableTailCallOpts(BasicBlock *BB); 345 bool placeDbgValues(Function &F); 346 bool canFormExtLd(const SmallVectorImpl<Instruction *> &MovedExts, 347 LoadInst *&LI, Instruction *&Inst, bool HasPromoted); 348 bool tryToPromoteExts(TypePromotionTransaction &TPT, 349 const SmallVectorImpl<Instruction *> &Exts, 350 SmallVectorImpl<Instruction *> &ProfitablyMovedExts, 351 unsigned CreatedInstsCost = 0); 352 bool mergeSExts(Function &F); 353 bool splitLargeGEPOffsets(); 354 bool performAddressTypePromotion( 355 Instruction *&Inst, 356 bool AllowPromotionWithoutCommonHeader, 357 bool HasPromoted, TypePromotionTransaction &TPT, 358 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts); 359 bool splitBranchCondition(Function &F); 360 bool simplifyOffsetableRelocate(Instruction &I); 361 }; 362 363 } // end anonymous namespace 364 365 char CodeGenPrepare::ID = 0; 366 367 INITIALIZE_PASS_BEGIN(CodeGenPrepare, DEBUG_TYPE, 368 "Optimize for code generation", false, false) 369 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 370 INITIALIZE_PASS_END(CodeGenPrepare, DEBUG_TYPE, 371 "Optimize for code generation", false, false) 372 373 FunctionPass *llvm::createCodeGenPreparePass() { return new CodeGenPrepare(); } 374 375 bool CodeGenPrepare::runOnFunction(Function &F) { 376 if (skipFunction(F)) 377 return false; 378 379 DL = &F.getParent()->getDataLayout(); 380 381 bool EverMadeChange = false; 382 // Clear per function information. 383 InsertedInsts.clear(); 384 PromotedInsts.clear(); 385 386 ModifiedDT = false; 387 if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>()) { 388 TM = &TPC->getTM<TargetMachine>(); 389 SubtargetInfo = TM->getSubtargetImpl(F); 390 TLI = SubtargetInfo->getTargetLowering(); 391 TRI = SubtargetInfo->getRegisterInfo(); 392 } 393 TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); 394 TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); 395 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); 396 BPI.reset(new BranchProbabilityInfo(F, *LI)); 397 BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); 398 OptSize = F.optForSize(); 399 400 ProfileSummaryInfo *PSI = 401 getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 402 if (ProfileGuidedSectionPrefix) { 403 if (PSI->isFunctionHotInCallGraph(&F, *BFI)) 404 F.setSectionPrefix(".hot"); 405 else if (PSI->isFunctionColdInCallGraph(&F, *BFI)) 406 F.setSectionPrefix(".unlikely"); 407 } 408 409 /// This optimization identifies DIV instructions that can be 410 /// profitably bypassed and carried out with a shorter, faster divide. 411 if (!OptSize && !PSI->hasHugeWorkingSetSize() && TLI && 412 TLI->isSlowDivBypassed()) { 413 const DenseMap<unsigned int, unsigned int> &BypassWidths = 414 TLI->getBypassSlowDivWidths(); 415 BasicBlock* BB = &*F.begin(); 416 while (BB != nullptr) { 417 // bypassSlowDivision may create new BBs, but we don't want to reapply the 418 // optimization to those blocks. 419 BasicBlock* Next = BB->getNextNode(); 420 EverMadeChange |= bypassSlowDivision(BB, BypassWidths); 421 BB = Next; 422 } 423 } 424 425 // Eliminate blocks that contain only PHI nodes and an 426 // unconditional branch. 427 EverMadeChange |= eliminateMostlyEmptyBlocks(F); 428 429 // llvm.dbg.value is far away from the value then iSel may not be able 430 // handle it properly. iSel will drop llvm.dbg.value if it can not 431 // find a node corresponding to the value. 432 EverMadeChange |= placeDbgValues(F); 433 434 if (!DisableBranchOpts) 435 EverMadeChange |= splitBranchCondition(F); 436 437 // Split some critical edges where one of the sources is an indirect branch, 438 // to help generate sane code for PHIs involving such edges. 439 EverMadeChange |= SplitIndirectBrCriticalEdges(F); 440 441 bool MadeChange = true; 442 while (MadeChange) { 443 MadeChange = false; 444 SeenChainsForSExt.clear(); 445 ValToSExtendedUses.clear(); 446 RemovedInsts.clear(); 447 LargeOffsetGEPMap.clear(); 448 LargeOffsetGEPID.clear(); 449 for (Function::iterator I = F.begin(); I != F.end(); ) { 450 BasicBlock *BB = &*I++; 451 bool ModifiedDTOnIteration = false; 452 MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration); 453 454 // Restart BB iteration if the dominator tree of the Function was changed 455 if (ModifiedDTOnIteration) 456 break; 457 } 458 if (EnableTypePromotionMerge && !ValToSExtendedUses.empty()) 459 MadeChange |= mergeSExts(F); 460 if (!LargeOffsetGEPMap.empty()) 461 MadeChange |= splitLargeGEPOffsets(); 462 463 // Really free removed instructions during promotion. 464 for (Instruction *I : RemovedInsts) 465 I->deleteValue(); 466 467 EverMadeChange |= MadeChange; 468 } 469 470 SunkAddrs.clear(); 471 472 if (!DisableBranchOpts) { 473 MadeChange = false; 474 // Use a set vector to get deterministic iteration order. The order the 475 // blocks are removed may affect whether or not PHI nodes in successors 476 // are removed. 477 SmallSetVector<BasicBlock*, 8> WorkList; 478 for (BasicBlock &BB : F) { 479 SmallVector<BasicBlock *, 2> Successors(succ_begin(&BB), succ_end(&BB)); 480 MadeChange |= ConstantFoldTerminator(&BB, true); 481 if (!MadeChange) continue; 482 483 for (SmallVectorImpl<BasicBlock*>::iterator 484 II = Successors.begin(), IE = Successors.end(); II != IE; ++II) 485 if (pred_begin(*II) == pred_end(*II)) 486 WorkList.insert(*II); 487 } 488 489 // Delete the dead blocks and any of their dead successors. 490 MadeChange |= !WorkList.empty(); 491 while (!WorkList.empty()) { 492 BasicBlock *BB = WorkList.pop_back_val(); 493 SmallVector<BasicBlock*, 2> Successors(succ_begin(BB), succ_end(BB)); 494 495 DeleteDeadBlock(BB); 496 497 for (SmallVectorImpl<BasicBlock*>::iterator 498 II = Successors.begin(), IE = Successors.end(); II != IE; ++II) 499 if (pred_begin(*II) == pred_end(*II)) 500 WorkList.insert(*II); 501 } 502 503 // Merge pairs of basic blocks with unconditional branches, connected by 504 // a single edge. 505 if (EverMadeChange || MadeChange) 506 MadeChange |= eliminateFallThrough(F); 507 508 EverMadeChange |= MadeChange; 509 } 510 511 if (!DisableGCOpts) { 512 SmallVector<Instruction *, 2> Statepoints; 513 for (BasicBlock &BB : F) 514 for (Instruction &I : BB) 515 if (isStatepoint(I)) 516 Statepoints.push_back(&I); 517 for (auto &I : Statepoints) 518 EverMadeChange |= simplifyOffsetableRelocate(*I); 519 } 520 521 return EverMadeChange; 522 } 523 524 /// Merge basic blocks which are connected by a single edge, where one of the 525 /// basic blocks has a single successor pointing to the other basic block, 526 /// which has a single predecessor. 527 bool CodeGenPrepare::eliminateFallThrough(Function &F) { 528 bool Changed = false; 529 // Scan all of the blocks in the function, except for the entry block. 530 // Use a temporary array to avoid iterator being invalidated when 531 // deleting blocks. 532 SmallVector<WeakTrackingVH, 16> Blocks; 533 for (auto &Block : llvm::make_range(std::next(F.begin()), F.end())) 534 Blocks.push_back(&Block); 535 536 for (auto &Block : Blocks) { 537 auto *BB = cast_or_null<BasicBlock>(Block); 538 if (!BB) 539 continue; 540 // If the destination block has a single pred, then this is a trivial 541 // edge, just collapse it. 542 BasicBlock *SinglePred = BB->getSinglePredecessor(); 543 544 // Don't merge if BB's address is taken. 545 if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue; 546 547 BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator()); 548 if (Term && !Term->isConditional()) { 549 Changed = true; 550 LLVM_DEBUG(dbgs() << "To merge:\n" << *BB << "\n\n\n"); 551 552 // Merge BB into SinglePred and delete it. 553 MergeBlockIntoPredecessor(BB); 554 } 555 } 556 return Changed; 557 } 558 559 /// Find a destination block from BB if BB is mergeable empty block. 560 BasicBlock *CodeGenPrepare::findDestBlockOfMergeableEmptyBlock(BasicBlock *BB) { 561 // If this block doesn't end with an uncond branch, ignore it. 562 BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); 563 if (!BI || !BI->isUnconditional()) 564 return nullptr; 565 566 // If the instruction before the branch (skipping debug info) isn't a phi 567 // node, then other stuff is happening here. 568 BasicBlock::iterator BBI = BI->getIterator(); 569 if (BBI != BB->begin()) { 570 --BBI; 571 while (isa<DbgInfoIntrinsic>(BBI)) { 572 if (BBI == BB->begin()) 573 break; 574 --BBI; 575 } 576 if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI)) 577 return nullptr; 578 } 579 580 // Do not break infinite loops. 581 BasicBlock *DestBB = BI->getSuccessor(0); 582 if (DestBB == BB) 583 return nullptr; 584 585 if (!canMergeBlocks(BB, DestBB)) 586 DestBB = nullptr; 587 588 return DestBB; 589 } 590 591 /// Eliminate blocks that contain only PHI nodes, debug info directives, and an 592 /// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split 593 /// edges in ways that are non-optimal for isel. Start by eliminating these 594 /// blocks so we can split them the way we want them. 595 bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F) { 596 SmallPtrSet<BasicBlock *, 16> Preheaders; 597 SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end()); 598 while (!LoopList.empty()) { 599 Loop *L = LoopList.pop_back_val(); 600 LoopList.insert(LoopList.end(), L->begin(), L->end()); 601 if (BasicBlock *Preheader = L->getLoopPreheader()) 602 Preheaders.insert(Preheader); 603 } 604 605 bool MadeChange = false; 606 // Copy blocks into a temporary array to avoid iterator invalidation issues 607 // as we remove them. 608 // Note that this intentionally skips the entry block. 609 SmallVector<WeakTrackingVH, 16> Blocks; 610 for (auto &Block : llvm::make_range(std::next(F.begin()), F.end())) 611 Blocks.push_back(&Block); 612 613 for (auto &Block : Blocks) { 614 BasicBlock *BB = cast_or_null<BasicBlock>(Block); 615 if (!BB) 616 continue; 617 BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(BB); 618 if (!DestBB || 619 !isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.count(BB))) 620 continue; 621 622 eliminateMostlyEmptyBlock(BB); 623 MadeChange = true; 624 } 625 return MadeChange; 626 } 627 628 bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB, 629 BasicBlock *DestBB, 630 bool isPreheader) { 631 // Do not delete loop preheaders if doing so would create a critical edge. 632 // Loop preheaders can be good locations to spill registers. If the 633 // preheader is deleted and we create a critical edge, registers may be 634 // spilled in the loop body instead. 635 if (!DisablePreheaderProtect && isPreheader && 636 !(BB->getSinglePredecessor() && 637 BB->getSinglePredecessor()->getSingleSuccessor())) 638 return false; 639 640 // Try to skip merging if the unique predecessor of BB is terminated by a 641 // switch or indirect branch instruction, and BB is used as an incoming block 642 // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to 643 // add COPY instructions in the predecessor of BB instead of BB (if it is not 644 // merged). Note that the critical edge created by merging such blocks wont be 645 // split in MachineSink because the jump table is not analyzable. By keeping 646 // such empty block (BB), ISel will place COPY instructions in BB, not in the 647 // predecessor of BB. 648 BasicBlock *Pred = BB->getUniquePredecessor(); 649 if (!Pred || 650 !(isa<SwitchInst>(Pred->getTerminator()) || 651 isa<IndirectBrInst>(Pred->getTerminator()))) 652 return true; 653 654 if (BB->getTerminator() != BB->getFirstNonPHI()) 655 return true; 656 657 // We use a simple cost heuristic which determine skipping merging is 658 // profitable if the cost of skipping merging is less than the cost of 659 // merging : Cost(skipping merging) < Cost(merging BB), where the 660 // Cost(skipping merging) is Freq(BB) * (Cost(Copy) + Cost(Branch)), and 661 // the Cost(merging BB) is Freq(Pred) * Cost(Copy). 662 // Assuming Cost(Copy) == Cost(Branch), we could simplify it to : 663 // Freq(Pred) / Freq(BB) > 2. 664 // Note that if there are multiple empty blocks sharing the same incoming 665 // value for the PHIs in the DestBB, we consider them together. In such 666 // case, Cost(merging BB) will be the sum of their frequencies. 667 668 if (!isa<PHINode>(DestBB->begin())) 669 return true; 670 671 SmallPtrSet<BasicBlock *, 16> SameIncomingValueBBs; 672 673 // Find all other incoming blocks from which incoming values of all PHIs in 674 // DestBB are the same as the ones from BB. 675 for (pred_iterator PI = pred_begin(DestBB), E = pred_end(DestBB); PI != E; 676 ++PI) { 677 BasicBlock *DestBBPred = *PI; 678 if (DestBBPred == BB) 679 continue; 680 681 if (llvm::all_of(DestBB->phis(), [&](const PHINode &DestPN) { 682 return DestPN.getIncomingValueForBlock(BB) == 683 DestPN.getIncomingValueForBlock(DestBBPred); 684 })) 685 SameIncomingValueBBs.insert(DestBBPred); 686 } 687 688 // See if all BB's incoming values are same as the value from Pred. In this 689 // case, no reason to skip merging because COPYs are expected to be place in 690 // Pred already. 691 if (SameIncomingValueBBs.count(Pred)) 692 return true; 693 694 BlockFrequency PredFreq = BFI->getBlockFreq(Pred); 695 BlockFrequency BBFreq = BFI->getBlockFreq(BB); 696 697 for (auto SameValueBB : SameIncomingValueBBs) 698 if (SameValueBB->getUniquePredecessor() == Pred && 699 DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB)) 700 BBFreq += BFI->getBlockFreq(SameValueBB); 701 702 return PredFreq.getFrequency() <= 703 BBFreq.getFrequency() * FreqRatioToSkipMerge; 704 } 705 706 /// Return true if we can merge BB into DestBB if there is a single 707 /// unconditional branch between them, and BB contains no other non-phi 708 /// instructions. 709 bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB, 710 const BasicBlock *DestBB) const { 711 // We only want to eliminate blocks whose phi nodes are used by phi nodes in 712 // the successor. If there are more complex condition (e.g. preheaders), 713 // don't mess around with them. 714 for (const PHINode &PN : BB->phis()) { 715 for (const User *U : PN.users()) { 716 const Instruction *UI = cast<Instruction>(U); 717 if (UI->getParent() != DestBB || !isa<PHINode>(UI)) 718 return false; 719 // If User is inside DestBB block and it is a PHINode then check 720 // incoming value. If incoming value is not from BB then this is 721 // a complex condition (e.g. preheaders) we want to avoid here. 722 if (UI->getParent() == DestBB) { 723 if (const PHINode *UPN = dyn_cast<PHINode>(UI)) 724 for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) { 725 Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I)); 726 if (Insn && Insn->getParent() == BB && 727 Insn->getParent() != UPN->getIncomingBlock(I)) 728 return false; 729 } 730 } 731 } 732 } 733 734 // If BB and DestBB contain any common predecessors, then the phi nodes in BB 735 // and DestBB may have conflicting incoming values for the block. If so, we 736 // can't merge the block. 737 const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin()); 738 if (!DestBBPN) return true; // no conflict. 739 740 // Collect the preds of BB. 741 SmallPtrSet<const BasicBlock*, 16> BBPreds; 742 if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { 743 // It is faster to get preds from a PHI than with pred_iterator. 744 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) 745 BBPreds.insert(BBPN->getIncomingBlock(i)); 746 } else { 747 BBPreds.insert(pred_begin(BB), pred_end(BB)); 748 } 749 750 // Walk the preds of DestBB. 751 for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) { 752 BasicBlock *Pred = DestBBPN->getIncomingBlock(i); 753 if (BBPreds.count(Pred)) { // Common predecessor? 754 for (const PHINode &PN : DestBB->phis()) { 755 const Value *V1 = PN.getIncomingValueForBlock(Pred); 756 const Value *V2 = PN.getIncomingValueForBlock(BB); 757 758 // If V2 is a phi node in BB, look up what the mapped value will be. 759 if (const PHINode *V2PN = dyn_cast<PHINode>(V2)) 760 if (V2PN->getParent() == BB) 761 V2 = V2PN->getIncomingValueForBlock(Pred); 762 763 // If there is a conflict, bail out. 764 if (V1 != V2) return false; 765 } 766 } 767 } 768 769 return true; 770 } 771 772 /// Eliminate a basic block that has only phi's and an unconditional branch in 773 /// it. 774 void CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) { 775 BranchInst *BI = cast<BranchInst>(BB->getTerminator()); 776 BasicBlock *DestBB = BI->getSuccessor(0); 777 778 LLVM_DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n" 779 << *BB << *DestBB); 780 781 // If the destination block has a single pred, then this is a trivial edge, 782 // just collapse it. 783 if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) { 784 if (SinglePred != DestBB) { 785 assert(SinglePred == BB && 786 "Single predecessor not the same as predecessor"); 787 // Merge DestBB into SinglePred/BB and delete it. 788 MergeBlockIntoPredecessor(DestBB); 789 // Note: BB(=SinglePred) will not be deleted on this path. 790 // DestBB(=its single successor) is the one that was deleted. 791 LLVM_DEBUG(dbgs() << "AFTER:\n" << *SinglePred << "\n\n\n"); 792 return; 793 } 794 } 795 796 // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB 797 // to handle the new incoming edges it is about to have. 798 for (PHINode &PN : DestBB->phis()) { 799 // Remove the incoming value for BB, and remember it. 800 Value *InVal = PN.removeIncomingValue(BB, false); 801 802 // Two options: either the InVal is a phi node defined in BB or it is some 803 // value that dominates BB. 804 PHINode *InValPhi = dyn_cast<PHINode>(InVal); 805 if (InValPhi && InValPhi->getParent() == BB) { 806 // Add all of the input values of the input PHI as inputs of this phi. 807 for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i) 808 PN.addIncoming(InValPhi->getIncomingValue(i), 809 InValPhi->getIncomingBlock(i)); 810 } else { 811 // Otherwise, add one instance of the dominating value for each edge that 812 // we will be adding. 813 if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) { 814 for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i) 815 PN.addIncoming(InVal, BBPN->getIncomingBlock(i)); 816 } else { 817 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) 818 PN.addIncoming(InVal, *PI); 819 } 820 } 821 } 822 823 // The PHIs are now updated, change everything that refers to BB to use 824 // DestBB and remove BB. 825 BB->replaceAllUsesWith(DestBB); 826 BB->eraseFromParent(); 827 ++NumBlocksElim; 828 829 LLVM_DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n"); 830 } 831 832 // Computes a map of base pointer relocation instructions to corresponding 833 // derived pointer relocation instructions given a vector of all relocate calls 834 static void computeBaseDerivedRelocateMap( 835 const SmallVectorImpl<GCRelocateInst *> &AllRelocateCalls, 836 DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>> 837 &RelocateInstMap) { 838 // Collect information in two maps: one primarily for locating the base object 839 // while filling the second map; the second map is the final structure holding 840 // a mapping between Base and corresponding Derived relocate calls 841 DenseMap<std::pair<unsigned, unsigned>, GCRelocateInst *> RelocateIdxMap; 842 for (auto *ThisRelocate : AllRelocateCalls) { 843 auto K = std::make_pair(ThisRelocate->getBasePtrIndex(), 844 ThisRelocate->getDerivedPtrIndex()); 845 RelocateIdxMap.insert(std::make_pair(K, ThisRelocate)); 846 } 847 for (auto &Item : RelocateIdxMap) { 848 std::pair<unsigned, unsigned> Key = Item.first; 849 if (Key.first == Key.second) 850 // Base relocation: nothing to insert 851 continue; 852 853 GCRelocateInst *I = Item.second; 854 auto BaseKey = std::make_pair(Key.first, Key.first); 855 856 // We're iterating over RelocateIdxMap so we cannot modify it. 857 auto MaybeBase = RelocateIdxMap.find(BaseKey); 858 if (MaybeBase == RelocateIdxMap.end()) 859 // TODO: We might want to insert a new base object relocate and gep off 860 // that, if there are enough derived object relocates. 861 continue; 862 863 RelocateInstMap[MaybeBase->second].push_back(I); 864 } 865 } 866 867 // Accepts a GEP and extracts the operands into a vector provided they're all 868 // small integer constants 869 static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP, 870 SmallVectorImpl<Value *> &OffsetV) { 871 for (unsigned i = 1; i < GEP->getNumOperands(); i++) { 872 // Only accept small constant integer operands 873 auto Op = dyn_cast<ConstantInt>(GEP->getOperand(i)); 874 if (!Op || Op->getZExtValue() > 20) 875 return false; 876 } 877 878 for (unsigned i = 1; i < GEP->getNumOperands(); i++) 879 OffsetV.push_back(GEP->getOperand(i)); 880 return true; 881 } 882 883 // Takes a RelocatedBase (base pointer relocation instruction) and Targets to 884 // replace, computes a replacement, and affects it. 885 static bool 886 simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase, 887 const SmallVectorImpl<GCRelocateInst *> &Targets) { 888 bool MadeChange = false; 889 // We must ensure the relocation of derived pointer is defined after 890 // relocation of base pointer. If we find a relocation corresponding to base 891 // defined earlier than relocation of base then we move relocation of base 892 // right before found relocation. We consider only relocation in the same 893 // basic block as relocation of base. Relocations from other basic block will 894 // be skipped by optimization and we do not care about them. 895 for (auto R = RelocatedBase->getParent()->getFirstInsertionPt(); 896 &*R != RelocatedBase; ++R) 897 if (auto RI = dyn_cast<GCRelocateInst>(R)) 898 if (RI->getStatepoint() == RelocatedBase->getStatepoint()) 899 if (RI->getBasePtrIndex() == RelocatedBase->getBasePtrIndex()) { 900 RelocatedBase->moveBefore(RI); 901 break; 902 } 903 904 for (GCRelocateInst *ToReplace : Targets) { 905 assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() && 906 "Not relocating a derived object of the original base object"); 907 if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) { 908 // A duplicate relocate call. TODO: coalesce duplicates. 909 continue; 910 } 911 912 if (RelocatedBase->getParent() != ToReplace->getParent()) { 913 // Base and derived relocates are in different basic blocks. 914 // In this case transform is only valid when base dominates derived 915 // relocate. However it would be too expensive to check dominance 916 // for each such relocate, so we skip the whole transformation. 917 continue; 918 } 919 920 Value *Base = ToReplace->getBasePtr(); 921 auto Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr()); 922 if (!Derived || Derived->getPointerOperand() != Base) 923 continue; 924 925 SmallVector<Value *, 2> OffsetV; 926 if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV)) 927 continue; 928 929 // Create a Builder and replace the target callsite with a gep 930 assert(RelocatedBase->getNextNode() && 931 "Should always have one since it's not a terminator"); 932 933 // Insert after RelocatedBase 934 IRBuilder<> Builder(RelocatedBase->getNextNode()); 935 Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); 936 937 // If gc_relocate does not match the actual type, cast it to the right type. 938 // In theory, there must be a bitcast after gc_relocate if the type does not 939 // match, and we should reuse it to get the derived pointer. But it could be 940 // cases like this: 941 // bb1: 942 // ... 943 // %g1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...) 944 // br label %merge 945 // 946 // bb2: 947 // ... 948 // %g2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...) 949 // br label %merge 950 // 951 // merge: 952 // %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ] 953 // %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)* 954 // 955 // In this case, we can not find the bitcast any more. So we insert a new bitcast 956 // no matter there is already one or not. In this way, we can handle all cases, and 957 // the extra bitcast should be optimized away in later passes. 958 Value *ActualRelocatedBase = RelocatedBase; 959 if (RelocatedBase->getType() != Base->getType()) { 960 ActualRelocatedBase = 961 Builder.CreateBitCast(RelocatedBase, Base->getType()); 962 } 963 Value *Replacement = Builder.CreateGEP( 964 Derived->getSourceElementType(), ActualRelocatedBase, makeArrayRef(OffsetV)); 965 Replacement->takeName(ToReplace); 966 // If the newly generated derived pointer's type does not match the original derived 967 // pointer's type, cast the new derived pointer to match it. Same reasoning as above. 968 Value *ActualReplacement = Replacement; 969 if (Replacement->getType() != ToReplace->getType()) { 970 ActualReplacement = 971 Builder.CreateBitCast(Replacement, ToReplace->getType()); 972 } 973 ToReplace->replaceAllUsesWith(ActualReplacement); 974 ToReplace->eraseFromParent(); 975 976 MadeChange = true; 977 } 978 return MadeChange; 979 } 980 981 // Turns this: 982 // 983 // %base = ... 984 // %ptr = gep %base + 15 985 // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr) 986 // %base' = relocate(%tok, i32 4, i32 4) 987 // %ptr' = relocate(%tok, i32 4, i32 5) 988 // %val = load %ptr' 989 // 990 // into this: 991 // 992 // %base = ... 993 // %ptr = gep %base + 15 994 // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr) 995 // %base' = gc.relocate(%tok, i32 4, i32 4) 996 // %ptr' = gep %base' + 15 997 // %val = load %ptr' 998 bool CodeGenPrepare::simplifyOffsetableRelocate(Instruction &I) { 999 bool MadeChange = false; 1000 SmallVector<GCRelocateInst *, 2> AllRelocateCalls; 1001 1002 for (auto *U : I.users()) 1003 if (GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U)) 1004 // Collect all the relocate calls associated with a statepoint 1005 AllRelocateCalls.push_back(Relocate); 1006 1007 // We need atleast one base pointer relocation + one derived pointer 1008 // relocation to mangle 1009 if (AllRelocateCalls.size() < 2) 1010 return false; 1011 1012 // RelocateInstMap is a mapping from the base relocate instruction to the 1013 // corresponding derived relocate instructions 1014 DenseMap<GCRelocateInst *, SmallVector<GCRelocateInst *, 2>> RelocateInstMap; 1015 computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap); 1016 if (RelocateInstMap.empty()) 1017 return false; 1018 1019 for (auto &Item : RelocateInstMap) 1020 // Item.first is the RelocatedBase to offset against 1021 // Item.second is the vector of Targets to replace 1022 MadeChange = simplifyRelocatesOffABase(Item.first, Item.second); 1023 return MadeChange; 1024 } 1025 1026 /// SinkCast - Sink the specified cast instruction into its user blocks 1027 static bool SinkCast(CastInst *CI) { 1028 BasicBlock *DefBB = CI->getParent(); 1029 1030 /// InsertedCasts - Only insert a cast in each block once. 1031 DenseMap<BasicBlock*, CastInst*> InsertedCasts; 1032 1033 bool MadeChange = false; 1034 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); 1035 UI != E; ) { 1036 Use &TheUse = UI.getUse(); 1037 Instruction *User = cast<Instruction>(*UI); 1038 1039 // Figure out which BB this cast is used in. For PHI's this is the 1040 // appropriate predecessor block. 1041 BasicBlock *UserBB = User->getParent(); 1042 if (PHINode *PN = dyn_cast<PHINode>(User)) { 1043 UserBB = PN->getIncomingBlock(TheUse); 1044 } 1045 1046 // Preincrement use iterator so we don't invalidate it. 1047 ++UI; 1048 1049 // The first insertion point of a block containing an EH pad is after the 1050 // pad. If the pad is the user, we cannot sink the cast past the pad. 1051 if (User->isEHPad()) 1052 continue; 1053 1054 // If the block selected to receive the cast is an EH pad that does not 1055 // allow non-PHI instructions before the terminator, we can't sink the 1056 // cast. 1057 if (UserBB->getTerminator()->isEHPad()) 1058 continue; 1059 1060 // If this user is in the same block as the cast, don't change the cast. 1061 if (UserBB == DefBB) continue; 1062 1063 // If we have already inserted a cast into this block, use it. 1064 CastInst *&InsertedCast = InsertedCasts[UserBB]; 1065 1066 if (!InsertedCast) { 1067 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 1068 assert(InsertPt != UserBB->end()); 1069 InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0), 1070 CI->getType(), "", &*InsertPt); 1071 InsertedCast->setDebugLoc(CI->getDebugLoc()); 1072 } 1073 1074 // Replace a use of the cast with a use of the new cast. 1075 TheUse = InsertedCast; 1076 MadeChange = true; 1077 ++NumCastUses; 1078 } 1079 1080 // If we removed all uses, nuke the cast. 1081 if (CI->use_empty()) { 1082 salvageDebugInfo(*CI); 1083 CI->eraseFromParent(); 1084 MadeChange = true; 1085 } 1086 1087 return MadeChange; 1088 } 1089 1090 /// If the specified cast instruction is a noop copy (e.g. it's casting from 1091 /// one pointer type to another, i32->i8 on PPC), sink it into user blocks to 1092 /// reduce the number of virtual registers that must be created and coalesced. 1093 /// 1094 /// Return true if any changes are made. 1095 static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI, 1096 const DataLayout &DL) { 1097 // Sink only "cheap" (or nop) address-space casts. This is a weaker condition 1098 // than sinking only nop casts, but is helpful on some platforms. 1099 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(CI)) { 1100 if (!TLI.isCheapAddrSpaceCast(ASC->getSrcAddressSpace(), 1101 ASC->getDestAddressSpace())) 1102 return false; 1103 } 1104 1105 // If this is a noop copy, 1106 EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType()); 1107 EVT DstVT = TLI.getValueType(DL, CI->getType()); 1108 1109 // This is an fp<->int conversion? 1110 if (SrcVT.isInteger() != DstVT.isInteger()) 1111 return false; 1112 1113 // If this is an extension, it will be a zero or sign extension, which 1114 // isn't a noop. 1115 if (SrcVT.bitsLT(DstVT)) return false; 1116 1117 // If these values will be promoted, find out what they will be promoted 1118 // to. This helps us consider truncates on PPC as noop copies when they 1119 // are. 1120 if (TLI.getTypeAction(CI->getContext(), SrcVT) == 1121 TargetLowering::TypePromoteInteger) 1122 SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT); 1123 if (TLI.getTypeAction(CI->getContext(), DstVT) == 1124 TargetLowering::TypePromoteInteger) 1125 DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT); 1126 1127 // If, after promotion, these are the same types, this is a noop copy. 1128 if (SrcVT != DstVT) 1129 return false; 1130 1131 return SinkCast(CI); 1132 } 1133 1134 /// Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if 1135 /// possible. 1136 /// 1137 /// Return true if any changes were made. 1138 static bool CombineUAddWithOverflow(CmpInst *CI) { 1139 Value *A, *B; 1140 Instruction *AddI; 1141 if (!match(CI, 1142 m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI)))) 1143 return false; 1144 1145 Type *Ty = AddI->getType(); 1146 if (!isa<IntegerType>(Ty)) 1147 return false; 1148 1149 // We don't want to move around uses of condition values this late, so we we 1150 // check if it is legal to create the call to the intrinsic in the basic 1151 // block containing the icmp: 1152 1153 if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse()) 1154 return false; 1155 1156 #ifndef NDEBUG 1157 // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption 1158 // for now: 1159 if (AddI->hasOneUse()) 1160 assert(*AddI->user_begin() == CI && "expected!"); 1161 #endif 1162 1163 Module *M = CI->getModule(); 1164 Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty); 1165 1166 auto *InsertPt = AddI->hasOneUse() ? CI : AddI; 1167 1168 auto *UAddWithOverflow = 1169 CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt); 1170 auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt); 1171 auto *Overflow = 1172 ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt); 1173 1174 CI->replaceAllUsesWith(Overflow); 1175 AddI->replaceAllUsesWith(UAdd); 1176 CI->eraseFromParent(); 1177 AddI->eraseFromParent(); 1178 return true; 1179 } 1180 1181 /// Sink the given CmpInst into user blocks to reduce the number of virtual 1182 /// registers that must be created and coalesced. This is a clear win except on 1183 /// targets with multiple condition code registers (PowerPC), where it might 1184 /// lose; some adjustment may be wanted there. 1185 /// 1186 /// Return true if any changes are made. 1187 static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) { 1188 BasicBlock *DefBB = CI->getParent(); 1189 1190 // Avoid sinking soft-FP comparisons, since this can move them into a loop. 1191 if (TLI && TLI->useSoftFloat() && isa<FCmpInst>(CI)) 1192 return false; 1193 1194 // Only insert a cmp in each block once. 1195 DenseMap<BasicBlock*, CmpInst*> InsertedCmps; 1196 1197 bool MadeChange = false; 1198 for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); 1199 UI != E; ) { 1200 Use &TheUse = UI.getUse(); 1201 Instruction *User = cast<Instruction>(*UI); 1202 1203 // Preincrement use iterator so we don't invalidate it. 1204 ++UI; 1205 1206 // Don't bother for PHI nodes. 1207 if (isa<PHINode>(User)) 1208 continue; 1209 1210 // Figure out which BB this cmp is used in. 1211 BasicBlock *UserBB = User->getParent(); 1212 1213 // If this user is in the same block as the cmp, don't change the cmp. 1214 if (UserBB == DefBB) continue; 1215 1216 // If we have already inserted a cmp into this block, use it. 1217 CmpInst *&InsertedCmp = InsertedCmps[UserBB]; 1218 1219 if (!InsertedCmp) { 1220 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 1221 assert(InsertPt != UserBB->end()); 1222 InsertedCmp = 1223 CmpInst::Create(CI->getOpcode(), CI->getPredicate(), 1224 CI->getOperand(0), CI->getOperand(1), "", &*InsertPt); 1225 // Propagate the debug info. 1226 InsertedCmp->setDebugLoc(CI->getDebugLoc()); 1227 } 1228 1229 // Replace a use of the cmp with a use of the new cmp. 1230 TheUse = InsertedCmp; 1231 MadeChange = true; 1232 ++NumCmpUses; 1233 } 1234 1235 // If we removed all uses, nuke the cmp. 1236 if (CI->use_empty()) { 1237 CI->eraseFromParent(); 1238 MadeChange = true; 1239 } 1240 1241 return MadeChange; 1242 } 1243 1244 static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI) { 1245 if (SinkCmpExpression(CI, TLI)) 1246 return true; 1247 1248 if (CombineUAddWithOverflow(CI)) 1249 return true; 1250 1251 return false; 1252 } 1253 1254 /// Duplicate and sink the given 'and' instruction into user blocks where it is 1255 /// used in a compare to allow isel to generate better code for targets where 1256 /// this operation can be combined. 1257 /// 1258 /// Return true if any changes are made. 1259 static bool sinkAndCmp0Expression(Instruction *AndI, 1260 const TargetLowering &TLI, 1261 SetOfInstrs &InsertedInsts) { 1262 // Double-check that we're not trying to optimize an instruction that was 1263 // already optimized by some other part of this pass. 1264 assert(!InsertedInsts.count(AndI) && 1265 "Attempting to optimize already optimized and instruction"); 1266 (void) InsertedInsts; 1267 1268 // Nothing to do for single use in same basic block. 1269 if (AndI->hasOneUse() && 1270 AndI->getParent() == cast<Instruction>(*AndI->user_begin())->getParent()) 1271 return false; 1272 1273 // Try to avoid cases where sinking/duplicating is likely to increase register 1274 // pressure. 1275 if (!isa<ConstantInt>(AndI->getOperand(0)) && 1276 !isa<ConstantInt>(AndI->getOperand(1)) && 1277 AndI->getOperand(0)->hasOneUse() && AndI->getOperand(1)->hasOneUse()) 1278 return false; 1279 1280 for (auto *U : AndI->users()) { 1281 Instruction *User = cast<Instruction>(U); 1282 1283 // Only sink for and mask feeding icmp with 0. 1284 if (!isa<ICmpInst>(User)) 1285 return false; 1286 1287 auto *CmpC = dyn_cast<ConstantInt>(User->getOperand(1)); 1288 if (!CmpC || !CmpC->isZero()) 1289 return false; 1290 } 1291 1292 if (!TLI.isMaskAndCmp0FoldingBeneficial(*AndI)) 1293 return false; 1294 1295 LLVM_DEBUG(dbgs() << "found 'and' feeding only icmp 0;\n"); 1296 LLVM_DEBUG(AndI->getParent()->dump()); 1297 1298 // Push the 'and' into the same block as the icmp 0. There should only be 1299 // one (icmp (and, 0)) in each block, since CSE/GVN should have removed any 1300 // others, so we don't need to keep track of which BBs we insert into. 1301 for (Value::user_iterator UI = AndI->user_begin(), E = AndI->user_end(); 1302 UI != E; ) { 1303 Use &TheUse = UI.getUse(); 1304 Instruction *User = cast<Instruction>(*UI); 1305 1306 // Preincrement use iterator so we don't invalidate it. 1307 ++UI; 1308 1309 LLVM_DEBUG(dbgs() << "sinking 'and' use: " << *User << "\n"); 1310 1311 // Keep the 'and' in the same place if the use is already in the same block. 1312 Instruction *InsertPt = 1313 User->getParent() == AndI->getParent() ? AndI : User; 1314 Instruction *InsertedAnd = 1315 BinaryOperator::Create(Instruction::And, AndI->getOperand(0), 1316 AndI->getOperand(1), "", InsertPt); 1317 // Propagate the debug info. 1318 InsertedAnd->setDebugLoc(AndI->getDebugLoc()); 1319 1320 // Replace a use of the 'and' with a use of the new 'and'. 1321 TheUse = InsertedAnd; 1322 ++NumAndUses; 1323 LLVM_DEBUG(User->getParent()->dump()); 1324 } 1325 1326 // We removed all uses, nuke the and. 1327 AndI->eraseFromParent(); 1328 return true; 1329 } 1330 1331 /// Check if the candidates could be combined with a shift instruction, which 1332 /// includes: 1333 /// 1. Truncate instruction 1334 /// 2. And instruction and the imm is a mask of the low bits: 1335 /// imm & (imm+1) == 0 1336 static bool isExtractBitsCandidateUse(Instruction *User) { 1337 if (!isa<TruncInst>(User)) { 1338 if (User->getOpcode() != Instruction::And || 1339 !isa<ConstantInt>(User->getOperand(1))) 1340 return false; 1341 1342 const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue(); 1343 1344 if ((Cimm & (Cimm + 1)).getBoolValue()) 1345 return false; 1346 } 1347 return true; 1348 } 1349 1350 /// Sink both shift and truncate instruction to the use of truncate's BB. 1351 static bool 1352 SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI, 1353 DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts, 1354 const TargetLowering &TLI, const DataLayout &DL) { 1355 BasicBlock *UserBB = User->getParent(); 1356 DenseMap<BasicBlock *, CastInst *> InsertedTruncs; 1357 TruncInst *TruncI = dyn_cast<TruncInst>(User); 1358 bool MadeChange = false; 1359 1360 for (Value::user_iterator TruncUI = TruncI->user_begin(), 1361 TruncE = TruncI->user_end(); 1362 TruncUI != TruncE;) { 1363 1364 Use &TruncTheUse = TruncUI.getUse(); 1365 Instruction *TruncUser = cast<Instruction>(*TruncUI); 1366 // Preincrement use iterator so we don't invalidate it. 1367 1368 ++TruncUI; 1369 1370 int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode()); 1371 if (!ISDOpcode) 1372 continue; 1373 1374 // If the use is actually a legal node, there will not be an 1375 // implicit truncate. 1376 // FIXME: always querying the result type is just an 1377 // approximation; some nodes' legality is determined by the 1378 // operand or other means. There's no good way to find out though. 1379 if (TLI.isOperationLegalOrCustom( 1380 ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true))) 1381 continue; 1382 1383 // Don't bother for PHI nodes. 1384 if (isa<PHINode>(TruncUser)) 1385 continue; 1386 1387 BasicBlock *TruncUserBB = TruncUser->getParent(); 1388 1389 if (UserBB == TruncUserBB) 1390 continue; 1391 1392 BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB]; 1393 CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB]; 1394 1395 if (!InsertedShift && !InsertedTrunc) { 1396 BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt(); 1397 assert(InsertPt != TruncUserBB->end()); 1398 // Sink the shift 1399 if (ShiftI->getOpcode() == Instruction::AShr) 1400 InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, 1401 "", &*InsertPt); 1402 else 1403 InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, 1404 "", &*InsertPt); 1405 1406 // Sink the trunc 1407 BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt(); 1408 TruncInsertPt++; 1409 assert(TruncInsertPt != TruncUserBB->end()); 1410 1411 InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift, 1412 TruncI->getType(), "", &*TruncInsertPt); 1413 1414 MadeChange = true; 1415 1416 TruncTheUse = InsertedTrunc; 1417 } 1418 } 1419 return MadeChange; 1420 } 1421 1422 /// Sink the shift *right* instruction into user blocks if the uses could 1423 /// potentially be combined with this shift instruction and generate BitExtract 1424 /// instruction. It will only be applied if the architecture supports BitExtract 1425 /// instruction. Here is an example: 1426 /// BB1: 1427 /// %x.extract.shift = lshr i64 %arg1, 32 1428 /// BB2: 1429 /// %x.extract.trunc = trunc i64 %x.extract.shift to i16 1430 /// ==> 1431 /// 1432 /// BB2: 1433 /// %x.extract.shift.1 = lshr i64 %arg1, 32 1434 /// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16 1435 /// 1436 /// CodeGen will recognize the pattern in BB2 and generate BitExtract 1437 /// instruction. 1438 /// Return true if any changes are made. 1439 static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI, 1440 const TargetLowering &TLI, 1441 const DataLayout &DL) { 1442 BasicBlock *DefBB = ShiftI->getParent(); 1443 1444 /// Only insert instructions in each block once. 1445 DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts; 1446 1447 bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType())); 1448 1449 bool MadeChange = false; 1450 for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end(); 1451 UI != E;) { 1452 Use &TheUse = UI.getUse(); 1453 Instruction *User = cast<Instruction>(*UI); 1454 // Preincrement use iterator so we don't invalidate it. 1455 ++UI; 1456 1457 // Don't bother for PHI nodes. 1458 if (isa<PHINode>(User)) 1459 continue; 1460 1461 if (!isExtractBitsCandidateUse(User)) 1462 continue; 1463 1464 BasicBlock *UserBB = User->getParent(); 1465 1466 if (UserBB == DefBB) { 1467 // If the shift and truncate instruction are in the same BB. The use of 1468 // the truncate(TruncUse) may still introduce another truncate if not 1469 // legal. In this case, we would like to sink both shift and truncate 1470 // instruction to the BB of TruncUse. 1471 // for example: 1472 // BB1: 1473 // i64 shift.result = lshr i64 opnd, imm 1474 // trunc.result = trunc shift.result to i16 1475 // 1476 // BB2: 1477 // ----> We will have an implicit truncate here if the architecture does 1478 // not have i16 compare. 1479 // cmp i16 trunc.result, opnd2 1480 // 1481 if (isa<TruncInst>(User) && shiftIsLegal 1482 // If the type of the truncate is legal, no truncate will be 1483 // introduced in other basic blocks. 1484 && 1485 (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType())))) 1486 MadeChange = 1487 SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL); 1488 1489 continue; 1490 } 1491 // If we have already inserted a shift into this block, use it. 1492 BinaryOperator *&InsertedShift = InsertedShifts[UserBB]; 1493 1494 if (!InsertedShift) { 1495 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 1496 assert(InsertPt != UserBB->end()); 1497 1498 if (ShiftI->getOpcode() == Instruction::AShr) 1499 InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, 1500 "", &*InsertPt); 1501 else 1502 InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, 1503 "", &*InsertPt); 1504 1505 MadeChange = true; 1506 } 1507 1508 // Replace a use of the shift with a use of the new shift. 1509 TheUse = InsertedShift; 1510 } 1511 1512 // If we removed all uses, nuke the shift. 1513 if (ShiftI->use_empty()) 1514 ShiftI->eraseFromParent(); 1515 1516 return MadeChange; 1517 } 1518 1519 /// If counting leading or trailing zeros is an expensive operation and a zero 1520 /// input is defined, add a check for zero to avoid calling the intrinsic. 1521 /// 1522 /// We want to transform: 1523 /// %z = call i64 @llvm.cttz.i64(i64 %A, i1 false) 1524 /// 1525 /// into: 1526 /// entry: 1527 /// %cmpz = icmp eq i64 %A, 0 1528 /// br i1 %cmpz, label %cond.end, label %cond.false 1529 /// cond.false: 1530 /// %z = call i64 @llvm.cttz.i64(i64 %A, i1 true) 1531 /// br label %cond.end 1532 /// cond.end: 1533 /// %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ] 1534 /// 1535 /// If the transform is performed, return true and set ModifiedDT to true. 1536 static bool despeculateCountZeros(IntrinsicInst *CountZeros, 1537 const TargetLowering *TLI, 1538 const DataLayout *DL, 1539 bool &ModifiedDT) { 1540 if (!TLI || !DL) 1541 return false; 1542 1543 // If a zero input is undefined, it doesn't make sense to despeculate that. 1544 if (match(CountZeros->getOperand(1), m_One())) 1545 return false; 1546 1547 // If it's cheap to speculate, there's nothing to do. 1548 auto IntrinsicID = CountZeros->getIntrinsicID(); 1549 if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz()) || 1550 (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz())) 1551 return false; 1552 1553 // Only handle legal scalar cases. Anything else requires too much work. 1554 Type *Ty = CountZeros->getType(); 1555 unsigned SizeInBits = Ty->getPrimitiveSizeInBits(); 1556 if (Ty->isVectorTy() || SizeInBits > DL->getLargestLegalIntTypeSizeInBits()) 1557 return false; 1558 1559 // The intrinsic will be sunk behind a compare against zero and branch. 1560 BasicBlock *StartBlock = CountZeros->getParent(); 1561 BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false"); 1562 1563 // Create another block after the count zero intrinsic. A PHI will be added 1564 // in this block to select the result of the intrinsic or the bit-width 1565 // constant if the input to the intrinsic is zero. 1566 BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(CountZeros)); 1567 BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end"); 1568 1569 // Set up a builder to create a compare, conditional branch, and PHI. 1570 IRBuilder<> Builder(CountZeros->getContext()); 1571 Builder.SetInsertPoint(StartBlock->getTerminator()); 1572 Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc()); 1573 1574 // Replace the unconditional branch that was created by the first split with 1575 // a compare against zero and a conditional branch. 1576 Value *Zero = Constant::getNullValue(Ty); 1577 Value *Cmp = Builder.CreateICmpEQ(CountZeros->getOperand(0), Zero, "cmpz"); 1578 Builder.CreateCondBr(Cmp, EndBlock, CallBlock); 1579 StartBlock->getTerminator()->eraseFromParent(); 1580 1581 // Create a PHI in the end block to select either the output of the intrinsic 1582 // or the bit width of the operand. 1583 Builder.SetInsertPoint(&EndBlock->front()); 1584 PHINode *PN = Builder.CreatePHI(Ty, 2, "ctz"); 1585 CountZeros->replaceAllUsesWith(PN); 1586 Value *BitWidth = Builder.getInt(APInt(SizeInBits, SizeInBits)); 1587 PN->addIncoming(BitWidth, StartBlock); 1588 PN->addIncoming(CountZeros, CallBlock); 1589 1590 // We are explicitly handling the zero case, so we can set the intrinsic's 1591 // undefined zero argument to 'true'. This will also prevent reprocessing the 1592 // intrinsic; we only despeculate when a zero input is defined. 1593 CountZeros->setArgOperand(1, Builder.getTrue()); 1594 ModifiedDT = true; 1595 return true; 1596 } 1597 1598 bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) { 1599 BasicBlock *BB = CI->getParent(); 1600 1601 // Lower inline assembly if we can. 1602 // If we found an inline asm expession, and if the target knows how to 1603 // lower it to normal LLVM code, do so now. 1604 if (TLI && isa<InlineAsm>(CI->getCalledValue())) { 1605 if (TLI->ExpandInlineAsm(CI)) { 1606 // Avoid invalidating the iterator. 1607 CurInstIterator = BB->begin(); 1608 // Avoid processing instructions out of order, which could cause 1609 // reuse before a value is defined. 1610 SunkAddrs.clear(); 1611 return true; 1612 } 1613 // Sink address computing for memory operands into the block. 1614 if (optimizeInlineAsmInst(CI)) 1615 return true; 1616 } 1617 1618 // Align the pointer arguments to this call if the target thinks it's a good 1619 // idea 1620 unsigned MinSize, PrefAlign; 1621 if (TLI && TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) { 1622 for (auto &Arg : CI->arg_operands()) { 1623 // We want to align both objects whose address is used directly and 1624 // objects whose address is used in casts and GEPs, though it only makes 1625 // sense for GEPs if the offset is a multiple of the desired alignment and 1626 // if size - offset meets the size threshold. 1627 if (!Arg->getType()->isPointerTy()) 1628 continue; 1629 APInt Offset(DL->getIndexSizeInBits( 1630 cast<PointerType>(Arg->getType())->getAddressSpace()), 1631 0); 1632 Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset); 1633 uint64_t Offset2 = Offset.getLimitedValue(); 1634 if ((Offset2 & (PrefAlign-1)) != 0) 1635 continue; 1636 AllocaInst *AI; 1637 if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign && 1638 DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2) 1639 AI->setAlignment(PrefAlign); 1640 // Global variables can only be aligned if they are defined in this 1641 // object (i.e. they are uniquely initialized in this object), and 1642 // over-aligning global variables that have an explicit section is 1643 // forbidden. 1644 GlobalVariable *GV; 1645 if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() && 1646 GV->getPointerAlignment(*DL) < PrefAlign && 1647 DL->getTypeAllocSize(GV->getValueType()) >= 1648 MinSize + Offset2) 1649 GV->setAlignment(PrefAlign); 1650 } 1651 // If this is a memcpy (or similar) then we may be able to improve the 1652 // alignment 1653 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) { 1654 unsigned DestAlign = getKnownAlignment(MI->getDest(), *DL); 1655 if (DestAlign > MI->getDestAlignment()) 1656 MI->setDestAlignment(DestAlign); 1657 if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) { 1658 unsigned SrcAlign = getKnownAlignment(MTI->getSource(), *DL); 1659 if (SrcAlign > MTI->getSourceAlignment()) 1660 MTI->setSourceAlignment(SrcAlign); 1661 } 1662 } 1663 } 1664 1665 // If we have a cold call site, try to sink addressing computation into the 1666 // cold block. This interacts with our handling for loads and stores to 1667 // ensure that we can fold all uses of a potential addressing computation 1668 // into their uses. TODO: generalize this to work over profiling data 1669 if (!OptSize && CI->hasFnAttr(Attribute::Cold)) 1670 for (auto &Arg : CI->arg_operands()) { 1671 if (!Arg->getType()->isPointerTy()) 1672 continue; 1673 unsigned AS = Arg->getType()->getPointerAddressSpace(); 1674 return optimizeMemoryInst(CI, Arg, Arg->getType(), AS); 1675 } 1676 1677 IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI); 1678 if (II) { 1679 switch (II->getIntrinsicID()) { 1680 default: break; 1681 case Intrinsic::objectsize: { 1682 // Lower all uses of llvm.objectsize.* 1683 ConstantInt *RetVal = 1684 lowerObjectSizeCall(II, *DL, TLInfo, /*MustSucceed=*/true); 1685 // Substituting this can cause recursive simplifications, which can 1686 // invalidate our iterator. Use a WeakTrackingVH to hold onto it in case 1687 // this 1688 // happens. 1689 Value *CurValue = &*CurInstIterator; 1690 WeakTrackingVH IterHandle(CurValue); 1691 1692 replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr); 1693 1694 // If the iterator instruction was recursively deleted, start over at the 1695 // start of the block. 1696 if (IterHandle != CurValue) { 1697 CurInstIterator = BB->begin(); 1698 SunkAddrs.clear(); 1699 } 1700 return true; 1701 } 1702 case Intrinsic::aarch64_stlxr: 1703 case Intrinsic::aarch64_stxr: { 1704 ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0)); 1705 if (!ExtVal || !ExtVal->hasOneUse() || 1706 ExtVal->getParent() == CI->getParent()) 1707 return false; 1708 // Sink a zext feeding stlxr/stxr before it, so it can be folded into it. 1709 ExtVal->moveBefore(CI); 1710 // Mark this instruction as "inserted by CGP", so that other 1711 // optimizations don't touch it. 1712 InsertedInsts.insert(ExtVal); 1713 return true; 1714 } 1715 case Intrinsic::launder_invariant_group: 1716 case Intrinsic::strip_invariant_group: 1717 II->replaceAllUsesWith(II->getArgOperand(0)); 1718 II->eraseFromParent(); 1719 return true; 1720 1721 case Intrinsic::cttz: 1722 case Intrinsic::ctlz: 1723 // If counting zeros is expensive, try to avoid it. 1724 return despeculateCountZeros(II, TLI, DL, ModifiedDT); 1725 } 1726 1727 if (TLI) { 1728 SmallVector<Value*, 2> PtrOps; 1729 Type *AccessTy; 1730 if (TLI->getAddrModeArguments(II, PtrOps, AccessTy)) 1731 while (!PtrOps.empty()) { 1732 Value *PtrVal = PtrOps.pop_back_val(); 1733 unsigned AS = PtrVal->getType()->getPointerAddressSpace(); 1734 if (optimizeMemoryInst(II, PtrVal, AccessTy, AS)) 1735 return true; 1736 } 1737 } 1738 } 1739 1740 // From here on out we're working with named functions. 1741 if (!CI->getCalledFunction()) return false; 1742 1743 // Lower all default uses of _chk calls. This is very similar 1744 // to what InstCombineCalls does, but here we are only lowering calls 1745 // to fortified library functions (e.g. __memcpy_chk) that have the default 1746 // "don't know" as the objectsize. Anything else should be left alone. 1747 FortifiedLibCallSimplifier Simplifier(TLInfo, true); 1748 if (Value *V = Simplifier.optimizeCall(CI)) { 1749 CI->replaceAllUsesWith(V); 1750 CI->eraseFromParent(); 1751 return true; 1752 } 1753 1754 return false; 1755 } 1756 1757 /// Look for opportunities to duplicate return instructions to the predecessor 1758 /// to enable tail call optimizations. The case it is currently looking for is: 1759 /// @code 1760 /// bb0: 1761 /// %tmp0 = tail call i32 @f0() 1762 /// br label %return 1763 /// bb1: 1764 /// %tmp1 = tail call i32 @f1() 1765 /// br label %return 1766 /// bb2: 1767 /// %tmp2 = tail call i32 @f2() 1768 /// br label %return 1769 /// return: 1770 /// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ] 1771 /// ret i32 %retval 1772 /// @endcode 1773 /// 1774 /// => 1775 /// 1776 /// @code 1777 /// bb0: 1778 /// %tmp0 = tail call i32 @f0() 1779 /// ret i32 %tmp0 1780 /// bb1: 1781 /// %tmp1 = tail call i32 @f1() 1782 /// ret i32 %tmp1 1783 /// bb2: 1784 /// %tmp2 = tail call i32 @f2() 1785 /// ret i32 %tmp2 1786 /// @endcode 1787 bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) { 1788 if (!TLI) 1789 return false; 1790 1791 ReturnInst *RetI = dyn_cast<ReturnInst>(BB->getTerminator()); 1792 if (!RetI) 1793 return false; 1794 1795 PHINode *PN = nullptr; 1796 BitCastInst *BCI = nullptr; 1797 Value *V = RetI->getReturnValue(); 1798 if (V) { 1799 BCI = dyn_cast<BitCastInst>(V); 1800 if (BCI) 1801 V = BCI->getOperand(0); 1802 1803 PN = dyn_cast<PHINode>(V); 1804 if (!PN) 1805 return false; 1806 } 1807 1808 if (PN && PN->getParent() != BB) 1809 return false; 1810 1811 // Make sure there are no instructions between the PHI and return, or that the 1812 // return is the first instruction in the block. 1813 if (PN) { 1814 BasicBlock::iterator BI = BB->begin(); 1815 do { ++BI; } while (isa<DbgInfoIntrinsic>(BI)); 1816 if (&*BI == BCI) 1817 // Also skip over the bitcast. 1818 ++BI; 1819 if (&*BI != RetI) 1820 return false; 1821 } else { 1822 BasicBlock::iterator BI = BB->begin(); 1823 while (isa<DbgInfoIntrinsic>(BI)) ++BI; 1824 if (&*BI != RetI) 1825 return false; 1826 } 1827 1828 /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail 1829 /// call. 1830 const Function *F = BB->getParent(); 1831 SmallVector<CallInst*, 4> TailCalls; 1832 if (PN) { 1833 for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) { 1834 CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I)); 1835 // Make sure the phi value is indeed produced by the tail call. 1836 if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) && 1837 TLI->mayBeEmittedAsTailCall(CI) && 1838 attributesPermitTailCall(F, CI, RetI, *TLI)) 1839 TailCalls.push_back(CI); 1840 } 1841 } else { 1842 SmallPtrSet<BasicBlock*, 4> VisitedBBs; 1843 for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) { 1844 if (!VisitedBBs.insert(*PI).second) 1845 continue; 1846 1847 BasicBlock::InstListType &InstList = (*PI)->getInstList(); 1848 BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin(); 1849 BasicBlock::InstListType::reverse_iterator RE = InstList.rend(); 1850 do { ++RI; } while (RI != RE && isa<DbgInfoIntrinsic>(&*RI)); 1851 if (RI == RE) 1852 continue; 1853 1854 CallInst *CI = dyn_cast<CallInst>(&*RI); 1855 if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI) && 1856 attributesPermitTailCall(F, CI, RetI, *TLI)) 1857 TailCalls.push_back(CI); 1858 } 1859 } 1860 1861 bool Changed = false; 1862 for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) { 1863 CallInst *CI = TailCalls[i]; 1864 CallSite CS(CI); 1865 1866 // Conservatively require the attributes of the call to match those of the 1867 // return. Ignore noalias because it doesn't affect the call sequence. 1868 AttributeList CalleeAttrs = CS.getAttributes(); 1869 if (AttrBuilder(CalleeAttrs, AttributeList::ReturnIndex) 1870 .removeAttribute(Attribute::NoAlias) != 1871 AttrBuilder(CalleeAttrs, AttributeList::ReturnIndex) 1872 .removeAttribute(Attribute::NoAlias)) 1873 continue; 1874 1875 // Make sure the call instruction is followed by an unconditional branch to 1876 // the return block. 1877 BasicBlock *CallBB = CI->getParent(); 1878 BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator()); 1879 if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB) 1880 continue; 1881 1882 // Duplicate the return into CallBB. 1883 (void)FoldReturnIntoUncondBranch(RetI, BB, CallBB); 1884 ModifiedDT = Changed = true; 1885 ++NumRetsDup; 1886 } 1887 1888 // If we eliminated all predecessors of the block, delete the block now. 1889 if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB)) 1890 BB->eraseFromParent(); 1891 1892 return Changed; 1893 } 1894 1895 //===----------------------------------------------------------------------===// 1896 // Memory Optimization 1897 //===----------------------------------------------------------------------===// 1898 1899 namespace { 1900 1901 /// This is an extended version of TargetLowering::AddrMode 1902 /// which holds actual Value*'s for register values. 1903 struct ExtAddrMode : public TargetLowering::AddrMode { 1904 Value *BaseReg = nullptr; 1905 Value *ScaledReg = nullptr; 1906 Value *OriginalValue = nullptr; 1907 1908 enum FieldName { 1909 NoField = 0x00, 1910 BaseRegField = 0x01, 1911 BaseGVField = 0x02, 1912 BaseOffsField = 0x04, 1913 ScaledRegField = 0x08, 1914 ScaleField = 0x10, 1915 MultipleFields = 0xff 1916 }; 1917 1918 ExtAddrMode() = default; 1919 1920 void print(raw_ostream &OS) const; 1921 void dump() const; 1922 1923 FieldName compare(const ExtAddrMode &other) { 1924 // First check that the types are the same on each field, as differing types 1925 // is something we can't cope with later on. 1926 if (BaseReg && other.BaseReg && 1927 BaseReg->getType() != other.BaseReg->getType()) 1928 return MultipleFields; 1929 if (BaseGV && other.BaseGV && 1930 BaseGV->getType() != other.BaseGV->getType()) 1931 return MultipleFields; 1932 if (ScaledReg && other.ScaledReg && 1933 ScaledReg->getType() != other.ScaledReg->getType()) 1934 return MultipleFields; 1935 1936 // Check each field to see if it differs. 1937 unsigned Result = NoField; 1938 if (BaseReg != other.BaseReg) 1939 Result |= BaseRegField; 1940 if (BaseGV != other.BaseGV) 1941 Result |= BaseGVField; 1942 if (BaseOffs != other.BaseOffs) 1943 Result |= BaseOffsField; 1944 if (ScaledReg != other.ScaledReg) 1945 Result |= ScaledRegField; 1946 // Don't count 0 as being a different scale, because that actually means 1947 // unscaled (which will already be counted by having no ScaledReg). 1948 if (Scale && other.Scale && Scale != other.Scale) 1949 Result |= ScaleField; 1950 1951 if (countPopulation(Result) > 1) 1952 return MultipleFields; 1953 else 1954 return static_cast<FieldName>(Result); 1955 } 1956 1957 // An AddrMode is trivial if it involves no calculation i.e. it is just a base 1958 // with no offset. 1959 bool isTrivial() { 1960 // An AddrMode is (BaseGV + BaseReg + BaseOffs + ScaleReg * Scale) so it is 1961 // trivial if at most one of these terms is nonzero, except that BaseGV and 1962 // BaseReg both being zero actually means a null pointer value, which we 1963 // consider to be 'non-zero' here. 1964 return !BaseOffs && !Scale && !(BaseGV && BaseReg); 1965 } 1966 1967 Value *GetFieldAsValue(FieldName Field, Type *IntPtrTy) { 1968 switch (Field) { 1969 default: 1970 return nullptr; 1971 case BaseRegField: 1972 return BaseReg; 1973 case BaseGVField: 1974 return BaseGV; 1975 case ScaledRegField: 1976 return ScaledReg; 1977 case BaseOffsField: 1978 return ConstantInt::get(IntPtrTy, BaseOffs); 1979 } 1980 } 1981 1982 void SetCombinedField(FieldName Field, Value *V, 1983 const SmallVectorImpl<ExtAddrMode> &AddrModes) { 1984 switch (Field) { 1985 default: 1986 llvm_unreachable("Unhandled fields are expected to be rejected earlier"); 1987 break; 1988 case ExtAddrMode::BaseRegField: 1989 BaseReg = V; 1990 break; 1991 case ExtAddrMode::BaseGVField: 1992 // A combined BaseGV is an Instruction, not a GlobalValue, so it goes 1993 // in the BaseReg field. 1994 assert(BaseReg == nullptr); 1995 BaseReg = V; 1996 BaseGV = nullptr; 1997 break; 1998 case ExtAddrMode::ScaledRegField: 1999 ScaledReg = V; 2000 // If we have a mix of scaled and unscaled addrmodes then we want scale 2001 // to be the scale and not zero. 2002 if (!Scale) 2003 for (const ExtAddrMode &AM : AddrModes) 2004 if (AM.Scale) { 2005 Scale = AM.Scale; 2006 break; 2007 } 2008 break; 2009 case ExtAddrMode::BaseOffsField: 2010 // The offset is no longer a constant, so it goes in ScaledReg with a 2011 // scale of 1. 2012 assert(ScaledReg == nullptr); 2013 ScaledReg = V; 2014 Scale = 1; 2015 BaseOffs = 0; 2016 break; 2017 } 2018 } 2019 }; 2020 2021 } // end anonymous namespace 2022 2023 #ifndef NDEBUG 2024 static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) { 2025 AM.print(OS); 2026 return OS; 2027 } 2028 #endif 2029 2030 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) 2031 void ExtAddrMode::print(raw_ostream &OS) const { 2032 bool NeedPlus = false; 2033 OS << "["; 2034 if (BaseGV) { 2035 OS << (NeedPlus ? " + " : "") 2036 << "GV:"; 2037 BaseGV->printAsOperand(OS, /*PrintType=*/false); 2038 NeedPlus = true; 2039 } 2040 2041 if (BaseOffs) { 2042 OS << (NeedPlus ? " + " : "") 2043 << BaseOffs; 2044 NeedPlus = true; 2045 } 2046 2047 if (BaseReg) { 2048 OS << (NeedPlus ? " + " : "") 2049 << "Base:"; 2050 BaseReg->printAsOperand(OS, /*PrintType=*/false); 2051 NeedPlus = true; 2052 } 2053 if (Scale) { 2054 OS << (NeedPlus ? " + " : "") 2055 << Scale << "*"; 2056 ScaledReg->printAsOperand(OS, /*PrintType=*/false); 2057 } 2058 2059 OS << ']'; 2060 } 2061 2062 LLVM_DUMP_METHOD void ExtAddrMode::dump() const { 2063 print(dbgs()); 2064 dbgs() << '\n'; 2065 } 2066 #endif 2067 2068 namespace { 2069 2070 /// This class provides transaction based operation on the IR. 2071 /// Every change made through this class is recorded in the internal state and 2072 /// can be undone (rollback) until commit is called. 2073 class TypePromotionTransaction { 2074 /// This represents the common interface of the individual transaction. 2075 /// Each class implements the logic for doing one specific modification on 2076 /// the IR via the TypePromotionTransaction. 2077 class TypePromotionAction { 2078 protected: 2079 /// The Instruction modified. 2080 Instruction *Inst; 2081 2082 public: 2083 /// Constructor of the action. 2084 /// The constructor performs the related action on the IR. 2085 TypePromotionAction(Instruction *Inst) : Inst(Inst) {} 2086 2087 virtual ~TypePromotionAction() = default; 2088 2089 /// Undo the modification done by this action. 2090 /// When this method is called, the IR must be in the same state as it was 2091 /// before this action was applied. 2092 /// \pre Undoing the action works if and only if the IR is in the exact same 2093 /// state as it was directly after this action was applied. 2094 virtual void undo() = 0; 2095 2096 /// Advocate every change made by this action. 2097 /// When the results on the IR of the action are to be kept, it is important 2098 /// to call this function, otherwise hidden information may be kept forever. 2099 virtual void commit() { 2100 // Nothing to be done, this action is not doing anything. 2101 } 2102 }; 2103 2104 /// Utility to remember the position of an instruction. 2105 class InsertionHandler { 2106 /// Position of an instruction. 2107 /// Either an instruction: 2108 /// - Is the first in a basic block: BB is used. 2109 /// - Has a previous instruction: PrevInst is used. 2110 union { 2111 Instruction *PrevInst; 2112 BasicBlock *BB; 2113 } Point; 2114 2115 /// Remember whether or not the instruction had a previous instruction. 2116 bool HasPrevInstruction; 2117 2118 public: 2119 /// Record the position of \p Inst. 2120 InsertionHandler(Instruction *Inst) { 2121 BasicBlock::iterator It = Inst->getIterator(); 2122 HasPrevInstruction = (It != (Inst->getParent()->begin())); 2123 if (HasPrevInstruction) 2124 Point.PrevInst = &*--It; 2125 else 2126 Point.BB = Inst->getParent(); 2127 } 2128 2129 /// Insert \p Inst at the recorded position. 2130 void insert(Instruction *Inst) { 2131 if (HasPrevInstruction) { 2132 if (Inst->getParent()) 2133 Inst->removeFromParent(); 2134 Inst->insertAfter(Point.PrevInst); 2135 } else { 2136 Instruction *Position = &*Point.BB->getFirstInsertionPt(); 2137 if (Inst->getParent()) 2138 Inst->moveBefore(Position); 2139 else 2140 Inst->insertBefore(Position); 2141 } 2142 } 2143 }; 2144 2145 /// Move an instruction before another. 2146 class InstructionMoveBefore : public TypePromotionAction { 2147 /// Original position of the instruction. 2148 InsertionHandler Position; 2149 2150 public: 2151 /// Move \p Inst before \p Before. 2152 InstructionMoveBefore(Instruction *Inst, Instruction *Before) 2153 : TypePromotionAction(Inst), Position(Inst) { 2154 LLVM_DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before 2155 << "\n"); 2156 Inst->moveBefore(Before); 2157 } 2158 2159 /// Move the instruction back to its original position. 2160 void undo() override { 2161 LLVM_DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n"); 2162 Position.insert(Inst); 2163 } 2164 }; 2165 2166 /// Set the operand of an instruction with a new value. 2167 class OperandSetter : public TypePromotionAction { 2168 /// Original operand of the instruction. 2169 Value *Origin; 2170 2171 /// Index of the modified instruction. 2172 unsigned Idx; 2173 2174 public: 2175 /// Set \p Idx operand of \p Inst with \p NewVal. 2176 OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal) 2177 : TypePromotionAction(Inst), Idx(Idx) { 2178 LLVM_DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n" 2179 << "for:" << *Inst << "\n" 2180 << "with:" << *NewVal << "\n"); 2181 Origin = Inst->getOperand(Idx); 2182 Inst->setOperand(Idx, NewVal); 2183 } 2184 2185 /// Restore the original value of the instruction. 2186 void undo() override { 2187 LLVM_DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n" 2188 << "for: " << *Inst << "\n" 2189 << "with: " << *Origin << "\n"); 2190 Inst->setOperand(Idx, Origin); 2191 } 2192 }; 2193 2194 /// Hide the operands of an instruction. 2195 /// Do as if this instruction was not using any of its operands. 2196 class OperandsHider : public TypePromotionAction { 2197 /// The list of original operands. 2198 SmallVector<Value *, 4> OriginalValues; 2199 2200 public: 2201 /// Remove \p Inst from the uses of the operands of \p Inst. 2202 OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) { 2203 LLVM_DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n"); 2204 unsigned NumOpnds = Inst->getNumOperands(); 2205 OriginalValues.reserve(NumOpnds); 2206 for (unsigned It = 0; It < NumOpnds; ++It) { 2207 // Save the current operand. 2208 Value *Val = Inst->getOperand(It); 2209 OriginalValues.push_back(Val); 2210 // Set a dummy one. 2211 // We could use OperandSetter here, but that would imply an overhead 2212 // that we are not willing to pay. 2213 Inst->setOperand(It, UndefValue::get(Val->getType())); 2214 } 2215 } 2216 2217 /// Restore the original list of uses. 2218 void undo() override { 2219 LLVM_DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n"); 2220 for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It) 2221 Inst->setOperand(It, OriginalValues[It]); 2222 } 2223 }; 2224 2225 /// Build a truncate instruction. 2226 class TruncBuilder : public TypePromotionAction { 2227 Value *Val; 2228 2229 public: 2230 /// Build a truncate instruction of \p Opnd producing a \p Ty 2231 /// result. 2232 /// trunc Opnd to Ty. 2233 TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) { 2234 IRBuilder<> Builder(Opnd); 2235 Val = Builder.CreateTrunc(Opnd, Ty, "promoted"); 2236 LLVM_DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n"); 2237 } 2238 2239 /// Get the built value. 2240 Value *getBuiltValue() { return Val; } 2241 2242 /// Remove the built instruction. 2243 void undo() override { 2244 LLVM_DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n"); 2245 if (Instruction *IVal = dyn_cast<Instruction>(Val)) 2246 IVal->eraseFromParent(); 2247 } 2248 }; 2249 2250 /// Build a sign extension instruction. 2251 class SExtBuilder : public TypePromotionAction { 2252 Value *Val; 2253 2254 public: 2255 /// Build a sign extension instruction of \p Opnd producing a \p Ty 2256 /// result. 2257 /// sext Opnd to Ty. 2258 SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty) 2259 : TypePromotionAction(InsertPt) { 2260 IRBuilder<> Builder(InsertPt); 2261 Val = Builder.CreateSExt(Opnd, Ty, "promoted"); 2262 LLVM_DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n"); 2263 } 2264 2265 /// Get the built value. 2266 Value *getBuiltValue() { return Val; } 2267 2268 /// Remove the built instruction. 2269 void undo() override { 2270 LLVM_DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n"); 2271 if (Instruction *IVal = dyn_cast<Instruction>(Val)) 2272 IVal->eraseFromParent(); 2273 } 2274 }; 2275 2276 /// Build a zero extension instruction. 2277 class ZExtBuilder : public TypePromotionAction { 2278 Value *Val; 2279 2280 public: 2281 /// Build a zero extension instruction of \p Opnd producing a \p Ty 2282 /// result. 2283 /// zext Opnd to Ty. 2284 ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty) 2285 : TypePromotionAction(InsertPt) { 2286 IRBuilder<> Builder(InsertPt); 2287 Val = Builder.CreateZExt(Opnd, Ty, "promoted"); 2288 LLVM_DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n"); 2289 } 2290 2291 /// Get the built value. 2292 Value *getBuiltValue() { return Val; } 2293 2294 /// Remove the built instruction. 2295 void undo() override { 2296 LLVM_DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n"); 2297 if (Instruction *IVal = dyn_cast<Instruction>(Val)) 2298 IVal->eraseFromParent(); 2299 } 2300 }; 2301 2302 /// Mutate an instruction to another type. 2303 class TypeMutator : public TypePromotionAction { 2304 /// Record the original type. 2305 Type *OrigTy; 2306 2307 public: 2308 /// Mutate the type of \p Inst into \p NewTy. 2309 TypeMutator(Instruction *Inst, Type *NewTy) 2310 : TypePromotionAction(Inst), OrigTy(Inst->getType()) { 2311 LLVM_DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy 2312 << "\n"); 2313 Inst->mutateType(NewTy); 2314 } 2315 2316 /// Mutate the instruction back to its original type. 2317 void undo() override { 2318 LLVM_DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy 2319 << "\n"); 2320 Inst->mutateType(OrigTy); 2321 } 2322 }; 2323 2324 /// Replace the uses of an instruction by another instruction. 2325 class UsesReplacer : public TypePromotionAction { 2326 /// Helper structure to keep track of the replaced uses. 2327 struct InstructionAndIdx { 2328 /// The instruction using the instruction. 2329 Instruction *Inst; 2330 2331 /// The index where this instruction is used for Inst. 2332 unsigned Idx; 2333 2334 InstructionAndIdx(Instruction *Inst, unsigned Idx) 2335 : Inst(Inst), Idx(Idx) {} 2336 }; 2337 2338 /// Keep track of the original uses (pair Instruction, Index). 2339 SmallVector<InstructionAndIdx, 4> OriginalUses; 2340 2341 using use_iterator = SmallVectorImpl<InstructionAndIdx>::iterator; 2342 2343 public: 2344 /// Replace all the use of \p Inst by \p New. 2345 UsesReplacer(Instruction *Inst, Value *New) : TypePromotionAction(Inst) { 2346 LLVM_DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New 2347 << "\n"); 2348 // Record the original uses. 2349 for (Use &U : Inst->uses()) { 2350 Instruction *UserI = cast<Instruction>(U.getUser()); 2351 OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo())); 2352 } 2353 // Now, we can replace the uses. 2354 Inst->replaceAllUsesWith(New); 2355 } 2356 2357 /// Reassign the original uses of Inst to Inst. 2358 void undo() override { 2359 LLVM_DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n"); 2360 for (use_iterator UseIt = OriginalUses.begin(), 2361 EndIt = OriginalUses.end(); 2362 UseIt != EndIt; ++UseIt) { 2363 UseIt->Inst->setOperand(UseIt->Idx, Inst); 2364 } 2365 } 2366 }; 2367 2368 /// Remove an instruction from the IR. 2369 class InstructionRemover : public TypePromotionAction { 2370 /// Original position of the instruction. 2371 InsertionHandler Inserter; 2372 2373 /// Helper structure to hide all the link to the instruction. In other 2374 /// words, this helps to do as if the instruction was removed. 2375 OperandsHider Hider; 2376 2377 /// Keep track of the uses replaced, if any. 2378 UsesReplacer *Replacer = nullptr; 2379 2380 /// Keep track of instructions removed. 2381 SetOfInstrs &RemovedInsts; 2382 2383 public: 2384 /// Remove all reference of \p Inst and optionally replace all its 2385 /// uses with New. 2386 /// \p RemovedInsts Keep track of the instructions removed by this Action. 2387 /// \pre If !Inst->use_empty(), then New != nullptr 2388 InstructionRemover(Instruction *Inst, SetOfInstrs &RemovedInsts, 2389 Value *New = nullptr) 2390 : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst), 2391 RemovedInsts(RemovedInsts) { 2392 if (New) 2393 Replacer = new UsesReplacer(Inst, New); 2394 LLVM_DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n"); 2395 RemovedInsts.insert(Inst); 2396 /// The instructions removed here will be freed after completing 2397 /// optimizeBlock() for all blocks as we need to keep track of the 2398 /// removed instructions during promotion. 2399 Inst->removeFromParent(); 2400 } 2401 2402 ~InstructionRemover() override { delete Replacer; } 2403 2404 /// Resurrect the instruction and reassign it to the proper uses if 2405 /// new value was provided when build this action. 2406 void undo() override { 2407 LLVM_DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n"); 2408 Inserter.insert(Inst); 2409 if (Replacer) 2410 Replacer->undo(); 2411 Hider.undo(); 2412 RemovedInsts.erase(Inst); 2413 } 2414 }; 2415 2416 public: 2417 /// Restoration point. 2418 /// The restoration point is a pointer to an action instead of an iterator 2419 /// because the iterator may be invalidated but not the pointer. 2420 using ConstRestorationPt = const TypePromotionAction *; 2421 2422 TypePromotionTransaction(SetOfInstrs &RemovedInsts) 2423 : RemovedInsts(RemovedInsts) {} 2424 2425 /// Advocate every changes made in that transaction. 2426 void commit(); 2427 2428 /// Undo all the changes made after the given point. 2429 void rollback(ConstRestorationPt Point); 2430 2431 /// Get the current restoration point. 2432 ConstRestorationPt getRestorationPoint() const; 2433 2434 /// \name API for IR modification with state keeping to support rollback. 2435 /// @{ 2436 /// Same as Instruction::setOperand. 2437 void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal); 2438 2439 /// Same as Instruction::eraseFromParent. 2440 void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr); 2441 2442 /// Same as Value::replaceAllUsesWith. 2443 void replaceAllUsesWith(Instruction *Inst, Value *New); 2444 2445 /// Same as Value::mutateType. 2446 void mutateType(Instruction *Inst, Type *NewTy); 2447 2448 /// Same as IRBuilder::createTrunc. 2449 Value *createTrunc(Instruction *Opnd, Type *Ty); 2450 2451 /// Same as IRBuilder::createSExt. 2452 Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty); 2453 2454 /// Same as IRBuilder::createZExt. 2455 Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty); 2456 2457 /// Same as Instruction::moveBefore. 2458 void moveBefore(Instruction *Inst, Instruction *Before); 2459 /// @} 2460 2461 private: 2462 /// The ordered list of actions made so far. 2463 SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions; 2464 2465 using CommitPt = SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator; 2466 2467 SetOfInstrs &RemovedInsts; 2468 }; 2469 2470 } // end anonymous namespace 2471 2472 void TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx, 2473 Value *NewVal) { 2474 Actions.push_back(llvm::make_unique<TypePromotionTransaction::OperandSetter>( 2475 Inst, Idx, NewVal)); 2476 } 2477 2478 void TypePromotionTransaction::eraseInstruction(Instruction *Inst, 2479 Value *NewVal) { 2480 Actions.push_back( 2481 llvm::make_unique<TypePromotionTransaction::InstructionRemover>( 2482 Inst, RemovedInsts, NewVal)); 2483 } 2484 2485 void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst, 2486 Value *New) { 2487 Actions.push_back( 2488 llvm::make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New)); 2489 } 2490 2491 void TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) { 2492 Actions.push_back( 2493 llvm::make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy)); 2494 } 2495 2496 Value *TypePromotionTransaction::createTrunc(Instruction *Opnd, 2497 Type *Ty) { 2498 std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty)); 2499 Value *Val = Ptr->getBuiltValue(); 2500 Actions.push_back(std::move(Ptr)); 2501 return Val; 2502 } 2503 2504 Value *TypePromotionTransaction::createSExt(Instruction *Inst, 2505 Value *Opnd, Type *Ty) { 2506 std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty)); 2507 Value *Val = Ptr->getBuiltValue(); 2508 Actions.push_back(std::move(Ptr)); 2509 return Val; 2510 } 2511 2512 Value *TypePromotionTransaction::createZExt(Instruction *Inst, 2513 Value *Opnd, Type *Ty) { 2514 std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty)); 2515 Value *Val = Ptr->getBuiltValue(); 2516 Actions.push_back(std::move(Ptr)); 2517 return Val; 2518 } 2519 2520 void TypePromotionTransaction::moveBefore(Instruction *Inst, 2521 Instruction *Before) { 2522 Actions.push_back( 2523 llvm::make_unique<TypePromotionTransaction::InstructionMoveBefore>( 2524 Inst, Before)); 2525 } 2526 2527 TypePromotionTransaction::ConstRestorationPt 2528 TypePromotionTransaction::getRestorationPoint() const { 2529 return !Actions.empty() ? Actions.back().get() : nullptr; 2530 } 2531 2532 void TypePromotionTransaction::commit() { 2533 for (CommitPt It = Actions.begin(), EndIt = Actions.end(); It != EndIt; 2534 ++It) 2535 (*It)->commit(); 2536 Actions.clear(); 2537 } 2538 2539 void TypePromotionTransaction::rollback( 2540 TypePromotionTransaction::ConstRestorationPt Point) { 2541 while (!Actions.empty() && Point != Actions.back().get()) { 2542 std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val(); 2543 Curr->undo(); 2544 } 2545 } 2546 2547 namespace { 2548 2549 /// A helper class for matching addressing modes. 2550 /// 2551 /// This encapsulates the logic for matching the target-legal addressing modes. 2552 class AddressingModeMatcher { 2553 SmallVectorImpl<Instruction*> &AddrModeInsts; 2554 const TargetLowering &TLI; 2555 const TargetRegisterInfo &TRI; 2556 const DataLayout &DL; 2557 2558 /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and 2559 /// the memory instruction that we're computing this address for. 2560 Type *AccessTy; 2561 unsigned AddrSpace; 2562 Instruction *MemoryInst; 2563 2564 /// This is the addressing mode that we're building up. This is 2565 /// part of the return value of this addressing mode matching stuff. 2566 ExtAddrMode &AddrMode; 2567 2568 /// The instructions inserted by other CodeGenPrepare optimizations. 2569 const SetOfInstrs &InsertedInsts; 2570 2571 /// A map from the instructions to their type before promotion. 2572 InstrToOrigTy &PromotedInsts; 2573 2574 /// The ongoing transaction where every action should be registered. 2575 TypePromotionTransaction &TPT; 2576 2577 // A GEP which has too large offset to be folded into the addressing mode. 2578 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP; 2579 2580 /// This is set to true when we should not do profitability checks. 2581 /// When true, IsProfitableToFoldIntoAddressingMode always returns true. 2582 bool IgnoreProfitability; 2583 2584 AddressingModeMatcher( 2585 SmallVectorImpl<Instruction *> &AMI, const TargetLowering &TLI, 2586 const TargetRegisterInfo &TRI, Type *AT, unsigned AS, Instruction *MI, 2587 ExtAddrMode &AM, const SetOfInstrs &InsertedInsts, 2588 InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT, 2589 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP) 2590 : AddrModeInsts(AMI), TLI(TLI), TRI(TRI), 2591 DL(MI->getModule()->getDataLayout()), AccessTy(AT), AddrSpace(AS), 2592 MemoryInst(MI), AddrMode(AM), InsertedInsts(InsertedInsts), 2593 PromotedInsts(PromotedInsts), TPT(TPT), LargeOffsetGEP(LargeOffsetGEP) { 2594 IgnoreProfitability = false; 2595 } 2596 2597 public: 2598 /// Find the maximal addressing mode that a load/store of V can fold, 2599 /// give an access type of AccessTy. This returns a list of involved 2600 /// instructions in AddrModeInsts. 2601 /// \p InsertedInsts The instructions inserted by other CodeGenPrepare 2602 /// optimizations. 2603 /// \p PromotedInsts maps the instructions to their type before promotion. 2604 /// \p The ongoing transaction where every action should be registered. 2605 static ExtAddrMode 2606 Match(Value *V, Type *AccessTy, unsigned AS, Instruction *MemoryInst, 2607 SmallVectorImpl<Instruction *> &AddrModeInsts, 2608 const TargetLowering &TLI, const TargetRegisterInfo &TRI, 2609 const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts, 2610 TypePromotionTransaction &TPT, 2611 std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP) { 2612 ExtAddrMode Result; 2613 2614 bool Success = AddressingModeMatcher(AddrModeInsts, TLI, TRI, AccessTy, AS, 2615 MemoryInst, Result, InsertedInsts, 2616 PromotedInsts, TPT, LargeOffsetGEP) 2617 .matchAddr(V, 0); 2618 (void)Success; assert(Success && "Couldn't select *anything*?"); 2619 return Result; 2620 } 2621 2622 private: 2623 bool matchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth); 2624 bool matchAddr(Value *Addr, unsigned Depth); 2625 bool matchOperationAddr(User *AddrInst, unsigned Opcode, unsigned Depth, 2626 bool *MovedAway = nullptr); 2627 bool isProfitableToFoldIntoAddressingMode(Instruction *I, 2628 ExtAddrMode &AMBefore, 2629 ExtAddrMode &AMAfter); 2630 bool valueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2); 2631 bool isPromotionProfitable(unsigned NewCost, unsigned OldCost, 2632 Value *PromotedOperand) const; 2633 }; 2634 2635 /// Keep track of simplification of Phi nodes. 2636 /// Accept the set of all phi nodes and erase phi node from this set 2637 /// if it is simplified. 2638 class SimplificationTracker { 2639 DenseMap<Value *, Value *> Storage; 2640 const SimplifyQuery &SQ; 2641 // Tracks newly created Phi nodes. We use a SetVector to get deterministic 2642 // order when iterating over the set in MatchPhiSet. 2643 SmallSetVector<PHINode *, 32> AllPhiNodes; 2644 // Tracks newly created Select nodes. 2645 SmallPtrSet<SelectInst *, 32> AllSelectNodes; 2646 2647 public: 2648 SimplificationTracker(const SimplifyQuery &sq) 2649 : SQ(sq) {} 2650 2651 Value *Get(Value *V) { 2652 do { 2653 auto SV = Storage.find(V); 2654 if (SV == Storage.end()) 2655 return V; 2656 V = SV->second; 2657 } while (true); 2658 } 2659 2660 Value *Simplify(Value *Val) { 2661 SmallVector<Value *, 32> WorkList; 2662 SmallPtrSet<Value *, 32> Visited; 2663 WorkList.push_back(Val); 2664 while (!WorkList.empty()) { 2665 auto P = WorkList.pop_back_val(); 2666 if (!Visited.insert(P).second) 2667 continue; 2668 if (auto *PI = dyn_cast<Instruction>(P)) 2669 if (Value *V = SimplifyInstruction(cast<Instruction>(PI), SQ)) { 2670 for (auto *U : PI->users()) 2671 WorkList.push_back(cast<Value>(U)); 2672 Put(PI, V); 2673 PI->replaceAllUsesWith(V); 2674 if (auto *PHI = dyn_cast<PHINode>(PI)) 2675 AllPhiNodes.remove(PHI); 2676 if (auto *Select = dyn_cast<SelectInst>(PI)) 2677 AllSelectNodes.erase(Select); 2678 PI->eraseFromParent(); 2679 } 2680 } 2681 return Get(Val); 2682 } 2683 2684 void Put(Value *From, Value *To) { 2685 Storage.insert({ From, To }); 2686 } 2687 2688 void ReplacePhi(PHINode *From, PHINode *To) { 2689 Value* OldReplacement = Get(From); 2690 while (OldReplacement != From) { 2691 From = To; 2692 To = dyn_cast<PHINode>(OldReplacement); 2693 OldReplacement = Get(From); 2694 } 2695 assert(Get(To) == To && "Replacement PHI node is already replaced."); 2696 Put(From, To); 2697 From->replaceAllUsesWith(To); 2698 AllPhiNodes.remove(From); 2699 From->eraseFromParent(); 2700 } 2701 2702 SmallSetVector<PHINode *, 32>& newPhiNodes() { return AllPhiNodes; } 2703 2704 void insertNewPhi(PHINode *PN) { AllPhiNodes.insert(PN); } 2705 2706 void insertNewSelect(SelectInst *SI) { AllSelectNodes.insert(SI); } 2707 2708 unsigned countNewPhiNodes() const { return AllPhiNodes.size(); } 2709 2710 unsigned countNewSelectNodes() const { return AllSelectNodes.size(); } 2711 2712 void destroyNewNodes(Type *CommonType) { 2713 // For safe erasing, replace the uses with dummy value first. 2714 auto Dummy = UndefValue::get(CommonType); 2715 for (auto I : AllPhiNodes) { 2716 I->replaceAllUsesWith(Dummy); 2717 I->eraseFromParent(); 2718 } 2719 AllPhiNodes.clear(); 2720 for (auto I : AllSelectNodes) { 2721 I->replaceAllUsesWith(Dummy); 2722 I->eraseFromParent(); 2723 } 2724 AllSelectNodes.clear(); 2725 } 2726 }; 2727 2728 /// A helper class for combining addressing modes. 2729 class AddressingModeCombiner { 2730 typedef std::pair<Value *, BasicBlock *> ValueInBB; 2731 typedef DenseMap<ValueInBB, Value *> FoldAddrToValueMapping; 2732 typedef std::pair<PHINode *, PHINode *> PHIPair; 2733 2734 private: 2735 /// The addressing modes we've collected. 2736 SmallVector<ExtAddrMode, 16> AddrModes; 2737 2738 /// The field in which the AddrModes differ, when we have more than one. 2739 ExtAddrMode::FieldName DifferentField = ExtAddrMode::NoField; 2740 2741 /// Are the AddrModes that we have all just equal to their original values? 2742 bool AllAddrModesTrivial = true; 2743 2744 /// Common Type for all different fields in addressing modes. 2745 Type *CommonType; 2746 2747 /// SimplifyQuery for simplifyInstruction utility. 2748 const SimplifyQuery &SQ; 2749 2750 /// Original Address. 2751 ValueInBB Original; 2752 2753 public: 2754 AddressingModeCombiner(const SimplifyQuery &_SQ, ValueInBB OriginalValue) 2755 : CommonType(nullptr), SQ(_SQ), Original(OriginalValue) {} 2756 2757 /// Get the combined AddrMode 2758 const ExtAddrMode &getAddrMode() const { 2759 return AddrModes[0]; 2760 } 2761 2762 /// Add a new AddrMode if it's compatible with the AddrModes we already 2763 /// have. 2764 /// \return True iff we succeeded in doing so. 2765 bool addNewAddrMode(ExtAddrMode &NewAddrMode) { 2766 // Take note of if we have any non-trivial AddrModes, as we need to detect 2767 // when all AddrModes are trivial as then we would introduce a phi or select 2768 // which just duplicates what's already there. 2769 AllAddrModesTrivial = AllAddrModesTrivial && NewAddrMode.isTrivial(); 2770 2771 // If this is the first addrmode then everything is fine. 2772 if (AddrModes.empty()) { 2773 AddrModes.emplace_back(NewAddrMode); 2774 return true; 2775 } 2776 2777 // Figure out how different this is from the other address modes, which we 2778 // can do just by comparing against the first one given that we only care 2779 // about the cumulative difference. 2780 ExtAddrMode::FieldName ThisDifferentField = 2781 AddrModes[0].compare(NewAddrMode); 2782 if (DifferentField == ExtAddrMode::NoField) 2783 DifferentField = ThisDifferentField; 2784 else if (DifferentField != ThisDifferentField) 2785 DifferentField = ExtAddrMode::MultipleFields; 2786 2787 // If NewAddrMode differs in more than one dimension we cannot handle it. 2788 bool CanHandle = DifferentField != ExtAddrMode::MultipleFields; 2789 2790 // If Scale Field is different then we reject. 2791 CanHandle = CanHandle && DifferentField != ExtAddrMode::ScaleField; 2792 2793 // We also must reject the case when base offset is different and 2794 // scale reg is not null, we cannot handle this case due to merge of 2795 // different offsets will be used as ScaleReg. 2796 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseOffsField || 2797 !NewAddrMode.ScaledReg); 2798 2799 // We also must reject the case when GV is different and BaseReg installed 2800 // due to we want to use base reg as a merge of GV values. 2801 CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseGVField || 2802 !NewAddrMode.HasBaseReg); 2803 2804 // Even if NewAddMode is the same we still need to collect it due to 2805 // original value is different. And later we will need all original values 2806 // as anchors during finding the common Phi node. 2807 if (CanHandle) 2808 AddrModes.emplace_back(NewAddrMode); 2809 else 2810 AddrModes.clear(); 2811 2812 return CanHandle; 2813 } 2814 2815 /// Combine the addressing modes we've collected into a single 2816 /// addressing mode. 2817 /// \return True iff we successfully combined them or we only had one so 2818 /// didn't need to combine them anyway. 2819 bool combineAddrModes() { 2820 // If we have no AddrModes then they can't be combined. 2821 if (AddrModes.size() == 0) 2822 return false; 2823 2824 // A single AddrMode can trivially be combined. 2825 if (AddrModes.size() == 1 || DifferentField == ExtAddrMode::NoField) 2826 return true; 2827 2828 // If the AddrModes we collected are all just equal to the value they are 2829 // derived from then combining them wouldn't do anything useful. 2830 if (AllAddrModesTrivial) 2831 return false; 2832 2833 if (!addrModeCombiningAllowed()) 2834 return false; 2835 2836 // Build a map between <original value, basic block where we saw it> to 2837 // value of base register. 2838 // Bail out if there is no common type. 2839 FoldAddrToValueMapping Map; 2840 if (!initializeMap(Map)) 2841 return false; 2842 2843 Value *CommonValue = findCommon(Map); 2844 if (CommonValue) 2845 AddrModes[0].SetCombinedField(DifferentField, CommonValue, AddrModes); 2846 return CommonValue != nullptr; 2847 } 2848 2849 private: 2850 /// Initialize Map with anchor values. For address seen in some BB 2851 /// we set the value of different field saw in this address. 2852 /// If address is not an instruction than basic block is set to null. 2853 /// At the same time we find a common type for different field we will 2854 /// use to create new Phi/Select nodes. Keep it in CommonType field. 2855 /// Return false if there is no common type found. 2856 bool initializeMap(FoldAddrToValueMapping &Map) { 2857 // Keep track of keys where the value is null. We will need to replace it 2858 // with constant null when we know the common type. 2859 SmallVector<ValueInBB, 2> NullValue; 2860 Type *IntPtrTy = SQ.DL.getIntPtrType(AddrModes[0].OriginalValue->getType()); 2861 for (auto &AM : AddrModes) { 2862 BasicBlock *BB = nullptr; 2863 if (Instruction *I = dyn_cast<Instruction>(AM.OriginalValue)) 2864 BB = I->getParent(); 2865 2866 Value *DV = AM.GetFieldAsValue(DifferentField, IntPtrTy); 2867 if (DV) { 2868 auto *Type = DV->getType(); 2869 if (CommonType && CommonType != Type) 2870 return false; 2871 CommonType = Type; 2872 Map[{ AM.OriginalValue, BB }] = DV; 2873 } else { 2874 NullValue.push_back({ AM.OriginalValue, BB }); 2875 } 2876 } 2877 assert(CommonType && "At least one non-null value must be!"); 2878 for (auto VIBB : NullValue) 2879 Map[VIBB] = Constant::getNullValue(CommonType); 2880 return true; 2881 } 2882 2883 /// We have mapping between value A and basic block where value A 2884 /// seen to other value B where B was a field in addressing mode represented 2885 /// by A. Also we have an original value C representing an address in some 2886 /// basic block. Traversing from C through phi and selects we ended up with 2887 /// A's in a map. This utility function tries to find a value V which is a 2888 /// field in addressing mode C and traversing through phi nodes and selects 2889 /// we will end up in corresponded values B in a map. 2890 /// The utility will create a new Phi/Selects if needed. 2891 // The simple example looks as follows: 2892 // BB1: 2893 // p1 = b1 + 40 2894 // br cond BB2, BB3 2895 // BB2: 2896 // p2 = b2 + 40 2897 // br BB3 2898 // BB3: 2899 // p = phi [p1, BB1], [p2, BB2] 2900 // v = load p 2901 // Map is 2902 // <p1, BB1> -> b1 2903 // <p2, BB2> -> b2 2904 // Request is 2905 // <p, BB3> -> ? 2906 // The function tries to find or build phi [b1, BB1], [b2, BB2] in BB3 2907 Value *findCommon(FoldAddrToValueMapping &Map) { 2908 // Tracks the simplification of newly created phi nodes. The reason we use 2909 // this mapping is because we will add new created Phi nodes in AddrToBase. 2910 // Simplification of Phi nodes is recursive, so some Phi node may 2911 // be simplified after we added it to AddrToBase. 2912 // Using this mapping we can find the current value in AddrToBase. 2913 SimplificationTracker ST(SQ); 2914 2915 // First step, DFS to create PHI nodes for all intermediate blocks. 2916 // Also fill traverse order for the second step. 2917 SmallVector<ValueInBB, 32> TraverseOrder; 2918 InsertPlaceholders(Map, TraverseOrder, ST); 2919 2920 // Second Step, fill new nodes by merged values and simplify if possible. 2921 FillPlaceholders(Map, TraverseOrder, ST); 2922 2923 if (!AddrSinkNewSelects && ST.countNewSelectNodes() > 0) { 2924 ST.destroyNewNodes(CommonType); 2925 return nullptr; 2926 } 2927 2928 // Now we'd like to match New Phi nodes to existed ones. 2929 unsigned PhiNotMatchedCount = 0; 2930 if (!MatchPhiSet(ST, AddrSinkNewPhis, PhiNotMatchedCount)) { 2931 ST.destroyNewNodes(CommonType); 2932 return nullptr; 2933 } 2934 2935 auto *Result = ST.Get(Map.find(Original)->second); 2936 if (Result) { 2937 NumMemoryInstsPhiCreated += ST.countNewPhiNodes() + PhiNotMatchedCount; 2938 NumMemoryInstsSelectCreated += ST.countNewSelectNodes(); 2939 } 2940 return Result; 2941 } 2942 2943 /// Try to match PHI node to Candidate. 2944 /// Matcher tracks the matched Phi nodes. 2945 bool MatchPhiNode(PHINode *PHI, PHINode *Candidate, 2946 SmallSetVector<PHIPair, 8> &Matcher, 2947 SmallSetVector<PHINode *, 32> &PhiNodesToMatch) { 2948 SmallVector<PHIPair, 8> WorkList; 2949 Matcher.insert({ PHI, Candidate }); 2950 WorkList.push_back({ PHI, Candidate }); 2951 SmallSet<PHIPair, 8> Visited; 2952 while (!WorkList.empty()) { 2953 auto Item = WorkList.pop_back_val(); 2954 if (!Visited.insert(Item).second) 2955 continue; 2956 // We iterate over all incoming values to Phi to compare them. 2957 // If values are different and both of them Phi and the first one is a 2958 // Phi we added (subject to match) and both of them is in the same basic 2959 // block then we can match our pair if values match. So we state that 2960 // these values match and add it to work list to verify that. 2961 for (auto B : Item.first->blocks()) { 2962 Value *FirstValue = Item.first->getIncomingValueForBlock(B); 2963 Value *SecondValue = Item.second->getIncomingValueForBlock(B); 2964 if (FirstValue == SecondValue) 2965 continue; 2966 2967 PHINode *FirstPhi = dyn_cast<PHINode>(FirstValue); 2968 PHINode *SecondPhi = dyn_cast<PHINode>(SecondValue); 2969 2970 // One of them is not Phi or 2971 // The first one is not Phi node from the set we'd like to match or 2972 // Phi nodes from different basic blocks then 2973 // we will not be able to match. 2974 if (!FirstPhi || !SecondPhi || !PhiNodesToMatch.count(FirstPhi) || 2975 FirstPhi->getParent() != SecondPhi->getParent()) 2976 return false; 2977 2978 // If we already matched them then continue. 2979 if (Matcher.count({ FirstPhi, SecondPhi })) 2980 continue; 2981 // So the values are different and does not match. So we need them to 2982 // match. 2983 Matcher.insert({ FirstPhi, SecondPhi }); 2984 // But me must check it. 2985 WorkList.push_back({ FirstPhi, SecondPhi }); 2986 } 2987 } 2988 return true; 2989 } 2990 2991 /// For the given set of PHI nodes (in the SimplificationTracker) try 2992 /// to find their equivalents. 2993 /// Returns false if this matching fails and creation of new Phi is disabled. 2994 bool MatchPhiSet(SimplificationTracker &ST, bool AllowNewPhiNodes, 2995 unsigned &PhiNotMatchedCount) { 2996 // Use a SetVector for Matched to make sure we do replacements (ReplacePhi) 2997 // in a deterministic order below. 2998 SmallSetVector<PHIPair, 8> Matched; 2999 SmallPtrSet<PHINode *, 8> WillNotMatch; 3000 SmallSetVector<PHINode *, 32> &PhiNodesToMatch = ST.newPhiNodes(); 3001 while (PhiNodesToMatch.size()) { 3002 PHINode *PHI = *PhiNodesToMatch.begin(); 3003 3004 // Add us, if no Phi nodes in the basic block we do not match. 3005 WillNotMatch.clear(); 3006 WillNotMatch.insert(PHI); 3007 3008 // Traverse all Phis until we found equivalent or fail to do that. 3009 bool IsMatched = false; 3010 for (auto &P : PHI->getParent()->phis()) { 3011 if (&P == PHI) 3012 continue; 3013 if ((IsMatched = MatchPhiNode(PHI, &P, Matched, PhiNodesToMatch))) 3014 break; 3015 // If it does not match, collect all Phi nodes from matcher. 3016 // if we end up with no match, them all these Phi nodes will not match 3017 // later. 3018 for (auto M : Matched) 3019 WillNotMatch.insert(M.first); 3020 Matched.clear(); 3021 } 3022 if (IsMatched) { 3023 // Replace all matched values and erase them. 3024 for (auto MV : Matched) 3025 ST.ReplacePhi(MV.first, MV.second); 3026 Matched.clear(); 3027 continue; 3028 } 3029 // If we are not allowed to create new nodes then bail out. 3030 if (!AllowNewPhiNodes) 3031 return false; 3032 // Just remove all seen values in matcher. They will not match anything. 3033 PhiNotMatchedCount += WillNotMatch.size(); 3034 for (auto *P : WillNotMatch) 3035 PhiNodesToMatch.remove(P); 3036 } 3037 return true; 3038 } 3039 /// Fill the placeholder with values from predecessors and simplify it. 3040 void FillPlaceholders(FoldAddrToValueMapping &Map, 3041 SmallVectorImpl<ValueInBB> &TraverseOrder, 3042 SimplificationTracker &ST) { 3043 while (!TraverseOrder.empty()) { 3044 auto Current = TraverseOrder.pop_back_val(); 3045 assert(Map.find(Current) != Map.end() && "No node to fill!!!"); 3046 Value *CurrentValue = Current.first; 3047 BasicBlock *CurrentBlock = Current.second; 3048 Value *V = Map[Current]; 3049 3050 if (SelectInst *Select = dyn_cast<SelectInst>(V)) { 3051 // CurrentValue also must be Select. 3052 auto *CurrentSelect = cast<SelectInst>(CurrentValue); 3053 auto *TrueValue = CurrentSelect->getTrueValue(); 3054 ValueInBB TrueItem = { TrueValue, isa<Instruction>(TrueValue) 3055 ? CurrentBlock 3056 : nullptr }; 3057 assert(Map.find(TrueItem) != Map.end() && "No True Value!"); 3058 Select->setTrueValue(ST.Get(Map[TrueItem])); 3059 auto *FalseValue = CurrentSelect->getFalseValue(); 3060 ValueInBB FalseItem = { FalseValue, isa<Instruction>(FalseValue) 3061 ? CurrentBlock 3062 : nullptr }; 3063 assert(Map.find(FalseItem) != Map.end() && "No False Value!"); 3064 Select->setFalseValue(ST.Get(Map[FalseItem])); 3065 } else { 3066 // Must be a Phi node then. 3067 PHINode *PHI = cast<PHINode>(V); 3068 // Fill the Phi node with values from predecessors. 3069 bool IsDefinedInThisBB = 3070 cast<Instruction>(CurrentValue)->getParent() == CurrentBlock; 3071 auto *CurrentPhi = dyn_cast<PHINode>(CurrentValue); 3072 for (auto B : predecessors(CurrentBlock)) { 3073 Value *PV = IsDefinedInThisBB 3074 ? CurrentPhi->getIncomingValueForBlock(B) 3075 : CurrentValue; 3076 ValueInBB item = { PV, isa<Instruction>(PV) ? B : nullptr }; 3077 assert(Map.find(item) != Map.end() && "No predecessor Value!"); 3078 PHI->addIncoming(ST.Get(Map[item]), B); 3079 } 3080 } 3081 // Simplify if possible. 3082 Map[Current] = ST.Simplify(V); 3083 } 3084 } 3085 3086 /// Starting from value recursively iterates over predecessors up to known 3087 /// ending values represented in a map. For each traversed block inserts 3088 /// a placeholder Phi or Select. 3089 /// Reports all new created Phi/Select nodes by adding them to set. 3090 /// Also reports and order in what basic blocks have been traversed. 3091 void InsertPlaceholders(FoldAddrToValueMapping &Map, 3092 SmallVectorImpl<ValueInBB> &TraverseOrder, 3093 SimplificationTracker &ST) { 3094 SmallVector<ValueInBB, 32> Worklist; 3095 assert((isa<PHINode>(Original.first) || isa<SelectInst>(Original.first)) && 3096 "Address must be a Phi or Select node"); 3097 auto *Dummy = UndefValue::get(CommonType); 3098 Worklist.push_back(Original); 3099 while (!Worklist.empty()) { 3100 auto Current = Worklist.pop_back_val(); 3101 // If value is not an instruction it is something global, constant, 3102 // parameter and we can say that this value is observable in any block. 3103 // Set block to null to denote it. 3104 // Also please take into account that it is how we build anchors. 3105 if (!isa<Instruction>(Current.first)) 3106 Current.second = nullptr; 3107 // if it is already visited or it is an ending value then skip it. 3108 if (Map.find(Current) != Map.end()) 3109 continue; 3110 TraverseOrder.push_back(Current); 3111 3112 Value *CurrentValue = Current.first; 3113 BasicBlock *CurrentBlock = Current.second; 3114 // CurrentValue must be a Phi node or select. All others must be covered 3115 // by anchors. 3116 Instruction *CurrentI = cast<Instruction>(CurrentValue); 3117 bool IsDefinedInThisBB = CurrentI->getParent() == CurrentBlock; 3118 3119 unsigned PredCount = pred_size(CurrentBlock); 3120 // if Current Value is not defined in this basic block we are interested 3121 // in values in predecessors. 3122 if (!IsDefinedInThisBB) { 3123 assert(PredCount && "Unreachable block?!"); 3124 PHINode *PHI = PHINode::Create(CommonType, PredCount, "sunk_phi", 3125 &CurrentBlock->front()); 3126 Map[Current] = PHI; 3127 ST.insertNewPhi(PHI); 3128 // Add all predecessors in work list. 3129 for (auto B : predecessors(CurrentBlock)) 3130 Worklist.push_back({ CurrentValue, B }); 3131 continue; 3132 } 3133 // Value is defined in this basic block. 3134 if (SelectInst *OrigSelect = dyn_cast<SelectInst>(CurrentI)) { 3135 // Is it OK to get metadata from OrigSelect?! 3136 // Create a Select placeholder with dummy value. 3137 SelectInst *Select = 3138 SelectInst::Create(OrigSelect->getCondition(), Dummy, Dummy, 3139 OrigSelect->getName(), OrigSelect, OrigSelect); 3140 Map[Current] = Select; 3141 ST.insertNewSelect(Select); 3142 // We are interested in True and False value in this basic block. 3143 Worklist.push_back({ OrigSelect->getTrueValue(), CurrentBlock }); 3144 Worklist.push_back({ OrigSelect->getFalseValue(), CurrentBlock }); 3145 } else { 3146 // It must be a Phi node then. 3147 auto *CurrentPhi = cast<PHINode>(CurrentI); 3148 // Create new Phi node for merge of bases. 3149 assert(PredCount && "Unreachable block?!"); 3150 PHINode *PHI = PHINode::Create(CommonType, PredCount, "sunk_phi", 3151 &CurrentBlock->front()); 3152 Map[Current] = PHI; 3153 ST.insertNewPhi(PHI); 3154 3155 // Add all predecessors in work list. 3156 for (auto B : predecessors(CurrentBlock)) 3157 Worklist.push_back({ CurrentPhi->getIncomingValueForBlock(B), B }); 3158 } 3159 } 3160 } 3161 3162 bool addrModeCombiningAllowed() { 3163 if (DisableComplexAddrModes) 3164 return false; 3165 switch (DifferentField) { 3166 default: 3167 return false; 3168 case ExtAddrMode::BaseRegField: 3169 return AddrSinkCombineBaseReg; 3170 case ExtAddrMode::BaseGVField: 3171 return AddrSinkCombineBaseGV; 3172 case ExtAddrMode::BaseOffsField: 3173 return AddrSinkCombineBaseOffs; 3174 case ExtAddrMode::ScaledRegField: 3175 return AddrSinkCombineScaledReg; 3176 } 3177 } 3178 }; 3179 } // end anonymous namespace 3180 3181 /// Try adding ScaleReg*Scale to the current addressing mode. 3182 /// Return true and update AddrMode if this addr mode is legal for the target, 3183 /// false if not. 3184 bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale, 3185 unsigned Depth) { 3186 // If Scale is 1, then this is the same as adding ScaleReg to the addressing 3187 // mode. Just process that directly. 3188 if (Scale == 1) 3189 return matchAddr(ScaleReg, Depth); 3190 3191 // If the scale is 0, it takes nothing to add this. 3192 if (Scale == 0) 3193 return true; 3194 3195 // If we already have a scale of this value, we can add to it, otherwise, we 3196 // need an available scale field. 3197 if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg) 3198 return false; 3199 3200 ExtAddrMode TestAddrMode = AddrMode; 3201 3202 // Add scale to turn X*4+X*3 -> X*7. This could also do things like 3203 // [A+B + A*7] -> [B+A*8]. 3204 TestAddrMode.Scale += Scale; 3205 TestAddrMode.ScaledReg = ScaleReg; 3206 3207 // If the new address isn't legal, bail out. 3208 if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) 3209 return false; 3210 3211 // It was legal, so commit it. 3212 AddrMode = TestAddrMode; 3213 3214 // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now 3215 // to see if ScaleReg is actually X+C. If so, we can turn this into adding 3216 // X*Scale + C*Scale to addr mode. 3217 ConstantInt *CI = nullptr; Value *AddLHS = nullptr; 3218 if (isa<Instruction>(ScaleReg) && // not a constant expr. 3219 match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) { 3220 TestAddrMode.ScaledReg = AddLHS; 3221 TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale; 3222 3223 // If this addressing mode is legal, commit it and remember that we folded 3224 // this instruction. 3225 if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) { 3226 AddrModeInsts.push_back(cast<Instruction>(ScaleReg)); 3227 AddrMode = TestAddrMode; 3228 return true; 3229 } 3230 } 3231 3232 // Otherwise, not (x+c)*scale, just return what we have. 3233 return true; 3234 } 3235 3236 /// This is a little filter, which returns true if an addressing computation 3237 /// involving I might be folded into a load/store accessing it. 3238 /// This doesn't need to be perfect, but needs to accept at least 3239 /// the set of instructions that MatchOperationAddr can. 3240 static bool MightBeFoldableInst(Instruction *I) { 3241 switch (I->getOpcode()) { 3242 case Instruction::BitCast: 3243 case Instruction::AddrSpaceCast: 3244 // Don't touch identity bitcasts. 3245 if (I->getType() == I->getOperand(0)->getType()) 3246 return false; 3247 return I->getType()->isIntOrPtrTy(); 3248 case Instruction::PtrToInt: 3249 // PtrToInt is always a noop, as we know that the int type is pointer sized. 3250 return true; 3251 case Instruction::IntToPtr: 3252 // We know the input is intptr_t, so this is foldable. 3253 return true; 3254 case Instruction::Add: 3255 return true; 3256 case Instruction::Mul: 3257 case Instruction::Shl: 3258 // Can only handle X*C and X << C. 3259 return isa<ConstantInt>(I->getOperand(1)); 3260 case Instruction::GetElementPtr: 3261 return true; 3262 default: 3263 return false; 3264 } 3265 } 3266 3267 /// Check whether or not \p Val is a legal instruction for \p TLI. 3268 /// \note \p Val is assumed to be the product of some type promotion. 3269 /// Therefore if \p Val has an undefined state in \p TLI, this is assumed 3270 /// to be legal, as the non-promoted value would have had the same state. 3271 static bool isPromotedInstructionLegal(const TargetLowering &TLI, 3272 const DataLayout &DL, Value *Val) { 3273 Instruction *PromotedInst = dyn_cast<Instruction>(Val); 3274 if (!PromotedInst) 3275 return false; 3276 int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode()); 3277 // If the ISDOpcode is undefined, it was undefined before the promotion. 3278 if (!ISDOpcode) 3279 return true; 3280 // Otherwise, check if the promoted instruction is legal or not. 3281 return TLI.isOperationLegalOrCustom( 3282 ISDOpcode, TLI.getValueType(DL, PromotedInst->getType())); 3283 } 3284 3285 namespace { 3286 3287 /// Hepler class to perform type promotion. 3288 class TypePromotionHelper { 3289 /// Utility function to add a promoted instruction \p ExtOpnd to 3290 /// \p PromotedInsts and record the type of extension we have seen. 3291 static void addPromotedInst(InstrToOrigTy &PromotedInsts, 3292 Instruction *ExtOpnd, 3293 bool IsSExt) { 3294 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension; 3295 InstrToOrigTy::iterator It = PromotedInsts.find(ExtOpnd); 3296 if (It != PromotedInsts.end()) { 3297 // If the new extension is same as original, the information in 3298 // PromotedInsts[ExtOpnd] is still correct. 3299 if (It->second.getInt() == ExtTy) 3300 return; 3301 3302 // Now the new extension is different from old extension, we make 3303 // the type information invalid by setting extension type to 3304 // BothExtension. 3305 ExtTy = BothExtension; 3306 } 3307 PromotedInsts[ExtOpnd] = TypeIsSExt(ExtOpnd->getType(), ExtTy); 3308 } 3309 3310 /// Utility function to query the original type of instruction \p Opnd 3311 /// with a matched extension type. If the extension doesn't match, we 3312 /// cannot use the information we had on the original type. 3313 /// BothExtension doesn't match any extension type. 3314 static const Type *getOrigType(const InstrToOrigTy &PromotedInsts, 3315 Instruction *Opnd, 3316 bool IsSExt) { 3317 ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension; 3318 InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd); 3319 if (It != PromotedInsts.end() && It->second.getInt() == ExtTy) 3320 return It->second.getPointer(); 3321 return nullptr; 3322 } 3323 3324 /// Utility function to check whether or not a sign or zero extension 3325 /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by 3326 /// either using the operands of \p Inst or promoting \p Inst. 3327 /// The type of the extension is defined by \p IsSExt. 3328 /// In other words, check if: 3329 /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType. 3330 /// #1 Promotion applies: 3331 /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...). 3332 /// #2 Operand reuses: 3333 /// ext opnd1 to ConsideredExtType. 3334 /// \p PromotedInsts maps the instructions to their type before promotion. 3335 static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType, 3336 const InstrToOrigTy &PromotedInsts, bool IsSExt); 3337 3338 /// Utility function to determine if \p OpIdx should be promoted when 3339 /// promoting \p Inst. 3340 static bool shouldExtOperand(const Instruction *Inst, int OpIdx) { 3341 return !(isa<SelectInst>(Inst) && OpIdx == 0); 3342 } 3343 3344 /// Utility function to promote the operand of \p Ext when this 3345 /// operand is a promotable trunc or sext or zext. 3346 /// \p PromotedInsts maps the instructions to their type before promotion. 3347 /// \p CreatedInstsCost[out] contains the cost of all instructions 3348 /// created to promote the operand of Ext. 3349 /// Newly added extensions are inserted in \p Exts. 3350 /// Newly added truncates are inserted in \p Truncs. 3351 /// Should never be called directly. 3352 /// \return The promoted value which is used instead of Ext. 3353 static Value *promoteOperandForTruncAndAnyExt( 3354 Instruction *Ext, TypePromotionTransaction &TPT, 3355 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 3356 SmallVectorImpl<Instruction *> *Exts, 3357 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI); 3358 3359 /// Utility function to promote the operand of \p Ext when this 3360 /// operand is promotable and is not a supported trunc or sext. 3361 /// \p PromotedInsts maps the instructions to their type before promotion. 3362 /// \p CreatedInstsCost[out] contains the cost of all the instructions 3363 /// created to promote the operand of Ext. 3364 /// Newly added extensions are inserted in \p Exts. 3365 /// Newly added truncates are inserted in \p Truncs. 3366 /// Should never be called directly. 3367 /// \return The promoted value which is used instead of Ext. 3368 static Value *promoteOperandForOther(Instruction *Ext, 3369 TypePromotionTransaction &TPT, 3370 InstrToOrigTy &PromotedInsts, 3371 unsigned &CreatedInstsCost, 3372 SmallVectorImpl<Instruction *> *Exts, 3373 SmallVectorImpl<Instruction *> *Truncs, 3374 const TargetLowering &TLI, bool IsSExt); 3375 3376 /// \see promoteOperandForOther. 3377 static Value *signExtendOperandForOther( 3378 Instruction *Ext, TypePromotionTransaction &TPT, 3379 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 3380 SmallVectorImpl<Instruction *> *Exts, 3381 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) { 3382 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost, 3383 Exts, Truncs, TLI, true); 3384 } 3385 3386 /// \see promoteOperandForOther. 3387 static Value *zeroExtendOperandForOther( 3388 Instruction *Ext, TypePromotionTransaction &TPT, 3389 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 3390 SmallVectorImpl<Instruction *> *Exts, 3391 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) { 3392 return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost, 3393 Exts, Truncs, TLI, false); 3394 } 3395 3396 public: 3397 /// Type for the utility function that promotes the operand of Ext. 3398 using Action = Value *(*)(Instruction *Ext, TypePromotionTransaction &TPT, 3399 InstrToOrigTy &PromotedInsts, 3400 unsigned &CreatedInstsCost, 3401 SmallVectorImpl<Instruction *> *Exts, 3402 SmallVectorImpl<Instruction *> *Truncs, 3403 const TargetLowering &TLI); 3404 3405 /// Given a sign/zero extend instruction \p Ext, return the appropriate 3406 /// action to promote the operand of \p Ext instead of using Ext. 3407 /// \return NULL if no promotable action is possible with the current 3408 /// sign extension. 3409 /// \p InsertedInsts keeps track of all the instructions inserted by the 3410 /// other CodeGenPrepare optimizations. This information is important 3411 /// because we do not want to promote these instructions as CodeGenPrepare 3412 /// will reinsert them later. Thus creating an infinite loop: create/remove. 3413 /// \p PromotedInsts maps the instructions to their type before promotion. 3414 static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts, 3415 const TargetLowering &TLI, 3416 const InstrToOrigTy &PromotedInsts); 3417 }; 3418 3419 } // end anonymous namespace 3420 3421 bool TypePromotionHelper::canGetThrough(const Instruction *Inst, 3422 Type *ConsideredExtType, 3423 const InstrToOrigTy &PromotedInsts, 3424 bool IsSExt) { 3425 // The promotion helper does not know how to deal with vector types yet. 3426 // To be able to fix that, we would need to fix the places where we 3427 // statically extend, e.g., constants and such. 3428 if (Inst->getType()->isVectorTy()) 3429 return false; 3430 3431 // We can always get through zext. 3432 if (isa<ZExtInst>(Inst)) 3433 return true; 3434 3435 // sext(sext) is ok too. 3436 if (IsSExt && isa<SExtInst>(Inst)) 3437 return true; 3438 3439 // We can get through binary operator, if it is legal. In other words, the 3440 // binary operator must have a nuw or nsw flag. 3441 const BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst); 3442 if (BinOp && isa<OverflowingBinaryOperator>(BinOp) && 3443 ((!IsSExt && BinOp->hasNoUnsignedWrap()) || 3444 (IsSExt && BinOp->hasNoSignedWrap()))) 3445 return true; 3446 3447 // ext(and(opnd, cst)) --> and(ext(opnd), ext(cst)) 3448 if ((Inst->getOpcode() == Instruction::And || 3449 Inst->getOpcode() == Instruction::Or)) 3450 return true; 3451 3452 // ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst)) 3453 if (Inst->getOpcode() == Instruction::Xor) { 3454 const ConstantInt *Cst = dyn_cast<ConstantInt>(Inst->getOperand(1)); 3455 // Make sure it is not a NOT. 3456 if (Cst && !Cst->getValue().isAllOnesValue()) 3457 return true; 3458 } 3459 3460 // zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst)) 3461 // It may change a poisoned value into a regular value, like 3462 // zext i32 (shrl i8 %val, 12) --> shrl i32 (zext i8 %val), 12 3463 // poisoned value regular value 3464 // It should be OK since undef covers valid value. 3465 if (Inst->getOpcode() == Instruction::LShr && !IsSExt) 3466 return true; 3467 3468 // and(ext(shl(opnd, cst)), cst) --> and(shl(ext(opnd), ext(cst)), cst) 3469 // It may change a poisoned value into a regular value, like 3470 // zext i32 (shl i8 %val, 12) --> shl i32 (zext i8 %val), 12 3471 // poisoned value regular value 3472 // It should be OK since undef covers valid value. 3473 if (Inst->getOpcode() == Instruction::Shl && Inst->hasOneUse()) { 3474 const Instruction *ExtInst = 3475 dyn_cast<const Instruction>(*Inst->user_begin()); 3476 if (ExtInst->hasOneUse()) { 3477 const Instruction *AndInst = 3478 dyn_cast<const Instruction>(*ExtInst->user_begin()); 3479 if (AndInst && AndInst->getOpcode() == Instruction::And) { 3480 const ConstantInt *Cst = dyn_cast<ConstantInt>(AndInst->getOperand(1)); 3481 if (Cst && 3482 Cst->getValue().isIntN(Inst->getType()->getIntegerBitWidth())) 3483 return true; 3484 } 3485 } 3486 } 3487 3488 // Check if we can do the following simplification. 3489 // ext(trunc(opnd)) --> ext(opnd) 3490 if (!isa<TruncInst>(Inst)) 3491 return false; 3492 3493 Value *OpndVal = Inst->getOperand(0); 3494 // Check if we can use this operand in the extension. 3495 // If the type is larger than the result type of the extension, we cannot. 3496 if (!OpndVal->getType()->isIntegerTy() || 3497 OpndVal->getType()->getIntegerBitWidth() > 3498 ConsideredExtType->getIntegerBitWidth()) 3499 return false; 3500 3501 // If the operand of the truncate is not an instruction, we will not have 3502 // any information on the dropped bits. 3503 // (Actually we could for constant but it is not worth the extra logic). 3504 Instruction *Opnd = dyn_cast<Instruction>(OpndVal); 3505 if (!Opnd) 3506 return false; 3507 3508 // Check if the source of the type is narrow enough. 3509 // I.e., check that trunc just drops extended bits of the same kind of 3510 // the extension. 3511 // #1 get the type of the operand and check the kind of the extended bits. 3512 const Type *OpndType = getOrigType(PromotedInsts, Opnd, IsSExt); 3513 if (OpndType) 3514 ; 3515 else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd))) 3516 OpndType = Opnd->getOperand(0)->getType(); 3517 else 3518 return false; 3519 3520 // #2 check that the truncate just drops extended bits. 3521 return Inst->getType()->getIntegerBitWidth() >= 3522 OpndType->getIntegerBitWidth(); 3523 } 3524 3525 TypePromotionHelper::Action TypePromotionHelper::getAction( 3526 Instruction *Ext, const SetOfInstrs &InsertedInsts, 3527 const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) { 3528 assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) && 3529 "Unexpected instruction type"); 3530 Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0)); 3531 Type *ExtTy = Ext->getType(); 3532 bool IsSExt = isa<SExtInst>(Ext); 3533 // If the operand of the extension is not an instruction, we cannot 3534 // get through. 3535 // If it, check we can get through. 3536 if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt)) 3537 return nullptr; 3538 3539 // Do not promote if the operand has been added by codegenprepare. 3540 // Otherwise, it means we are undoing an optimization that is likely to be 3541 // redone, thus causing potential infinite loop. 3542 if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd)) 3543 return nullptr; 3544 3545 // SExt or Trunc instructions. 3546 // Return the related handler. 3547 if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) || 3548 isa<ZExtInst>(ExtOpnd)) 3549 return promoteOperandForTruncAndAnyExt; 3550 3551 // Regular instruction. 3552 // Abort early if we will have to insert non-free instructions. 3553 if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType())) 3554 return nullptr; 3555 return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther; 3556 } 3557 3558 Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt( 3559 Instruction *SExt, TypePromotionTransaction &TPT, 3560 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 3561 SmallVectorImpl<Instruction *> *Exts, 3562 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) { 3563 // By construction, the operand of SExt is an instruction. Otherwise we cannot 3564 // get through it and this method should not be called. 3565 Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0)); 3566 Value *ExtVal = SExt; 3567 bool HasMergedNonFreeExt = false; 3568 if (isa<ZExtInst>(SExtOpnd)) { 3569 // Replace s|zext(zext(opnd)) 3570 // => zext(opnd). 3571 HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd); 3572 Value *ZExt = 3573 TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType()); 3574 TPT.replaceAllUsesWith(SExt, ZExt); 3575 TPT.eraseInstruction(SExt); 3576 ExtVal = ZExt; 3577 } else { 3578 // Replace z|sext(trunc(opnd)) or sext(sext(opnd)) 3579 // => z|sext(opnd). 3580 TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0)); 3581 } 3582 CreatedInstsCost = 0; 3583 3584 // Remove dead code. 3585 if (SExtOpnd->use_empty()) 3586 TPT.eraseInstruction(SExtOpnd); 3587 3588 // Check if the extension is still needed. 3589 Instruction *ExtInst = dyn_cast<Instruction>(ExtVal); 3590 if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) { 3591 if (ExtInst) { 3592 if (Exts) 3593 Exts->push_back(ExtInst); 3594 CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt; 3595 } 3596 return ExtVal; 3597 } 3598 3599 // At this point we have: ext ty opnd to ty. 3600 // Reassign the uses of ExtInst to the opnd and remove ExtInst. 3601 Value *NextVal = ExtInst->getOperand(0); 3602 TPT.eraseInstruction(ExtInst, NextVal); 3603 return NextVal; 3604 } 3605 3606 Value *TypePromotionHelper::promoteOperandForOther( 3607 Instruction *Ext, TypePromotionTransaction &TPT, 3608 InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost, 3609 SmallVectorImpl<Instruction *> *Exts, 3610 SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI, 3611 bool IsSExt) { 3612 // By construction, the operand of Ext is an instruction. Otherwise we cannot 3613 // get through it and this method should not be called. 3614 Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0)); 3615 CreatedInstsCost = 0; 3616 if (!ExtOpnd->hasOneUse()) { 3617 // ExtOpnd will be promoted. 3618 // All its uses, but Ext, will need to use a truncated value of the 3619 // promoted version. 3620 // Create the truncate now. 3621 Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType()); 3622 if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) { 3623 // Insert it just after the definition. 3624 ITrunc->moveAfter(ExtOpnd); 3625 if (Truncs) 3626 Truncs->push_back(ITrunc); 3627 } 3628 3629 TPT.replaceAllUsesWith(ExtOpnd, Trunc); 3630 // Restore the operand of Ext (which has been replaced by the previous call 3631 // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext. 3632 TPT.setOperand(Ext, 0, ExtOpnd); 3633 } 3634 3635 // Get through the Instruction: 3636 // 1. Update its type. 3637 // 2. Replace the uses of Ext by Inst. 3638 // 3. Extend each operand that needs to be extended. 3639 3640 // Remember the original type of the instruction before promotion. 3641 // This is useful to know that the high bits are sign extended bits. 3642 addPromotedInst(PromotedInsts, ExtOpnd, IsSExt); 3643 // Step #1. 3644 TPT.mutateType(ExtOpnd, Ext->getType()); 3645 // Step #2. 3646 TPT.replaceAllUsesWith(Ext, ExtOpnd); 3647 // Step #3. 3648 Instruction *ExtForOpnd = Ext; 3649 3650 LLVM_DEBUG(dbgs() << "Propagate Ext to operands\n"); 3651 for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx; 3652 ++OpIdx) { 3653 LLVM_DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n'); 3654 if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() || 3655 !shouldExtOperand(ExtOpnd, OpIdx)) { 3656 LLVM_DEBUG(dbgs() << "No need to propagate\n"); 3657 continue; 3658 } 3659 // Check if we can statically extend the operand. 3660 Value *Opnd = ExtOpnd->getOperand(OpIdx); 3661 if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) { 3662 LLVM_DEBUG(dbgs() << "Statically extend\n"); 3663 unsigned BitWidth = Ext->getType()->getIntegerBitWidth(); 3664 APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth) 3665 : Cst->getValue().zext(BitWidth); 3666 TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal)); 3667 continue; 3668 } 3669 // UndefValue are typed, so we have to statically sign extend them. 3670 if (isa<UndefValue>(Opnd)) { 3671 LLVM_DEBUG(dbgs() << "Statically extend\n"); 3672 TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType())); 3673 continue; 3674 } 3675 3676 // Otherwise we have to explicitly sign extend the operand. 3677 // Check if Ext was reused to extend an operand. 3678 if (!ExtForOpnd) { 3679 // If yes, create a new one. 3680 LLVM_DEBUG(dbgs() << "More operands to ext\n"); 3681 Value *ValForExtOpnd = IsSExt ? TPT.createSExt(Ext, Opnd, Ext->getType()) 3682 : TPT.createZExt(Ext, Opnd, Ext->getType()); 3683 if (!isa<Instruction>(ValForExtOpnd)) { 3684 TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd); 3685 continue; 3686 } 3687 ExtForOpnd = cast<Instruction>(ValForExtOpnd); 3688 } 3689 if (Exts) 3690 Exts->push_back(ExtForOpnd); 3691 TPT.setOperand(ExtForOpnd, 0, Opnd); 3692 3693 // Move the sign extension before the insertion point. 3694 TPT.moveBefore(ExtForOpnd, ExtOpnd); 3695 TPT.setOperand(ExtOpnd, OpIdx, ExtForOpnd); 3696 CreatedInstsCost += !TLI.isExtFree(ExtForOpnd); 3697 // If more sext are required, new instructions will have to be created. 3698 ExtForOpnd = nullptr; 3699 } 3700 if (ExtForOpnd == Ext) { 3701 LLVM_DEBUG(dbgs() << "Extension is useless now\n"); 3702 TPT.eraseInstruction(Ext); 3703 } 3704 return ExtOpnd; 3705 } 3706 3707 /// Check whether or not promoting an instruction to a wider type is profitable. 3708 /// \p NewCost gives the cost of extension instructions created by the 3709 /// promotion. 3710 /// \p OldCost gives the cost of extension instructions before the promotion 3711 /// plus the number of instructions that have been 3712 /// matched in the addressing mode the promotion. 3713 /// \p PromotedOperand is the value that has been promoted. 3714 /// \return True if the promotion is profitable, false otherwise. 3715 bool AddressingModeMatcher::isPromotionProfitable( 3716 unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const { 3717 LLVM_DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost 3718 << '\n'); 3719 // The cost of the new extensions is greater than the cost of the 3720 // old extension plus what we folded. 3721 // This is not profitable. 3722 if (NewCost > OldCost) 3723 return false; 3724 if (NewCost < OldCost) 3725 return true; 3726 // The promotion is neutral but it may help folding the sign extension in 3727 // loads for instance. 3728 // Check that we did not create an illegal instruction. 3729 return isPromotedInstructionLegal(TLI, DL, PromotedOperand); 3730 } 3731 3732 /// Given an instruction or constant expr, see if we can fold the operation 3733 /// into the addressing mode. If so, update the addressing mode and return 3734 /// true, otherwise return false without modifying AddrMode. 3735 /// If \p MovedAway is not NULL, it contains the information of whether or 3736 /// not AddrInst has to be folded into the addressing mode on success. 3737 /// If \p MovedAway == true, \p AddrInst will not be part of the addressing 3738 /// because it has been moved away. 3739 /// Thus AddrInst must not be added in the matched instructions. 3740 /// This state can happen when AddrInst is a sext, since it may be moved away. 3741 /// Therefore, AddrInst may not be valid when MovedAway is true and it must 3742 /// not be referenced anymore. 3743 bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, 3744 unsigned Depth, 3745 bool *MovedAway) { 3746 // Avoid exponential behavior on extremely deep expression trees. 3747 if (Depth >= 5) return false; 3748 3749 // By default, all matched instructions stay in place. 3750 if (MovedAway) 3751 *MovedAway = false; 3752 3753 switch (Opcode) { 3754 case Instruction::PtrToInt: 3755 // PtrToInt is always a noop, as we know that the int type is pointer sized. 3756 return matchAddr(AddrInst->getOperand(0), Depth); 3757 case Instruction::IntToPtr: { 3758 auto AS = AddrInst->getType()->getPointerAddressSpace(); 3759 auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS)); 3760 // This inttoptr is a no-op if the integer type is pointer sized. 3761 if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy) 3762 return matchAddr(AddrInst->getOperand(0), Depth); 3763 return false; 3764 } 3765 case Instruction::BitCast: 3766 // BitCast is always a noop, and we can handle it as long as it is 3767 // int->int or pointer->pointer (we don't want int<->fp or something). 3768 if (AddrInst->getOperand(0)->getType()->isIntOrPtrTy() && 3769 // Don't touch identity bitcasts. These were probably put here by LSR, 3770 // and we don't want to mess around with them. Assume it knows what it 3771 // is doing. 3772 AddrInst->getOperand(0)->getType() != AddrInst->getType()) 3773 return matchAddr(AddrInst->getOperand(0), Depth); 3774 return false; 3775 case Instruction::AddrSpaceCast: { 3776 unsigned SrcAS 3777 = AddrInst->getOperand(0)->getType()->getPointerAddressSpace(); 3778 unsigned DestAS = AddrInst->getType()->getPointerAddressSpace(); 3779 if (TLI.isNoopAddrSpaceCast(SrcAS, DestAS)) 3780 return matchAddr(AddrInst->getOperand(0), Depth); 3781 return false; 3782 } 3783 case Instruction::Add: { 3784 // Check to see if we can merge in the RHS then the LHS. If so, we win. 3785 ExtAddrMode BackupAddrMode = AddrMode; 3786 unsigned OldSize = AddrModeInsts.size(); 3787 // Start a transaction at this point. 3788 // The LHS may match but not the RHS. 3789 // Therefore, we need a higher level restoration point to undo partially 3790 // matched operation. 3791 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 3792 TPT.getRestorationPoint(); 3793 3794 if (matchAddr(AddrInst->getOperand(1), Depth+1) && 3795 matchAddr(AddrInst->getOperand(0), Depth+1)) 3796 return true; 3797 3798 // Restore the old addr mode info. 3799 AddrMode = BackupAddrMode; 3800 AddrModeInsts.resize(OldSize); 3801 TPT.rollback(LastKnownGood); 3802 3803 // Otherwise this was over-aggressive. Try merging in the LHS then the RHS. 3804 if (matchAddr(AddrInst->getOperand(0), Depth+1) && 3805 matchAddr(AddrInst->getOperand(1), Depth+1)) 3806 return true; 3807 3808 // Otherwise we definitely can't merge the ADD in. 3809 AddrMode = BackupAddrMode; 3810 AddrModeInsts.resize(OldSize); 3811 TPT.rollback(LastKnownGood); 3812 break; 3813 } 3814 //case Instruction::Or: 3815 // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD. 3816 //break; 3817 case Instruction::Mul: 3818 case Instruction::Shl: { 3819 // Can only handle X*C and X << C. 3820 ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1)); 3821 if (!RHS || RHS->getBitWidth() > 64) 3822 return false; 3823 int64_t Scale = RHS->getSExtValue(); 3824 if (Opcode == Instruction::Shl) 3825 Scale = 1LL << Scale; 3826 3827 return matchScaledValue(AddrInst->getOperand(0), Scale, Depth); 3828 } 3829 case Instruction::GetElementPtr: { 3830 // Scan the GEP. We check it if it contains constant offsets and at most 3831 // one variable offset. 3832 int VariableOperand = -1; 3833 unsigned VariableScale = 0; 3834 3835 int64_t ConstantOffset = 0; 3836 gep_type_iterator GTI = gep_type_begin(AddrInst); 3837 for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) { 3838 if (StructType *STy = GTI.getStructTypeOrNull()) { 3839 const StructLayout *SL = DL.getStructLayout(STy); 3840 unsigned Idx = 3841 cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue(); 3842 ConstantOffset += SL->getElementOffset(Idx); 3843 } else { 3844 uint64_t TypeSize = DL.getTypeAllocSize(GTI.getIndexedType()); 3845 if (ConstantInt *CI = dyn_cast<ConstantInt>(AddrInst->getOperand(i))) { 3846 ConstantOffset += CI->getSExtValue() * TypeSize; 3847 } else if (TypeSize) { // Scales of zero don't do anything. 3848 // We only allow one variable index at the moment. 3849 if (VariableOperand != -1) 3850 return false; 3851 3852 // Remember the variable index. 3853 VariableOperand = i; 3854 VariableScale = TypeSize; 3855 } 3856 } 3857 } 3858 3859 // A common case is for the GEP to only do a constant offset. In this case, 3860 // just add it to the disp field and check validity. 3861 if (VariableOperand == -1) { 3862 AddrMode.BaseOffs += ConstantOffset; 3863 if (ConstantOffset == 0 || 3864 TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) { 3865 // Check to see if we can fold the base pointer in too. 3866 if (matchAddr(AddrInst->getOperand(0), Depth+1)) 3867 return true; 3868 } else if (EnableGEPOffsetSplit && isa<GetElementPtrInst>(AddrInst) && 3869 TLI.shouldConsiderGEPOffsetSplit() && Depth == 0 && 3870 ConstantOffset > 0) { 3871 // Record GEPs with non-zero offsets as candidates for splitting in the 3872 // event that the offset cannot fit into the r+i addressing mode. 3873 // Simple and common case that only one GEP is used in calculating the 3874 // address for the memory access. 3875 Value *Base = AddrInst->getOperand(0); 3876 auto *BaseI = dyn_cast<Instruction>(Base); 3877 auto *GEP = cast<GetElementPtrInst>(AddrInst); 3878 if (isa<Argument>(Base) || isa<GlobalValue>(Base) || 3879 (BaseI && !isa<CastInst>(BaseI) && 3880 !isa<GetElementPtrInst>(BaseI))) { 3881 // If the base is an instruction, make sure the GEP is not in the same 3882 // basic block as the base. If the base is an argument or global 3883 // value, make sure the GEP is not in the entry block. Otherwise, 3884 // instruction selection can undo the split. Also make sure the 3885 // parent block allows inserting non-PHI instructions before the 3886 // terminator. 3887 BasicBlock *Parent = 3888 BaseI ? BaseI->getParent() : &GEP->getFunction()->getEntryBlock(); 3889 if (GEP->getParent() != Parent && !Parent->getTerminator()->isEHPad()) 3890 LargeOffsetGEP = std::make_pair(GEP, ConstantOffset); 3891 } 3892 } 3893 AddrMode.BaseOffs -= ConstantOffset; 3894 return false; 3895 } 3896 3897 // Save the valid addressing mode in case we can't match. 3898 ExtAddrMode BackupAddrMode = AddrMode; 3899 unsigned OldSize = AddrModeInsts.size(); 3900 3901 // See if the scale and offset amount is valid for this target. 3902 AddrMode.BaseOffs += ConstantOffset; 3903 3904 // Match the base operand of the GEP. 3905 if (!matchAddr(AddrInst->getOperand(0), Depth+1)) { 3906 // If it couldn't be matched, just stuff the value in a register. 3907 if (AddrMode.HasBaseReg) { 3908 AddrMode = BackupAddrMode; 3909 AddrModeInsts.resize(OldSize); 3910 return false; 3911 } 3912 AddrMode.HasBaseReg = true; 3913 AddrMode.BaseReg = AddrInst->getOperand(0); 3914 } 3915 3916 // Match the remaining variable portion of the GEP. 3917 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale, 3918 Depth)) { 3919 // If it couldn't be matched, try stuffing the base into a register 3920 // instead of matching it, and retrying the match of the scale. 3921 AddrMode = BackupAddrMode; 3922 AddrModeInsts.resize(OldSize); 3923 if (AddrMode.HasBaseReg) 3924 return false; 3925 AddrMode.HasBaseReg = true; 3926 AddrMode.BaseReg = AddrInst->getOperand(0); 3927 AddrMode.BaseOffs += ConstantOffset; 3928 if (!matchScaledValue(AddrInst->getOperand(VariableOperand), 3929 VariableScale, Depth)) { 3930 // If even that didn't work, bail. 3931 AddrMode = BackupAddrMode; 3932 AddrModeInsts.resize(OldSize); 3933 return false; 3934 } 3935 } 3936 3937 return true; 3938 } 3939 case Instruction::SExt: 3940 case Instruction::ZExt: { 3941 Instruction *Ext = dyn_cast<Instruction>(AddrInst); 3942 if (!Ext) 3943 return false; 3944 3945 // Try to move this ext out of the way of the addressing mode. 3946 // Ask for a method for doing so. 3947 TypePromotionHelper::Action TPH = 3948 TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts); 3949 if (!TPH) 3950 return false; 3951 3952 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 3953 TPT.getRestorationPoint(); 3954 unsigned CreatedInstsCost = 0; 3955 unsigned ExtCost = !TLI.isExtFree(Ext); 3956 Value *PromotedOperand = 3957 TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI); 3958 // SExt has been moved away. 3959 // Thus either it will be rematched later in the recursive calls or it is 3960 // gone. Anyway, we must not fold it into the addressing mode at this point. 3961 // E.g., 3962 // op = add opnd, 1 3963 // idx = ext op 3964 // addr = gep base, idx 3965 // is now: 3966 // promotedOpnd = ext opnd <- no match here 3967 // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls) 3968 // addr = gep base, op <- match 3969 if (MovedAway) 3970 *MovedAway = true; 3971 3972 assert(PromotedOperand && 3973 "TypePromotionHelper should have filtered out those cases"); 3974 3975 ExtAddrMode BackupAddrMode = AddrMode; 3976 unsigned OldSize = AddrModeInsts.size(); 3977 3978 if (!matchAddr(PromotedOperand, Depth) || 3979 // The total of the new cost is equal to the cost of the created 3980 // instructions. 3981 // The total of the old cost is equal to the cost of the extension plus 3982 // what we have saved in the addressing mode. 3983 !isPromotionProfitable(CreatedInstsCost, 3984 ExtCost + (AddrModeInsts.size() - OldSize), 3985 PromotedOperand)) { 3986 AddrMode = BackupAddrMode; 3987 AddrModeInsts.resize(OldSize); 3988 LLVM_DEBUG(dbgs() << "Sign extension does not pay off: rollback\n"); 3989 TPT.rollback(LastKnownGood); 3990 return false; 3991 } 3992 return true; 3993 } 3994 } 3995 return false; 3996 } 3997 3998 /// If we can, try to add the value of 'Addr' into the current addressing mode. 3999 /// If Addr can't be added to AddrMode this returns false and leaves AddrMode 4000 /// unmodified. This assumes that Addr is either a pointer type or intptr_t 4001 /// for the target. 4002 /// 4003 bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) { 4004 // Start a transaction at this point that we will rollback if the matching 4005 // fails. 4006 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 4007 TPT.getRestorationPoint(); 4008 if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) { 4009 // Fold in immediates if legal for the target. 4010 AddrMode.BaseOffs += CI->getSExtValue(); 4011 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) 4012 return true; 4013 AddrMode.BaseOffs -= CI->getSExtValue(); 4014 } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) { 4015 // If this is a global variable, try to fold it into the addressing mode. 4016 if (!AddrMode.BaseGV) { 4017 AddrMode.BaseGV = GV; 4018 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) 4019 return true; 4020 AddrMode.BaseGV = nullptr; 4021 } 4022 } else if (Instruction *I = dyn_cast<Instruction>(Addr)) { 4023 ExtAddrMode BackupAddrMode = AddrMode; 4024 unsigned OldSize = AddrModeInsts.size(); 4025 4026 // Check to see if it is possible to fold this operation. 4027 bool MovedAway = false; 4028 if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) { 4029 // This instruction may have been moved away. If so, there is nothing 4030 // to check here. 4031 if (MovedAway) 4032 return true; 4033 // Okay, it's possible to fold this. Check to see if it is actually 4034 // *profitable* to do so. We use a simple cost model to avoid increasing 4035 // register pressure too much. 4036 if (I->hasOneUse() || 4037 isProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) { 4038 AddrModeInsts.push_back(I); 4039 return true; 4040 } 4041 4042 // It isn't profitable to do this, roll back. 4043 //cerr << "NOT FOLDING: " << *I; 4044 AddrMode = BackupAddrMode; 4045 AddrModeInsts.resize(OldSize); 4046 TPT.rollback(LastKnownGood); 4047 } 4048 } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) { 4049 if (matchOperationAddr(CE, CE->getOpcode(), Depth)) 4050 return true; 4051 TPT.rollback(LastKnownGood); 4052 } else if (isa<ConstantPointerNull>(Addr)) { 4053 // Null pointer gets folded without affecting the addressing mode. 4054 return true; 4055 } 4056 4057 // Worse case, the target should support [reg] addressing modes. :) 4058 if (!AddrMode.HasBaseReg) { 4059 AddrMode.HasBaseReg = true; 4060 AddrMode.BaseReg = Addr; 4061 // Still check for legality in case the target supports [imm] but not [i+r]. 4062 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) 4063 return true; 4064 AddrMode.HasBaseReg = false; 4065 AddrMode.BaseReg = nullptr; 4066 } 4067 4068 // If the base register is already taken, see if we can do [r+r]. 4069 if (AddrMode.Scale == 0) { 4070 AddrMode.Scale = 1; 4071 AddrMode.ScaledReg = Addr; 4072 if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) 4073 return true; 4074 AddrMode.Scale = 0; 4075 AddrMode.ScaledReg = nullptr; 4076 } 4077 // Couldn't match. 4078 TPT.rollback(LastKnownGood); 4079 return false; 4080 } 4081 4082 /// Check to see if all uses of OpVal by the specified inline asm call are due 4083 /// to memory operands. If so, return true, otherwise return false. 4084 static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal, 4085 const TargetLowering &TLI, 4086 const TargetRegisterInfo &TRI) { 4087 const Function *F = CI->getFunction(); 4088 TargetLowering::AsmOperandInfoVector TargetConstraints = 4089 TLI.ParseConstraints(F->getParent()->getDataLayout(), &TRI, 4090 ImmutableCallSite(CI)); 4091 4092 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 4093 TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 4094 4095 // Compute the constraint code and ConstraintType to use. 4096 TLI.ComputeConstraintToUse(OpInfo, SDValue()); 4097 4098 // If this asm operand is our Value*, and if it isn't an indirect memory 4099 // operand, we can't fold it! 4100 if (OpInfo.CallOperandVal == OpVal && 4101 (OpInfo.ConstraintType != TargetLowering::C_Memory || 4102 !OpInfo.isIndirect)) 4103 return false; 4104 } 4105 4106 return true; 4107 } 4108 4109 // Max number of memory uses to look at before aborting the search to conserve 4110 // compile time. 4111 static constexpr int MaxMemoryUsesToScan = 20; 4112 4113 /// Recursively walk all the uses of I until we find a memory use. 4114 /// If we find an obviously non-foldable instruction, return true. 4115 /// Add the ultimately found memory instructions to MemoryUses. 4116 static bool FindAllMemoryUses( 4117 Instruction *I, 4118 SmallVectorImpl<std::pair<Instruction *, unsigned>> &MemoryUses, 4119 SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetLowering &TLI, 4120 const TargetRegisterInfo &TRI, int SeenInsts = 0) { 4121 // If we already considered this instruction, we're done. 4122 if (!ConsideredInsts.insert(I).second) 4123 return false; 4124 4125 // If this is an obviously unfoldable instruction, bail out. 4126 if (!MightBeFoldableInst(I)) 4127 return true; 4128 4129 const bool OptSize = I->getFunction()->optForSize(); 4130 4131 // Loop over all the uses, recursively processing them. 4132 for (Use &U : I->uses()) { 4133 // Conservatively return true if we're seeing a large number or a deep chain 4134 // of users. This avoids excessive compilation times in pathological cases. 4135 if (SeenInsts++ >= MaxMemoryUsesToScan) 4136 return true; 4137 4138 Instruction *UserI = cast<Instruction>(U.getUser()); 4139 if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) { 4140 MemoryUses.push_back(std::make_pair(LI, U.getOperandNo())); 4141 continue; 4142 } 4143 4144 if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) { 4145 unsigned opNo = U.getOperandNo(); 4146 if (opNo != StoreInst::getPointerOperandIndex()) 4147 return true; // Storing addr, not into addr. 4148 MemoryUses.push_back(std::make_pair(SI, opNo)); 4149 continue; 4150 } 4151 4152 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(UserI)) { 4153 unsigned opNo = U.getOperandNo(); 4154 if (opNo != AtomicRMWInst::getPointerOperandIndex()) 4155 return true; // Storing addr, not into addr. 4156 MemoryUses.push_back(std::make_pair(RMW, opNo)); 4157 continue; 4158 } 4159 4160 if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(UserI)) { 4161 unsigned opNo = U.getOperandNo(); 4162 if (opNo != AtomicCmpXchgInst::getPointerOperandIndex()) 4163 return true; // Storing addr, not into addr. 4164 MemoryUses.push_back(std::make_pair(CmpX, opNo)); 4165 continue; 4166 } 4167 4168 if (CallInst *CI = dyn_cast<CallInst>(UserI)) { 4169 // If this is a cold call, we can sink the addressing calculation into 4170 // the cold path. See optimizeCallInst 4171 if (!OptSize && CI->hasFnAttr(Attribute::Cold)) 4172 continue; 4173 4174 InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue()); 4175 if (!IA) return true; 4176 4177 // If this is a memory operand, we're cool, otherwise bail out. 4178 if (!IsOperandAMemoryOperand(CI, IA, I, TLI, TRI)) 4179 return true; 4180 continue; 4181 } 4182 4183 if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TLI, TRI, 4184 SeenInsts)) 4185 return true; 4186 } 4187 4188 return false; 4189 } 4190 4191 /// Return true if Val is already known to be live at the use site that we're 4192 /// folding it into. If so, there is no cost to include it in the addressing 4193 /// mode. KnownLive1 and KnownLive2 are two values that we know are live at the 4194 /// instruction already. 4195 bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,Value *KnownLive1, 4196 Value *KnownLive2) { 4197 // If Val is either of the known-live values, we know it is live! 4198 if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2) 4199 return true; 4200 4201 // All values other than instructions and arguments (e.g. constants) are live. 4202 if (!isa<Instruction>(Val) && !isa<Argument>(Val)) return true; 4203 4204 // If Val is a constant sized alloca in the entry block, it is live, this is 4205 // true because it is just a reference to the stack/frame pointer, which is 4206 // live for the whole function. 4207 if (AllocaInst *AI = dyn_cast<AllocaInst>(Val)) 4208 if (AI->isStaticAlloca()) 4209 return true; 4210 4211 // Check to see if this value is already used in the memory instruction's 4212 // block. If so, it's already live into the block at the very least, so we 4213 // can reasonably fold it. 4214 return Val->isUsedInBasicBlock(MemoryInst->getParent()); 4215 } 4216 4217 /// It is possible for the addressing mode of the machine to fold the specified 4218 /// instruction into a load or store that ultimately uses it. 4219 /// However, the specified instruction has multiple uses. 4220 /// Given this, it may actually increase register pressure to fold it 4221 /// into the load. For example, consider this code: 4222 /// 4223 /// X = ... 4224 /// Y = X+1 4225 /// use(Y) -> nonload/store 4226 /// Z = Y+1 4227 /// load Z 4228 /// 4229 /// In this case, Y has multiple uses, and can be folded into the load of Z 4230 /// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to 4231 /// be live at the use(Y) line. If we don't fold Y into load Z, we use one 4232 /// fewer register. Since Y can't be folded into "use(Y)" we don't increase the 4233 /// number of computations either. 4234 /// 4235 /// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If 4236 /// X was live across 'load Z' for other reasons, we actually *would* want to 4237 /// fold the addressing mode in the Z case. This would make Y die earlier. 4238 bool AddressingModeMatcher:: 4239 isProfitableToFoldIntoAddressingMode(Instruction *I, ExtAddrMode &AMBefore, 4240 ExtAddrMode &AMAfter) { 4241 if (IgnoreProfitability) return true; 4242 4243 // AMBefore is the addressing mode before this instruction was folded into it, 4244 // and AMAfter is the addressing mode after the instruction was folded. Get 4245 // the set of registers referenced by AMAfter and subtract out those 4246 // referenced by AMBefore: this is the set of values which folding in this 4247 // address extends the lifetime of. 4248 // 4249 // Note that there are only two potential values being referenced here, 4250 // BaseReg and ScaleReg (global addresses are always available, as are any 4251 // folded immediates). 4252 Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg; 4253 4254 // If the BaseReg or ScaledReg was referenced by the previous addrmode, their 4255 // lifetime wasn't extended by adding this instruction. 4256 if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg)) 4257 BaseReg = nullptr; 4258 if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg)) 4259 ScaledReg = nullptr; 4260 4261 // If folding this instruction (and it's subexprs) didn't extend any live 4262 // ranges, we're ok with it. 4263 if (!BaseReg && !ScaledReg) 4264 return true; 4265 4266 // If all uses of this instruction can have the address mode sunk into them, 4267 // we can remove the addressing mode and effectively trade one live register 4268 // for another (at worst.) In this context, folding an addressing mode into 4269 // the use is just a particularly nice way of sinking it. 4270 SmallVector<std::pair<Instruction*,unsigned>, 16> MemoryUses; 4271 SmallPtrSet<Instruction*, 16> ConsideredInsts; 4272 if (FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TLI, TRI)) 4273 return false; // Has a non-memory, non-foldable use! 4274 4275 // Now that we know that all uses of this instruction are part of a chain of 4276 // computation involving only operations that could theoretically be folded 4277 // into a memory use, loop over each of these memory operation uses and see 4278 // if they could *actually* fold the instruction. The assumption is that 4279 // addressing modes are cheap and that duplicating the computation involved 4280 // many times is worthwhile, even on a fastpath. For sinking candidates 4281 // (i.e. cold call sites), this serves as a way to prevent excessive code 4282 // growth since most architectures have some reasonable small and fast way to 4283 // compute an effective address. (i.e LEA on x86) 4284 SmallVector<Instruction*, 32> MatchedAddrModeInsts; 4285 for (unsigned i = 0, e = MemoryUses.size(); i != e; ++i) { 4286 Instruction *User = MemoryUses[i].first; 4287 unsigned OpNo = MemoryUses[i].second; 4288 4289 // Get the access type of this use. If the use isn't a pointer, we don't 4290 // know what it accesses. 4291 Value *Address = User->getOperand(OpNo); 4292 PointerType *AddrTy = dyn_cast<PointerType>(Address->getType()); 4293 if (!AddrTy) 4294 return false; 4295 Type *AddressAccessTy = AddrTy->getElementType(); 4296 unsigned AS = AddrTy->getAddressSpace(); 4297 4298 // Do a match against the root of this address, ignoring profitability. This 4299 // will tell us if the addressing mode for the memory operation will 4300 // *actually* cover the shared instruction. 4301 ExtAddrMode Result; 4302 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(nullptr, 4303 0); 4304 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 4305 TPT.getRestorationPoint(); 4306 AddressingModeMatcher Matcher( 4307 MatchedAddrModeInsts, TLI, TRI, AddressAccessTy, AS, MemoryInst, Result, 4308 InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP); 4309 Matcher.IgnoreProfitability = true; 4310 bool Success = Matcher.matchAddr(Address, 0); 4311 (void)Success; assert(Success && "Couldn't select *anything*?"); 4312 4313 // The match was to check the profitability, the changes made are not 4314 // part of the original matcher. Therefore, they should be dropped 4315 // otherwise the original matcher will not present the right state. 4316 TPT.rollback(LastKnownGood); 4317 4318 // If the match didn't cover I, then it won't be shared by it. 4319 if (!is_contained(MatchedAddrModeInsts, I)) 4320 return false; 4321 4322 MatchedAddrModeInsts.clear(); 4323 } 4324 4325 return true; 4326 } 4327 4328 /// Return true if the specified values are defined in a 4329 /// different basic block than BB. 4330 static bool IsNonLocalValue(Value *V, BasicBlock *BB) { 4331 if (Instruction *I = dyn_cast<Instruction>(V)) 4332 return I->getParent() != BB; 4333 return false; 4334 } 4335 4336 /// Sink addressing mode computation immediate before MemoryInst if doing so 4337 /// can be done without increasing register pressure. The need for the 4338 /// register pressure constraint means this can end up being an all or nothing 4339 /// decision for all uses of the same addressing computation. 4340 /// 4341 /// Load and Store Instructions often have addressing modes that can do 4342 /// significant amounts of computation. As such, instruction selection will try 4343 /// to get the load or store to do as much computation as possible for the 4344 /// program. The problem is that isel can only see within a single block. As 4345 /// such, we sink as much legal addressing mode work into the block as possible. 4346 /// 4347 /// This method is used to optimize both load/store and inline asms with memory 4348 /// operands. It's also used to sink addressing computations feeding into cold 4349 /// call sites into their (cold) basic block. 4350 /// 4351 /// The motivation for handling sinking into cold blocks is that doing so can 4352 /// both enable other address mode sinking (by satisfying the register pressure 4353 /// constraint above), and reduce register pressure globally (by removing the 4354 /// addressing mode computation from the fast path entirely.). 4355 bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, 4356 Type *AccessTy, unsigned AddrSpace) { 4357 Value *Repl = Addr; 4358 4359 // Try to collapse single-value PHI nodes. This is necessary to undo 4360 // unprofitable PRE transformations. 4361 SmallVector<Value*, 8> worklist; 4362 SmallPtrSet<Value*, 16> Visited; 4363 worklist.push_back(Addr); 4364 4365 // Use a worklist to iteratively look through PHI and select nodes, and 4366 // ensure that the addressing mode obtained from the non-PHI/select roots of 4367 // the graph are compatible. 4368 bool PhiOrSelectSeen = false; 4369 SmallVector<Instruction*, 16> AddrModeInsts; 4370 const SimplifyQuery SQ(*DL, TLInfo); 4371 AddressingModeCombiner AddrModes(SQ, { Addr, MemoryInst->getParent() }); 4372 TypePromotionTransaction TPT(RemovedInsts); 4373 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 4374 TPT.getRestorationPoint(); 4375 while (!worklist.empty()) { 4376 Value *V = worklist.back(); 4377 worklist.pop_back(); 4378 4379 // We allow traversing cyclic Phi nodes. 4380 // In case of success after this loop we ensure that traversing through 4381 // Phi nodes ends up with all cases to compute address of the form 4382 // BaseGV + Base + Scale * Index + Offset 4383 // where Scale and Offset are constans and BaseGV, Base and Index 4384 // are exactly the same Values in all cases. 4385 // It means that BaseGV, Scale and Offset dominate our memory instruction 4386 // and have the same value as they had in address computation represented 4387 // as Phi. So we can safely sink address computation to memory instruction. 4388 if (!Visited.insert(V).second) 4389 continue; 4390 4391 // For a PHI node, push all of its incoming values. 4392 if (PHINode *P = dyn_cast<PHINode>(V)) { 4393 for (Value *IncValue : P->incoming_values()) 4394 worklist.push_back(IncValue); 4395 PhiOrSelectSeen = true; 4396 continue; 4397 } 4398 // Similar for select. 4399 if (SelectInst *SI = dyn_cast<SelectInst>(V)) { 4400 worklist.push_back(SI->getFalseValue()); 4401 worklist.push_back(SI->getTrueValue()); 4402 PhiOrSelectSeen = true; 4403 continue; 4404 } 4405 4406 // For non-PHIs, determine the addressing mode being computed. Note that 4407 // the result may differ depending on what other uses our candidate 4408 // addressing instructions might have. 4409 AddrModeInsts.clear(); 4410 std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(nullptr, 4411 0); 4412 ExtAddrMode NewAddrMode = AddressingModeMatcher::Match( 4413 V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, *TRI, 4414 InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP); 4415 4416 GetElementPtrInst *GEP = LargeOffsetGEP.first; 4417 if (GEP && GEP->getParent() != MemoryInst->getParent() && 4418 !NewGEPBases.count(GEP)) { 4419 // If splitting the underlying data structure can reduce the offset of a 4420 // GEP, collect the GEP. Skip the GEPs that are the new bases of 4421 // previously split data structures. 4422 LargeOffsetGEPMap[GEP->getPointerOperand()].push_back(LargeOffsetGEP); 4423 if (LargeOffsetGEPID.find(GEP) == LargeOffsetGEPID.end()) 4424 LargeOffsetGEPID[GEP] = LargeOffsetGEPID.size(); 4425 } 4426 4427 NewAddrMode.OriginalValue = V; 4428 if (!AddrModes.addNewAddrMode(NewAddrMode)) 4429 break; 4430 } 4431 4432 // Try to combine the AddrModes we've collected. If we couldn't collect any, 4433 // or we have multiple but either couldn't combine them or combining them 4434 // wouldn't do anything useful, bail out now. 4435 if (!AddrModes.combineAddrModes()) { 4436 TPT.rollback(LastKnownGood); 4437 return false; 4438 } 4439 TPT.commit(); 4440 4441 // Get the combined AddrMode (or the only AddrMode, if we only had one). 4442 ExtAddrMode AddrMode = AddrModes.getAddrMode(); 4443 4444 // If all the instructions matched are already in this BB, don't do anything. 4445 // If we saw a Phi node then it is not local definitely, and if we saw a select 4446 // then we want to push the address calculation past it even if it's already 4447 // in this BB. 4448 if (!PhiOrSelectSeen && none_of(AddrModeInsts, [&](Value *V) { 4449 return IsNonLocalValue(V, MemoryInst->getParent()); 4450 })) { 4451 LLVM_DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode 4452 << "\n"); 4453 return false; 4454 } 4455 4456 // Insert this computation right after this user. Since our caller is 4457 // scanning from the top of the BB to the bottom, reuse of the expr are 4458 // guaranteed to happen later. 4459 IRBuilder<> Builder(MemoryInst); 4460 4461 // Now that we determined the addressing expression we want to use and know 4462 // that we have to sink it into this block. Check to see if we have already 4463 // done this for some other load/store instr in this block. If so, reuse 4464 // the computation. Before attempting reuse, check if the address is valid 4465 // as it may have been erased. 4466 4467 WeakTrackingVH SunkAddrVH = SunkAddrs[Addr]; 4468 4469 Value * SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr; 4470 if (SunkAddr) { 4471 LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode 4472 << " for " << *MemoryInst << "\n"); 4473 if (SunkAddr->getType() != Addr->getType()) 4474 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); 4475 } else if (AddrSinkUsingGEPs || 4476 (!AddrSinkUsingGEPs.getNumOccurrences() && TM && TTI->useAA())) { 4477 // By default, we use the GEP-based method when AA is used later. This 4478 // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities. 4479 LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode 4480 << " for " << *MemoryInst << "\n"); 4481 Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); 4482 Value *ResultPtr = nullptr, *ResultIndex = nullptr; 4483 4484 // First, find the pointer. 4485 if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) { 4486 ResultPtr = AddrMode.BaseReg; 4487 AddrMode.BaseReg = nullptr; 4488 } 4489 4490 if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) { 4491 // We can't add more than one pointer together, nor can we scale a 4492 // pointer (both of which seem meaningless). 4493 if (ResultPtr || AddrMode.Scale != 1) 4494 return false; 4495 4496 ResultPtr = AddrMode.ScaledReg; 4497 AddrMode.Scale = 0; 4498 } 4499 4500 // It is only safe to sign extend the BaseReg if we know that the math 4501 // required to create it did not overflow before we extend it. Since 4502 // the original IR value was tossed in favor of a constant back when 4503 // the AddrMode was created we need to bail out gracefully if widths 4504 // do not match instead of extending it. 4505 // 4506 // (See below for code to add the scale.) 4507 if (AddrMode.Scale) { 4508 Type *ScaledRegTy = AddrMode.ScaledReg->getType(); 4509 if (cast<IntegerType>(IntPtrTy)->getBitWidth() > 4510 cast<IntegerType>(ScaledRegTy)->getBitWidth()) 4511 return false; 4512 } 4513 4514 if (AddrMode.BaseGV) { 4515 if (ResultPtr) 4516 return false; 4517 4518 ResultPtr = AddrMode.BaseGV; 4519 } 4520 4521 // If the real base value actually came from an inttoptr, then the matcher 4522 // will look through it and provide only the integer value. In that case, 4523 // use it here. 4524 if (!DL->isNonIntegralPointerType(Addr->getType())) { 4525 if (!ResultPtr && AddrMode.BaseReg) { 4526 ResultPtr = Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(), 4527 "sunkaddr"); 4528 AddrMode.BaseReg = nullptr; 4529 } else if (!ResultPtr && AddrMode.Scale == 1) { 4530 ResultPtr = Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(), 4531 "sunkaddr"); 4532 AddrMode.Scale = 0; 4533 } 4534 } 4535 4536 if (!ResultPtr && 4537 !AddrMode.BaseReg && !AddrMode.Scale && !AddrMode.BaseOffs) { 4538 SunkAddr = Constant::getNullValue(Addr->getType()); 4539 } else if (!ResultPtr) { 4540 return false; 4541 } else { 4542 Type *I8PtrTy = 4543 Builder.getInt8PtrTy(Addr->getType()->getPointerAddressSpace()); 4544 Type *I8Ty = Builder.getInt8Ty(); 4545 4546 // Start with the base register. Do this first so that subsequent address 4547 // matching finds it last, which will prevent it from trying to match it 4548 // as the scaled value in case it happens to be a mul. That would be 4549 // problematic if we've sunk a different mul for the scale, because then 4550 // we'd end up sinking both muls. 4551 if (AddrMode.BaseReg) { 4552 Value *V = AddrMode.BaseReg; 4553 if (V->getType() != IntPtrTy) 4554 V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr"); 4555 4556 ResultIndex = V; 4557 } 4558 4559 // Add the scale value. 4560 if (AddrMode.Scale) { 4561 Value *V = AddrMode.ScaledReg; 4562 if (V->getType() == IntPtrTy) { 4563 // done. 4564 } else { 4565 assert(cast<IntegerType>(IntPtrTy)->getBitWidth() < 4566 cast<IntegerType>(V->getType())->getBitWidth() && 4567 "We can't transform if ScaledReg is too narrow"); 4568 V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); 4569 } 4570 4571 if (AddrMode.Scale != 1) 4572 V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale), 4573 "sunkaddr"); 4574 if (ResultIndex) 4575 ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr"); 4576 else 4577 ResultIndex = V; 4578 } 4579 4580 // Add in the Base Offset if present. 4581 if (AddrMode.BaseOffs) { 4582 Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); 4583 if (ResultIndex) { 4584 // We need to add this separately from the scale above to help with 4585 // SDAG consecutive load/store merging. 4586 if (ResultPtr->getType() != I8PtrTy) 4587 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy); 4588 ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); 4589 } 4590 4591 ResultIndex = V; 4592 } 4593 4594 if (!ResultIndex) { 4595 SunkAddr = ResultPtr; 4596 } else { 4597 if (ResultPtr->getType() != I8PtrTy) 4598 ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy); 4599 SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); 4600 } 4601 4602 if (SunkAddr->getType() != Addr->getType()) 4603 SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType()); 4604 } 4605 } else { 4606 // We'd require a ptrtoint/inttoptr down the line, which we can't do for 4607 // non-integral pointers, so in that case bail out now. 4608 Type *BaseTy = AddrMode.BaseReg ? AddrMode.BaseReg->getType() : nullptr; 4609 Type *ScaleTy = AddrMode.Scale ? AddrMode.ScaledReg->getType() : nullptr; 4610 PointerType *BasePtrTy = dyn_cast_or_null<PointerType>(BaseTy); 4611 PointerType *ScalePtrTy = dyn_cast_or_null<PointerType>(ScaleTy); 4612 if (DL->isNonIntegralPointerType(Addr->getType()) || 4613 (BasePtrTy && DL->isNonIntegralPointerType(BasePtrTy)) || 4614 (ScalePtrTy && DL->isNonIntegralPointerType(ScalePtrTy)) || 4615 (AddrMode.BaseGV && 4616 DL->isNonIntegralPointerType(AddrMode.BaseGV->getType()))) 4617 return false; 4618 4619 LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode 4620 << " for " << *MemoryInst << "\n"); 4621 Type *IntPtrTy = DL->getIntPtrType(Addr->getType()); 4622 Value *Result = nullptr; 4623 4624 // Start with the base register. Do this first so that subsequent address 4625 // matching finds it last, which will prevent it from trying to match it 4626 // as the scaled value in case it happens to be a mul. That would be 4627 // problematic if we've sunk a different mul for the scale, because then 4628 // we'd end up sinking both muls. 4629 if (AddrMode.BaseReg) { 4630 Value *V = AddrMode.BaseReg; 4631 if (V->getType()->isPointerTy()) 4632 V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr"); 4633 if (V->getType() != IntPtrTy) 4634 V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr"); 4635 Result = V; 4636 } 4637 4638 // Add the scale value. 4639 if (AddrMode.Scale) { 4640 Value *V = AddrMode.ScaledReg; 4641 if (V->getType() == IntPtrTy) { 4642 // done. 4643 } else if (V->getType()->isPointerTy()) { 4644 V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr"); 4645 } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() < 4646 cast<IntegerType>(V->getType())->getBitWidth()) { 4647 V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr"); 4648 } else { 4649 // It is only safe to sign extend the BaseReg if we know that the math 4650 // required to create it did not overflow before we extend it. Since 4651 // the original IR value was tossed in favor of a constant back when 4652 // the AddrMode was created we need to bail out gracefully if widths 4653 // do not match instead of extending it. 4654 Instruction *I = dyn_cast_or_null<Instruction>(Result); 4655 if (I && (Result != AddrMode.BaseReg)) 4656 I->eraseFromParent(); 4657 return false; 4658 } 4659 if (AddrMode.Scale != 1) 4660 V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale), 4661 "sunkaddr"); 4662 if (Result) 4663 Result = Builder.CreateAdd(Result, V, "sunkaddr"); 4664 else 4665 Result = V; 4666 } 4667 4668 // Add in the BaseGV if present. 4669 if (AddrMode.BaseGV) { 4670 Value *V = Builder.CreatePtrToInt(AddrMode.BaseGV, IntPtrTy, "sunkaddr"); 4671 if (Result) 4672 Result = Builder.CreateAdd(Result, V, "sunkaddr"); 4673 else 4674 Result = V; 4675 } 4676 4677 // Add in the Base Offset if present. 4678 if (AddrMode.BaseOffs) { 4679 Value *V = ConstantInt::get(IntPtrTy, AddrMode.BaseOffs); 4680 if (Result) 4681 Result = Builder.CreateAdd(Result, V, "sunkaddr"); 4682 else 4683 Result = V; 4684 } 4685 4686 if (!Result) 4687 SunkAddr = Constant::getNullValue(Addr->getType()); 4688 else 4689 SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr"); 4690 } 4691 4692 MemoryInst->replaceUsesOfWith(Repl, SunkAddr); 4693 // Store the newly computed address into the cache. In the case we reused a 4694 // value, this should be idempotent. 4695 SunkAddrs[Addr] = WeakTrackingVH(SunkAddr); 4696 4697 // If we have no uses, recursively delete the value and all dead instructions 4698 // using it. 4699 if (Repl->use_empty()) { 4700 // This can cause recursive deletion, which can invalidate our iterator. 4701 // Use a WeakTrackingVH to hold onto it in case this happens. 4702 Value *CurValue = &*CurInstIterator; 4703 WeakTrackingVH IterHandle(CurValue); 4704 BasicBlock *BB = CurInstIterator->getParent(); 4705 4706 RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo); 4707 4708 if (IterHandle != CurValue) { 4709 // If the iterator instruction was recursively deleted, start over at the 4710 // start of the block. 4711 CurInstIterator = BB->begin(); 4712 SunkAddrs.clear(); 4713 } 4714 } 4715 ++NumMemoryInsts; 4716 return true; 4717 } 4718 4719 /// If there are any memory operands, use OptimizeMemoryInst to sink their 4720 /// address computing into the block when possible / profitable. 4721 bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) { 4722 bool MadeChange = false; 4723 4724 const TargetRegisterInfo *TRI = 4725 TM->getSubtargetImpl(*CS->getFunction())->getRegisterInfo(); 4726 TargetLowering::AsmOperandInfoVector TargetConstraints = 4727 TLI->ParseConstraints(*DL, TRI, CS); 4728 unsigned ArgNo = 0; 4729 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) { 4730 TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i]; 4731 4732 // Compute the constraint code and ConstraintType to use. 4733 TLI->ComputeConstraintToUse(OpInfo, SDValue()); 4734 4735 if (OpInfo.ConstraintType == TargetLowering::C_Memory && 4736 OpInfo.isIndirect) { 4737 Value *OpVal = CS->getArgOperand(ArgNo++); 4738 MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u); 4739 } else if (OpInfo.Type == InlineAsm::isInput) 4740 ArgNo++; 4741 } 4742 4743 return MadeChange; 4744 } 4745 4746 /// Check if all the uses of \p Val are equivalent (or free) zero or 4747 /// sign extensions. 4748 static bool hasSameExtUse(Value *Val, const TargetLowering &TLI) { 4749 assert(!Val->use_empty() && "Input must have at least one use"); 4750 const Instruction *FirstUser = cast<Instruction>(*Val->user_begin()); 4751 bool IsSExt = isa<SExtInst>(FirstUser); 4752 Type *ExtTy = FirstUser->getType(); 4753 for (const User *U : Val->users()) { 4754 const Instruction *UI = cast<Instruction>(U); 4755 if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI))) 4756 return false; 4757 Type *CurTy = UI->getType(); 4758 // Same input and output types: Same instruction after CSE. 4759 if (CurTy == ExtTy) 4760 continue; 4761 4762 // If IsSExt is true, we are in this situation: 4763 // a = Val 4764 // b = sext ty1 a to ty2 4765 // c = sext ty1 a to ty3 4766 // Assuming ty2 is shorter than ty3, this could be turned into: 4767 // a = Val 4768 // b = sext ty1 a to ty2 4769 // c = sext ty2 b to ty3 4770 // However, the last sext is not free. 4771 if (IsSExt) 4772 return false; 4773 4774 // This is a ZExt, maybe this is free to extend from one type to another. 4775 // In that case, we would not account for a different use. 4776 Type *NarrowTy; 4777 Type *LargeTy; 4778 if (ExtTy->getScalarType()->getIntegerBitWidth() > 4779 CurTy->getScalarType()->getIntegerBitWidth()) { 4780 NarrowTy = CurTy; 4781 LargeTy = ExtTy; 4782 } else { 4783 NarrowTy = ExtTy; 4784 LargeTy = CurTy; 4785 } 4786 4787 if (!TLI.isZExtFree(NarrowTy, LargeTy)) 4788 return false; 4789 } 4790 // All uses are the same or can be derived from one another for free. 4791 return true; 4792 } 4793 4794 /// Try to speculatively promote extensions in \p Exts and continue 4795 /// promoting through newly promoted operands recursively as far as doing so is 4796 /// profitable. Save extensions profitably moved up, in \p ProfitablyMovedExts. 4797 /// When some promotion happened, \p TPT contains the proper state to revert 4798 /// them. 4799 /// 4800 /// \return true if some promotion happened, false otherwise. 4801 bool CodeGenPrepare::tryToPromoteExts( 4802 TypePromotionTransaction &TPT, const SmallVectorImpl<Instruction *> &Exts, 4803 SmallVectorImpl<Instruction *> &ProfitablyMovedExts, 4804 unsigned CreatedInstsCost) { 4805 bool Promoted = false; 4806 4807 // Iterate over all the extensions to try to promote them. 4808 for (auto I : Exts) { 4809 // Early check if we directly have ext(load). 4810 if (isa<LoadInst>(I->getOperand(0))) { 4811 ProfitablyMovedExts.push_back(I); 4812 continue; 4813 } 4814 4815 // Check whether or not we want to do any promotion. The reason we have 4816 // this check inside the for loop is to catch the case where an extension 4817 // is directly fed by a load because in such case the extension can be moved 4818 // up without any promotion on its operands. 4819 if (!TLI || !TLI->enableExtLdPromotion() || DisableExtLdPromotion) 4820 return false; 4821 4822 // Get the action to perform the promotion. 4823 TypePromotionHelper::Action TPH = 4824 TypePromotionHelper::getAction(I, InsertedInsts, *TLI, PromotedInsts); 4825 // Check if we can promote. 4826 if (!TPH) { 4827 // Save the current extension as we cannot move up through its operand. 4828 ProfitablyMovedExts.push_back(I); 4829 continue; 4830 } 4831 4832 // Save the current state. 4833 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 4834 TPT.getRestorationPoint(); 4835 SmallVector<Instruction *, 4> NewExts; 4836 unsigned NewCreatedInstsCost = 0; 4837 unsigned ExtCost = !TLI->isExtFree(I); 4838 // Promote. 4839 Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost, 4840 &NewExts, nullptr, *TLI); 4841 assert(PromotedVal && 4842 "TypePromotionHelper should have filtered out those cases"); 4843 4844 // We would be able to merge only one extension in a load. 4845 // Therefore, if we have more than 1 new extension we heuristically 4846 // cut this search path, because it means we degrade the code quality. 4847 // With exactly 2, the transformation is neutral, because we will merge 4848 // one extension but leave one. However, we optimistically keep going, 4849 // because the new extension may be removed too. 4850 long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost; 4851 // FIXME: It would be possible to propagate a negative value instead of 4852 // conservatively ceiling it to 0. 4853 TotalCreatedInstsCost = 4854 std::max((long long)0, (TotalCreatedInstsCost - ExtCost)); 4855 if (!StressExtLdPromotion && 4856 (TotalCreatedInstsCost > 1 || 4857 !isPromotedInstructionLegal(*TLI, *DL, PromotedVal))) { 4858 // This promotion is not profitable, rollback to the previous state, and 4859 // save the current extension in ProfitablyMovedExts as the latest 4860 // speculative promotion turned out to be unprofitable. 4861 TPT.rollback(LastKnownGood); 4862 ProfitablyMovedExts.push_back(I); 4863 continue; 4864 } 4865 // Continue promoting NewExts as far as doing so is profitable. 4866 SmallVector<Instruction *, 2> NewlyMovedExts; 4867 (void)tryToPromoteExts(TPT, NewExts, NewlyMovedExts, TotalCreatedInstsCost); 4868 bool NewPromoted = false; 4869 for (auto ExtInst : NewlyMovedExts) { 4870 Instruction *MovedExt = cast<Instruction>(ExtInst); 4871 Value *ExtOperand = MovedExt->getOperand(0); 4872 // If we have reached to a load, we need this extra profitability check 4873 // as it could potentially be merged into an ext(load). 4874 if (isa<LoadInst>(ExtOperand) && 4875 !(StressExtLdPromotion || NewCreatedInstsCost <= ExtCost || 4876 (ExtOperand->hasOneUse() || hasSameExtUse(ExtOperand, *TLI)))) 4877 continue; 4878 4879 ProfitablyMovedExts.push_back(MovedExt); 4880 NewPromoted = true; 4881 } 4882 4883 // If none of speculative promotions for NewExts is profitable, rollback 4884 // and save the current extension (I) as the last profitable extension. 4885 if (!NewPromoted) { 4886 TPT.rollback(LastKnownGood); 4887 ProfitablyMovedExts.push_back(I); 4888 continue; 4889 } 4890 // The promotion is profitable. 4891 Promoted = true; 4892 } 4893 return Promoted; 4894 } 4895 4896 /// Merging redundant sexts when one is dominating the other. 4897 bool CodeGenPrepare::mergeSExts(Function &F) { 4898 DominatorTree DT(F); 4899 bool Changed = false; 4900 for (auto &Entry : ValToSExtendedUses) { 4901 SExts &Insts = Entry.second; 4902 SExts CurPts; 4903 for (Instruction *Inst : Insts) { 4904 if (RemovedInsts.count(Inst) || !isa<SExtInst>(Inst) || 4905 Inst->getOperand(0) != Entry.first) 4906 continue; 4907 bool inserted = false; 4908 for (auto &Pt : CurPts) { 4909 if (DT.dominates(Inst, Pt)) { 4910 Pt->replaceAllUsesWith(Inst); 4911 RemovedInsts.insert(Pt); 4912 Pt->removeFromParent(); 4913 Pt = Inst; 4914 inserted = true; 4915 Changed = true; 4916 break; 4917 } 4918 if (!DT.dominates(Pt, Inst)) 4919 // Give up if we need to merge in a common dominator as the 4920 // experiments show it is not profitable. 4921 continue; 4922 Inst->replaceAllUsesWith(Pt); 4923 RemovedInsts.insert(Inst); 4924 Inst->removeFromParent(); 4925 inserted = true; 4926 Changed = true; 4927 break; 4928 } 4929 if (!inserted) 4930 CurPts.push_back(Inst); 4931 } 4932 } 4933 return Changed; 4934 } 4935 4936 // Spliting large data structures so that the GEPs accessing them can have 4937 // smaller offsets so that they can be sunk to the same blocks as their users. 4938 // For example, a large struct starting from %base is splitted into two parts 4939 // where the second part starts from %new_base. 4940 // 4941 // Before: 4942 // BB0: 4943 // %base = 4944 // 4945 // BB1: 4946 // %gep0 = gep %base, off0 4947 // %gep1 = gep %base, off1 4948 // %gep2 = gep %base, off2 4949 // 4950 // BB2: 4951 // %load1 = load %gep0 4952 // %load2 = load %gep1 4953 // %load3 = load %gep2 4954 // 4955 // After: 4956 // BB0: 4957 // %base = 4958 // %new_base = gep %base, off0 4959 // 4960 // BB1: 4961 // %new_gep0 = %new_base 4962 // %new_gep1 = gep %new_base, off1 - off0 4963 // %new_gep2 = gep %new_base, off2 - off0 4964 // 4965 // BB2: 4966 // %load1 = load i32, i32* %new_gep0 4967 // %load2 = load i32, i32* %new_gep1 4968 // %load3 = load i32, i32* %new_gep2 4969 // 4970 // %new_gep1 and %new_gep2 can be sunk to BB2 now after the splitting because 4971 // their offsets are smaller enough to fit into the addressing mode. 4972 bool CodeGenPrepare::splitLargeGEPOffsets() { 4973 bool Changed = false; 4974 for (auto &Entry : LargeOffsetGEPMap) { 4975 Value *OldBase = Entry.first; 4976 SmallVectorImpl<std::pair<AssertingVH<GetElementPtrInst>, int64_t>> 4977 &LargeOffsetGEPs = Entry.second; 4978 auto compareGEPOffset = 4979 [&](const std::pair<GetElementPtrInst *, int64_t> &LHS, 4980 const std::pair<GetElementPtrInst *, int64_t> &RHS) { 4981 if (LHS.first == RHS.first) 4982 return false; 4983 if (LHS.second != RHS.second) 4984 return LHS.second < RHS.second; 4985 return LargeOffsetGEPID[LHS.first] < LargeOffsetGEPID[RHS.first]; 4986 }; 4987 // Sorting all the GEPs of the same data structures based on the offsets. 4988 llvm::sort(LargeOffsetGEPs.begin(), LargeOffsetGEPs.end(), 4989 compareGEPOffset); 4990 LargeOffsetGEPs.erase( 4991 std::unique(LargeOffsetGEPs.begin(), LargeOffsetGEPs.end()), 4992 LargeOffsetGEPs.end()); 4993 // Skip if all the GEPs have the same offsets. 4994 if (LargeOffsetGEPs.front().second == LargeOffsetGEPs.back().second) 4995 continue; 4996 GetElementPtrInst *BaseGEP = LargeOffsetGEPs.begin()->first; 4997 int64_t BaseOffset = LargeOffsetGEPs.begin()->second; 4998 Value *NewBaseGEP = nullptr; 4999 5000 auto LargeOffsetGEP = LargeOffsetGEPs.begin(); 5001 while (LargeOffsetGEP != LargeOffsetGEPs.end()) { 5002 GetElementPtrInst *GEP = LargeOffsetGEP->first; 5003 int64_t Offset = LargeOffsetGEP->second; 5004 if (Offset != BaseOffset) { 5005 TargetLowering::AddrMode AddrMode; 5006 AddrMode.BaseOffs = Offset - BaseOffset; 5007 // The result type of the GEP might not be the type of the memory 5008 // access. 5009 if (!TLI->isLegalAddressingMode(*DL, AddrMode, 5010 GEP->getResultElementType(), 5011 GEP->getAddressSpace())) { 5012 // We need to create a new base if the offset to the current base is 5013 // too large to fit into the addressing mode. So, a very large struct 5014 // may be splitted into several parts. 5015 BaseGEP = GEP; 5016 BaseOffset = Offset; 5017 NewBaseGEP = nullptr; 5018 } 5019 } 5020 5021 // Generate a new GEP to replace the current one. 5022 IRBuilder<> Builder(GEP); 5023 Type *IntPtrTy = DL->getIntPtrType(GEP->getType()); 5024 Type *I8PtrTy = 5025 Builder.getInt8PtrTy(GEP->getType()->getPointerAddressSpace()); 5026 Type *I8Ty = Builder.getInt8Ty(); 5027 5028 if (!NewBaseGEP) { 5029 // Create a new base if we don't have one yet. Find the insertion 5030 // pointer for the new base first. 5031 BasicBlock::iterator NewBaseInsertPt; 5032 BasicBlock *NewBaseInsertBB; 5033 if (auto *BaseI = dyn_cast<Instruction>(OldBase)) { 5034 // If the base of the struct is an instruction, the new base will be 5035 // inserted close to it. 5036 NewBaseInsertBB = BaseI->getParent(); 5037 if (isa<PHINode>(BaseI)) 5038 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt(); 5039 else if (InvokeInst *Invoke = dyn_cast<InvokeInst>(BaseI)) { 5040 NewBaseInsertBB = 5041 SplitEdge(NewBaseInsertBB, Invoke->getNormalDest()); 5042 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt(); 5043 } else 5044 NewBaseInsertPt = std::next(BaseI->getIterator()); 5045 } else { 5046 // If the current base is an argument or global value, the new base 5047 // will be inserted to the entry block. 5048 NewBaseInsertBB = &BaseGEP->getFunction()->getEntryBlock(); 5049 NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt(); 5050 } 5051 IRBuilder<> NewBaseBuilder(NewBaseInsertBB, NewBaseInsertPt); 5052 // Create a new base. 5053 Value *BaseIndex = ConstantInt::get(IntPtrTy, BaseOffset); 5054 NewBaseGEP = OldBase; 5055 if (NewBaseGEP->getType() != I8PtrTy) 5056 NewBaseGEP = NewBaseBuilder.CreatePointerCast(NewBaseGEP, I8PtrTy); 5057 NewBaseGEP = 5058 NewBaseBuilder.CreateGEP(I8Ty, NewBaseGEP, BaseIndex, "splitgep"); 5059 NewGEPBases.insert(NewBaseGEP); 5060 } 5061 5062 Value *NewGEP = NewBaseGEP; 5063 if (Offset == BaseOffset) { 5064 if (GEP->getType() != I8PtrTy) 5065 NewGEP = Builder.CreatePointerCast(NewGEP, GEP->getType()); 5066 } else { 5067 // Calculate the new offset for the new GEP. 5068 Value *Index = ConstantInt::get(IntPtrTy, Offset - BaseOffset); 5069 NewGEP = Builder.CreateGEP(I8Ty, NewBaseGEP, Index); 5070 5071 if (GEP->getType() != I8PtrTy) 5072 NewGEP = Builder.CreatePointerCast(NewGEP, GEP->getType()); 5073 } 5074 GEP->replaceAllUsesWith(NewGEP); 5075 LargeOffsetGEPID.erase(GEP); 5076 LargeOffsetGEP = LargeOffsetGEPs.erase(LargeOffsetGEP); 5077 GEP->eraseFromParent(); 5078 Changed = true; 5079 } 5080 } 5081 return Changed; 5082 } 5083 5084 /// Return true, if an ext(load) can be formed from an extension in 5085 /// \p MovedExts. 5086 bool CodeGenPrepare::canFormExtLd( 5087 const SmallVectorImpl<Instruction *> &MovedExts, LoadInst *&LI, 5088 Instruction *&Inst, bool HasPromoted) { 5089 for (auto *MovedExtInst : MovedExts) { 5090 if (isa<LoadInst>(MovedExtInst->getOperand(0))) { 5091 LI = cast<LoadInst>(MovedExtInst->getOperand(0)); 5092 Inst = MovedExtInst; 5093 break; 5094 } 5095 } 5096 if (!LI) 5097 return false; 5098 5099 // If they're already in the same block, there's nothing to do. 5100 // Make the cheap checks first if we did not promote. 5101 // If we promoted, we need to check if it is indeed profitable. 5102 if (!HasPromoted && LI->getParent() == Inst->getParent()) 5103 return false; 5104 5105 return TLI->isExtLoad(LI, Inst, *DL); 5106 } 5107 5108 /// Move a zext or sext fed by a load into the same basic block as the load, 5109 /// unless conditions are unfavorable. This allows SelectionDAG to fold the 5110 /// extend into the load. 5111 /// 5112 /// E.g., 5113 /// \code 5114 /// %ld = load i32* %addr 5115 /// %add = add nuw i32 %ld, 4 5116 /// %zext = zext i32 %add to i64 5117 // \endcode 5118 /// => 5119 /// \code 5120 /// %ld = load i32* %addr 5121 /// %zext = zext i32 %ld to i64 5122 /// %add = add nuw i64 %zext, 4 5123 /// \encode 5124 /// Note that the promotion in %add to i64 is done in tryToPromoteExts(), which 5125 /// allow us to match zext(load i32*) to i64. 5126 /// 5127 /// Also, try to promote the computations used to obtain a sign extended 5128 /// value used into memory accesses. 5129 /// E.g., 5130 /// \code 5131 /// a = add nsw i32 b, 3 5132 /// d = sext i32 a to i64 5133 /// e = getelementptr ..., i64 d 5134 /// \endcode 5135 /// => 5136 /// \code 5137 /// f = sext i32 b to i64 5138 /// a = add nsw i64 f, 3 5139 /// e = getelementptr ..., i64 a 5140 /// \endcode 5141 /// 5142 /// \p Inst[in/out] the extension may be modified during the process if some 5143 /// promotions apply. 5144 bool CodeGenPrepare::optimizeExt(Instruction *&Inst) { 5145 // ExtLoad formation and address type promotion infrastructure requires TLI to 5146 // be effective. 5147 if (!TLI) 5148 return false; 5149 5150 bool AllowPromotionWithoutCommonHeader = false; 5151 /// See if it is an interesting sext operations for the address type 5152 /// promotion before trying to promote it, e.g., the ones with the right 5153 /// type and used in memory accesses. 5154 bool ATPConsiderable = TTI->shouldConsiderAddressTypePromotion( 5155 *Inst, AllowPromotionWithoutCommonHeader); 5156 TypePromotionTransaction TPT(RemovedInsts); 5157 TypePromotionTransaction::ConstRestorationPt LastKnownGood = 5158 TPT.getRestorationPoint(); 5159 SmallVector<Instruction *, 1> Exts; 5160 SmallVector<Instruction *, 2> SpeculativelyMovedExts; 5161 Exts.push_back(Inst); 5162 5163 bool HasPromoted = tryToPromoteExts(TPT, Exts, SpeculativelyMovedExts); 5164 5165 // Look for a load being extended. 5166 LoadInst *LI = nullptr; 5167 Instruction *ExtFedByLoad; 5168 5169 // Try to promote a chain of computation if it allows to form an extended 5170 // load. 5171 if (canFormExtLd(SpeculativelyMovedExts, LI, ExtFedByLoad, HasPromoted)) { 5172 assert(LI && ExtFedByLoad && "Expect a valid load and extension"); 5173 TPT.commit(); 5174 // Move the extend into the same block as the load 5175 ExtFedByLoad->moveAfter(LI); 5176 // CGP does not check if the zext would be speculatively executed when moved 5177 // to the same basic block as the load. Preserving its original location 5178 // would pessimize the debugging experience, as well as negatively impact 5179 // the quality of sample pgo. We don't want to use "line 0" as that has a 5180 // size cost in the line-table section and logically the zext can be seen as 5181 // part of the load. Therefore we conservatively reuse the same debug 5182 // location for the load and the zext. 5183 ExtFedByLoad->setDebugLoc(LI->getDebugLoc()); 5184 ++NumExtsMoved; 5185 Inst = ExtFedByLoad; 5186 return true; 5187 } 5188 5189 // Continue promoting SExts if known as considerable depending on targets. 5190 if (ATPConsiderable && 5191 performAddressTypePromotion(Inst, AllowPromotionWithoutCommonHeader, 5192 HasPromoted, TPT, SpeculativelyMovedExts)) 5193 return true; 5194 5195 TPT.rollback(LastKnownGood); 5196 return false; 5197 } 5198 5199 // Perform address type promotion if doing so is profitable. 5200 // If AllowPromotionWithoutCommonHeader == false, we should find other sext 5201 // instructions that sign extended the same initial value. However, if 5202 // AllowPromotionWithoutCommonHeader == true, we expect promoting the 5203 // extension is just profitable. 5204 bool CodeGenPrepare::performAddressTypePromotion( 5205 Instruction *&Inst, bool AllowPromotionWithoutCommonHeader, 5206 bool HasPromoted, TypePromotionTransaction &TPT, 5207 SmallVectorImpl<Instruction *> &SpeculativelyMovedExts) { 5208 bool Promoted = false; 5209 SmallPtrSet<Instruction *, 1> UnhandledExts; 5210 bool AllSeenFirst = true; 5211 for (auto I : SpeculativelyMovedExts) { 5212 Value *HeadOfChain = I->getOperand(0); 5213 DenseMap<Value *, Instruction *>::iterator AlreadySeen = 5214 SeenChainsForSExt.find(HeadOfChain); 5215 // If there is an unhandled SExt which has the same header, try to promote 5216 // it as well. 5217 if (AlreadySeen != SeenChainsForSExt.end()) { 5218 if (AlreadySeen->second != nullptr) 5219 UnhandledExts.insert(AlreadySeen->second); 5220 AllSeenFirst = false; 5221 } 5222 } 5223 5224 if (!AllSeenFirst || (AllowPromotionWithoutCommonHeader && 5225 SpeculativelyMovedExts.size() == 1)) { 5226 TPT.commit(); 5227 if (HasPromoted) 5228 Promoted = true; 5229 for (auto I : SpeculativelyMovedExts) { 5230 Value *HeadOfChain = I->getOperand(0); 5231 SeenChainsForSExt[HeadOfChain] = nullptr; 5232 ValToSExtendedUses[HeadOfChain].push_back(I); 5233 } 5234 // Update Inst as promotion happen. 5235 Inst = SpeculativelyMovedExts.pop_back_val(); 5236 } else { 5237 // This is the first chain visited from the header, keep the current chain 5238 // as unhandled. Defer to promote this until we encounter another SExt 5239 // chain derived from the same header. 5240 for (auto I : SpeculativelyMovedExts) { 5241 Value *HeadOfChain = I->getOperand(0); 5242 SeenChainsForSExt[HeadOfChain] = Inst; 5243 } 5244 return false; 5245 } 5246 5247 if (!AllSeenFirst && !UnhandledExts.empty()) 5248 for (auto VisitedSExt : UnhandledExts) { 5249 if (RemovedInsts.count(VisitedSExt)) 5250 continue; 5251 TypePromotionTransaction TPT(RemovedInsts); 5252 SmallVector<Instruction *, 1> Exts; 5253 SmallVector<Instruction *, 2> Chains; 5254 Exts.push_back(VisitedSExt); 5255 bool HasPromoted = tryToPromoteExts(TPT, Exts, Chains); 5256 TPT.commit(); 5257 if (HasPromoted) 5258 Promoted = true; 5259 for (auto I : Chains) { 5260 Value *HeadOfChain = I->getOperand(0); 5261 // Mark this as handled. 5262 SeenChainsForSExt[HeadOfChain] = nullptr; 5263 ValToSExtendedUses[HeadOfChain].push_back(I); 5264 } 5265 } 5266 return Promoted; 5267 } 5268 5269 bool CodeGenPrepare::optimizeExtUses(Instruction *I) { 5270 BasicBlock *DefBB = I->getParent(); 5271 5272 // If the result of a {s|z}ext and its source are both live out, rewrite all 5273 // other uses of the source with result of extension. 5274 Value *Src = I->getOperand(0); 5275 if (Src->hasOneUse()) 5276 return false; 5277 5278 // Only do this xform if truncating is free. 5279 if (TLI && !TLI->isTruncateFree(I->getType(), Src->getType())) 5280 return false; 5281 5282 // Only safe to perform the optimization if the source is also defined in 5283 // this block. 5284 if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent()) 5285 return false; 5286 5287 bool DefIsLiveOut = false; 5288 for (User *U : I->users()) { 5289 Instruction *UI = cast<Instruction>(U); 5290 5291 // Figure out which BB this ext is used in. 5292 BasicBlock *UserBB = UI->getParent(); 5293 if (UserBB == DefBB) continue; 5294 DefIsLiveOut = true; 5295 break; 5296 } 5297 if (!DefIsLiveOut) 5298 return false; 5299 5300 // Make sure none of the uses are PHI nodes. 5301 for (User *U : Src->users()) { 5302 Instruction *UI = cast<Instruction>(U); 5303 BasicBlock *UserBB = UI->getParent(); 5304 if (UserBB == DefBB) continue; 5305 // Be conservative. We don't want this xform to end up introducing 5306 // reloads just before load / store instructions. 5307 if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI)) 5308 return false; 5309 } 5310 5311 // InsertedTruncs - Only insert one trunc in each block once. 5312 DenseMap<BasicBlock*, Instruction*> InsertedTruncs; 5313 5314 bool MadeChange = false; 5315 for (Use &U : Src->uses()) { 5316 Instruction *User = cast<Instruction>(U.getUser()); 5317 5318 // Figure out which BB this ext is used in. 5319 BasicBlock *UserBB = User->getParent(); 5320 if (UserBB == DefBB) continue; 5321 5322 // Both src and def are live in this block. Rewrite the use. 5323 Instruction *&InsertedTrunc = InsertedTruncs[UserBB]; 5324 5325 if (!InsertedTrunc) { 5326 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 5327 assert(InsertPt != UserBB->end()); 5328 InsertedTrunc = new TruncInst(I, Src->getType(), "", &*InsertPt); 5329 InsertedInsts.insert(InsertedTrunc); 5330 } 5331 5332 // Replace a use of the {s|z}ext source with a use of the result. 5333 U = InsertedTrunc; 5334 ++NumExtUses; 5335 MadeChange = true; 5336 } 5337 5338 return MadeChange; 5339 } 5340 5341 // Find loads whose uses only use some of the loaded value's bits. Add an "and" 5342 // just after the load if the target can fold this into one extload instruction, 5343 // with the hope of eliminating some of the other later "and" instructions using 5344 // the loaded value. "and"s that are made trivially redundant by the insertion 5345 // of the new "and" are removed by this function, while others (e.g. those whose 5346 // path from the load goes through a phi) are left for isel to potentially 5347 // remove. 5348 // 5349 // For example: 5350 // 5351 // b0: 5352 // x = load i32 5353 // ... 5354 // b1: 5355 // y = and x, 0xff 5356 // z = use y 5357 // 5358 // becomes: 5359 // 5360 // b0: 5361 // x = load i32 5362 // x' = and x, 0xff 5363 // ... 5364 // b1: 5365 // z = use x' 5366 // 5367 // whereas: 5368 // 5369 // b0: 5370 // x1 = load i32 5371 // ... 5372 // b1: 5373 // x2 = load i32 5374 // ... 5375 // b2: 5376 // x = phi x1, x2 5377 // y = and x, 0xff 5378 // 5379 // becomes (after a call to optimizeLoadExt for each load): 5380 // 5381 // b0: 5382 // x1 = load i32 5383 // x1' = and x1, 0xff 5384 // ... 5385 // b1: 5386 // x2 = load i32 5387 // x2' = and x2, 0xff 5388 // ... 5389 // b2: 5390 // x = phi x1', x2' 5391 // y = and x, 0xff 5392 bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) { 5393 if (!Load->isSimple() || !Load->getType()->isIntOrPtrTy()) 5394 return false; 5395 5396 // Skip loads we've already transformed. 5397 if (Load->hasOneUse() && 5398 InsertedInsts.count(cast<Instruction>(*Load->user_begin()))) 5399 return false; 5400 5401 // Look at all uses of Load, looking through phis, to determine how many bits 5402 // of the loaded value are needed. 5403 SmallVector<Instruction *, 8> WorkList; 5404 SmallPtrSet<Instruction *, 16> Visited; 5405 SmallVector<Instruction *, 8> AndsToMaybeRemove; 5406 for (auto *U : Load->users()) 5407 WorkList.push_back(cast<Instruction>(U)); 5408 5409 EVT LoadResultVT = TLI->getValueType(*DL, Load->getType()); 5410 unsigned BitWidth = LoadResultVT.getSizeInBits(); 5411 APInt DemandBits(BitWidth, 0); 5412 APInt WidestAndBits(BitWidth, 0); 5413 5414 while (!WorkList.empty()) { 5415 Instruction *I = WorkList.back(); 5416 WorkList.pop_back(); 5417 5418 // Break use-def graph loops. 5419 if (!Visited.insert(I).second) 5420 continue; 5421 5422 // For a PHI node, push all of its users. 5423 if (auto *Phi = dyn_cast<PHINode>(I)) { 5424 for (auto *U : Phi->users()) 5425 WorkList.push_back(cast<Instruction>(U)); 5426 continue; 5427 } 5428 5429 switch (I->getOpcode()) { 5430 case Instruction::And: { 5431 auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1)); 5432 if (!AndC) 5433 return false; 5434 APInt AndBits = AndC->getValue(); 5435 DemandBits |= AndBits; 5436 // Keep track of the widest and mask we see. 5437 if (AndBits.ugt(WidestAndBits)) 5438 WidestAndBits = AndBits; 5439 if (AndBits == WidestAndBits && I->getOperand(0) == Load) 5440 AndsToMaybeRemove.push_back(I); 5441 break; 5442 } 5443 5444 case Instruction::Shl: { 5445 auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1)); 5446 if (!ShlC) 5447 return false; 5448 uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1); 5449 DemandBits.setLowBits(BitWidth - ShiftAmt); 5450 break; 5451 } 5452 5453 case Instruction::Trunc: { 5454 EVT TruncVT = TLI->getValueType(*DL, I->getType()); 5455 unsigned TruncBitWidth = TruncVT.getSizeInBits(); 5456 DemandBits.setLowBits(TruncBitWidth); 5457 break; 5458 } 5459 5460 default: 5461 return false; 5462 } 5463 } 5464 5465 uint32_t ActiveBits = DemandBits.getActiveBits(); 5466 // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the 5467 // target even if isLoadExtLegal says an i1 EXTLOAD is valid. For example, 5468 // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but 5469 // (and (load x) 1) is not matched as a single instruction, rather as a LDR 5470 // followed by an AND. 5471 // TODO: Look into removing this restriction by fixing backends to either 5472 // return false for isLoadExtLegal for i1 or have them select this pattern to 5473 // a single instruction. 5474 // 5475 // Also avoid hoisting if we didn't see any ands with the exact DemandBits 5476 // mask, since these are the only ands that will be removed by isel. 5477 if (ActiveBits <= 1 || !DemandBits.isMask(ActiveBits) || 5478 WidestAndBits != DemandBits) 5479 return false; 5480 5481 LLVMContext &Ctx = Load->getType()->getContext(); 5482 Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits); 5483 EVT TruncVT = TLI->getValueType(*DL, TruncTy); 5484 5485 // Reject cases that won't be matched as extloads. 5486 if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() || 5487 !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT)) 5488 return false; 5489 5490 IRBuilder<> Builder(Load->getNextNode()); 5491 auto *NewAnd = dyn_cast<Instruction>( 5492 Builder.CreateAnd(Load, ConstantInt::get(Ctx, DemandBits))); 5493 // Mark this instruction as "inserted by CGP", so that other 5494 // optimizations don't touch it. 5495 InsertedInsts.insert(NewAnd); 5496 5497 // Replace all uses of load with new and (except for the use of load in the 5498 // new and itself). 5499 Load->replaceAllUsesWith(NewAnd); 5500 NewAnd->setOperand(0, Load); 5501 5502 // Remove any and instructions that are now redundant. 5503 for (auto *And : AndsToMaybeRemove) 5504 // Check that the and mask is the same as the one we decided to put on the 5505 // new and. 5506 if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) { 5507 And->replaceAllUsesWith(NewAnd); 5508 if (&*CurInstIterator == And) 5509 CurInstIterator = std::next(And->getIterator()); 5510 And->eraseFromParent(); 5511 ++NumAndUses; 5512 } 5513 5514 ++NumAndsAdded; 5515 return true; 5516 } 5517 5518 /// Check if V (an operand of a select instruction) is an expensive instruction 5519 /// that is only used once. 5520 static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V) { 5521 auto *I = dyn_cast<Instruction>(V); 5522 // If it's safe to speculatively execute, then it should not have side 5523 // effects; therefore, it's safe to sink and possibly *not* execute. 5524 return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) && 5525 TTI->getUserCost(I) >= TargetTransformInfo::TCC_Expensive; 5526 } 5527 5528 /// Returns true if a SelectInst should be turned into an explicit branch. 5529 static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI, 5530 const TargetLowering *TLI, 5531 SelectInst *SI) { 5532 // If even a predictable select is cheap, then a branch can't be cheaper. 5533 if (!TLI->isPredictableSelectExpensive()) 5534 return false; 5535 5536 // FIXME: This should use the same heuristics as IfConversion to determine 5537 // whether a select is better represented as a branch. 5538 5539 // If metadata tells us that the select condition is obviously predictable, 5540 // then we want to replace the select with a branch. 5541 uint64_t TrueWeight, FalseWeight; 5542 if (SI->extractProfMetadata(TrueWeight, FalseWeight)) { 5543 uint64_t Max = std::max(TrueWeight, FalseWeight); 5544 uint64_t Sum = TrueWeight + FalseWeight; 5545 if (Sum != 0) { 5546 auto Probability = BranchProbability::getBranchProbability(Max, Sum); 5547 if (Probability > TLI->getPredictableBranchThreshold()) 5548 return true; 5549 } 5550 } 5551 5552 CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition()); 5553 5554 // If a branch is predictable, an out-of-order CPU can avoid blocking on its 5555 // comparison condition. If the compare has more than one use, there's 5556 // probably another cmov or setcc around, so it's not worth emitting a branch. 5557 if (!Cmp || !Cmp->hasOneUse()) 5558 return false; 5559 5560 // If either operand of the select is expensive and only needed on one side 5561 // of the select, we should form a branch. 5562 if (sinkSelectOperand(TTI, SI->getTrueValue()) || 5563 sinkSelectOperand(TTI, SI->getFalseValue())) 5564 return true; 5565 5566 return false; 5567 } 5568 5569 /// If \p isTrue is true, return the true value of \p SI, otherwise return 5570 /// false value of \p SI. If the true/false value of \p SI is defined by any 5571 /// select instructions in \p Selects, look through the defining select 5572 /// instruction until the true/false value is not defined in \p Selects. 5573 static Value *getTrueOrFalseValue( 5574 SelectInst *SI, bool isTrue, 5575 const SmallPtrSet<const Instruction *, 2> &Selects) { 5576 Value *V; 5577 5578 for (SelectInst *DefSI = SI; DefSI != nullptr && Selects.count(DefSI); 5579 DefSI = dyn_cast<SelectInst>(V)) { 5580 assert(DefSI->getCondition() == SI->getCondition() && 5581 "The condition of DefSI does not match with SI"); 5582 V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue()); 5583 } 5584 return V; 5585 } 5586 5587 /// If we have a SelectInst that will likely profit from branch prediction, 5588 /// turn it into a branch. 5589 bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) { 5590 // Find all consecutive select instructions that share the same condition. 5591 SmallVector<SelectInst *, 2> ASI; 5592 ASI.push_back(SI); 5593 for (BasicBlock::iterator It = ++BasicBlock::iterator(SI); 5594 It != SI->getParent()->end(); ++It) { 5595 SelectInst *I = dyn_cast<SelectInst>(&*It); 5596 if (I && SI->getCondition() == I->getCondition()) { 5597 ASI.push_back(I); 5598 } else { 5599 break; 5600 } 5601 } 5602 5603 SelectInst *LastSI = ASI.back(); 5604 // Increment the current iterator to skip all the rest of select instructions 5605 // because they will be either "not lowered" or "all lowered" to branch. 5606 CurInstIterator = std::next(LastSI->getIterator()); 5607 5608 bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1); 5609 5610 // Can we convert the 'select' to CF ? 5611 if (DisableSelectToBranch || OptSize || !TLI || VectorCond || 5612 SI->getMetadata(LLVMContext::MD_unpredictable)) 5613 return false; 5614 5615 TargetLowering::SelectSupportKind SelectKind; 5616 if (VectorCond) 5617 SelectKind = TargetLowering::VectorMaskSelect; 5618 else if (SI->getType()->isVectorTy()) 5619 SelectKind = TargetLowering::ScalarCondVectorVal; 5620 else 5621 SelectKind = TargetLowering::ScalarValSelect; 5622 5623 if (TLI->isSelectSupported(SelectKind) && 5624 !isFormingBranchFromSelectProfitable(TTI, TLI, SI)) 5625 return false; 5626 5627 ModifiedDT = true; 5628 5629 // Transform a sequence like this: 5630 // start: 5631 // %cmp = cmp uge i32 %a, %b 5632 // %sel = select i1 %cmp, i32 %c, i32 %d 5633 // 5634 // Into: 5635 // start: 5636 // %cmp = cmp uge i32 %a, %b 5637 // br i1 %cmp, label %select.true, label %select.false 5638 // select.true: 5639 // br label %select.end 5640 // select.false: 5641 // br label %select.end 5642 // select.end: 5643 // %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ] 5644 // 5645 // In addition, we may sink instructions that produce %c or %d from 5646 // the entry block into the destination(s) of the new branch. 5647 // If the true or false blocks do not contain a sunken instruction, that 5648 // block and its branch may be optimized away. In that case, one side of the 5649 // first branch will point directly to select.end, and the corresponding PHI 5650 // predecessor block will be the start block. 5651 5652 // First, we split the block containing the select into 2 blocks. 5653 BasicBlock *StartBlock = SI->getParent(); 5654 BasicBlock::iterator SplitPt = ++(BasicBlock::iterator(LastSI)); 5655 BasicBlock *EndBlock = StartBlock->splitBasicBlock(SplitPt, "select.end"); 5656 5657 // Delete the unconditional branch that was just created by the split. 5658 StartBlock->getTerminator()->eraseFromParent(); 5659 5660 // These are the new basic blocks for the conditional branch. 5661 // At least one will become an actual new basic block. 5662 BasicBlock *TrueBlock = nullptr; 5663 BasicBlock *FalseBlock = nullptr; 5664 BranchInst *TrueBranch = nullptr; 5665 BranchInst *FalseBranch = nullptr; 5666 5667 // Sink expensive instructions into the conditional blocks to avoid executing 5668 // them speculatively. 5669 for (SelectInst *SI : ASI) { 5670 if (sinkSelectOperand(TTI, SI->getTrueValue())) { 5671 if (TrueBlock == nullptr) { 5672 TrueBlock = BasicBlock::Create(SI->getContext(), "select.true.sink", 5673 EndBlock->getParent(), EndBlock); 5674 TrueBranch = BranchInst::Create(EndBlock, TrueBlock); 5675 } 5676 auto *TrueInst = cast<Instruction>(SI->getTrueValue()); 5677 TrueInst->moveBefore(TrueBranch); 5678 } 5679 if (sinkSelectOperand(TTI, SI->getFalseValue())) { 5680 if (FalseBlock == nullptr) { 5681 FalseBlock = BasicBlock::Create(SI->getContext(), "select.false.sink", 5682 EndBlock->getParent(), EndBlock); 5683 FalseBranch = BranchInst::Create(EndBlock, FalseBlock); 5684 } 5685 auto *FalseInst = cast<Instruction>(SI->getFalseValue()); 5686 FalseInst->moveBefore(FalseBranch); 5687 } 5688 } 5689 5690 // If there was nothing to sink, then arbitrarily choose the 'false' side 5691 // for a new input value to the PHI. 5692 if (TrueBlock == FalseBlock) { 5693 assert(TrueBlock == nullptr && 5694 "Unexpected basic block transform while optimizing select"); 5695 5696 FalseBlock = BasicBlock::Create(SI->getContext(), "select.false", 5697 EndBlock->getParent(), EndBlock); 5698 BranchInst::Create(EndBlock, FalseBlock); 5699 } 5700 5701 // Insert the real conditional branch based on the original condition. 5702 // If we did not create a new block for one of the 'true' or 'false' paths 5703 // of the condition, it means that side of the branch goes to the end block 5704 // directly and the path originates from the start block from the point of 5705 // view of the new PHI. 5706 BasicBlock *TT, *FT; 5707 if (TrueBlock == nullptr) { 5708 TT = EndBlock; 5709 FT = FalseBlock; 5710 TrueBlock = StartBlock; 5711 } else if (FalseBlock == nullptr) { 5712 TT = TrueBlock; 5713 FT = EndBlock; 5714 FalseBlock = StartBlock; 5715 } else { 5716 TT = TrueBlock; 5717 FT = FalseBlock; 5718 } 5719 IRBuilder<>(SI).CreateCondBr(SI->getCondition(), TT, FT, SI); 5720 5721 SmallPtrSet<const Instruction *, 2> INS; 5722 INS.insert(ASI.begin(), ASI.end()); 5723 // Use reverse iterator because later select may use the value of the 5724 // earlier select, and we need to propagate value through earlier select 5725 // to get the PHI operand. 5726 for (auto It = ASI.rbegin(); It != ASI.rend(); ++It) { 5727 SelectInst *SI = *It; 5728 // The select itself is replaced with a PHI Node. 5729 PHINode *PN = PHINode::Create(SI->getType(), 2, "", &EndBlock->front()); 5730 PN->takeName(SI); 5731 PN->addIncoming(getTrueOrFalseValue(SI, true, INS), TrueBlock); 5732 PN->addIncoming(getTrueOrFalseValue(SI, false, INS), FalseBlock); 5733 5734 SI->replaceAllUsesWith(PN); 5735 SI->eraseFromParent(); 5736 INS.erase(SI); 5737 ++NumSelectsExpanded; 5738 } 5739 5740 // Instruct OptimizeBlock to skip to the next block. 5741 CurInstIterator = StartBlock->end(); 5742 return true; 5743 } 5744 5745 static bool isBroadcastShuffle(ShuffleVectorInst *SVI) { 5746 SmallVector<int, 16> Mask(SVI->getShuffleMask()); 5747 int SplatElem = -1; 5748 for (unsigned i = 0; i < Mask.size(); ++i) { 5749 if (SplatElem != -1 && Mask[i] != -1 && Mask[i] != SplatElem) 5750 return false; 5751 SplatElem = Mask[i]; 5752 } 5753 5754 return true; 5755 } 5756 5757 /// Some targets have expensive vector shifts if the lanes aren't all the same 5758 /// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases 5759 /// it's often worth sinking a shufflevector splat down to its use so that 5760 /// codegen can spot all lanes are identical. 5761 bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) { 5762 BasicBlock *DefBB = SVI->getParent(); 5763 5764 // Only do this xform if variable vector shifts are particularly expensive. 5765 if (!TLI || !TLI->isVectorShiftByScalarCheap(SVI->getType())) 5766 return false; 5767 5768 // We only expect better codegen by sinking a shuffle if we can recognise a 5769 // constant splat. 5770 if (!isBroadcastShuffle(SVI)) 5771 return false; 5772 5773 // InsertedShuffles - Only insert a shuffle in each block once. 5774 DenseMap<BasicBlock*, Instruction*> InsertedShuffles; 5775 5776 bool MadeChange = false; 5777 for (User *U : SVI->users()) { 5778 Instruction *UI = cast<Instruction>(U); 5779 5780 // Figure out which BB this ext is used in. 5781 BasicBlock *UserBB = UI->getParent(); 5782 if (UserBB == DefBB) continue; 5783 5784 // For now only apply this when the splat is used by a shift instruction. 5785 if (!UI->isShift()) continue; 5786 5787 // Everything checks out, sink the shuffle if the user's block doesn't 5788 // already have a copy. 5789 Instruction *&InsertedShuffle = InsertedShuffles[UserBB]; 5790 5791 if (!InsertedShuffle) { 5792 BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); 5793 assert(InsertPt != UserBB->end()); 5794 InsertedShuffle = 5795 new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1), 5796 SVI->getOperand(2), "", &*InsertPt); 5797 } 5798 5799 UI->replaceUsesOfWith(SVI, InsertedShuffle); 5800 MadeChange = true; 5801 } 5802 5803 // If we removed all uses, nuke the shuffle. 5804 if (SVI->use_empty()) { 5805 SVI->eraseFromParent(); 5806 MadeChange = true; 5807 } 5808 5809 return MadeChange; 5810 } 5811 5812 bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) { 5813 if (!TLI || !DL) 5814 return false; 5815 5816 Value *Cond = SI->getCondition(); 5817 Type *OldType = Cond->getType(); 5818 LLVMContext &Context = Cond->getContext(); 5819 MVT RegType = TLI->getRegisterType(Context, TLI->getValueType(*DL, OldType)); 5820 unsigned RegWidth = RegType.getSizeInBits(); 5821 5822 if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth()) 5823 return false; 5824 5825 // If the register width is greater than the type width, expand the condition 5826 // of the switch instruction and each case constant to the width of the 5827 // register. By widening the type of the switch condition, subsequent 5828 // comparisons (for case comparisons) will not need to be extended to the 5829 // preferred register width, so we will potentially eliminate N-1 extends, 5830 // where N is the number of cases in the switch. 5831 auto *NewType = Type::getIntNTy(Context, RegWidth); 5832 5833 // Zero-extend the switch condition and case constants unless the switch 5834 // condition is a function argument that is already being sign-extended. 5835 // In that case, we can avoid an unnecessary mask/extension by sign-extending 5836 // everything instead. 5837 Instruction::CastOps ExtType = Instruction::ZExt; 5838 if (auto *Arg = dyn_cast<Argument>(Cond)) 5839 if (Arg->hasSExtAttr()) 5840 ExtType = Instruction::SExt; 5841 5842 auto *ExtInst = CastInst::Create(ExtType, Cond, NewType); 5843 ExtInst->insertBefore(SI); 5844 SI->setCondition(ExtInst); 5845 for (auto Case : SI->cases()) { 5846 APInt NarrowConst = Case.getCaseValue()->getValue(); 5847 APInt WideConst = (ExtType == Instruction::ZExt) ? 5848 NarrowConst.zext(RegWidth) : NarrowConst.sext(RegWidth); 5849 Case.setValue(ConstantInt::get(Context, WideConst)); 5850 } 5851 5852 return true; 5853 } 5854 5855 5856 namespace { 5857 5858 /// Helper class to promote a scalar operation to a vector one. 5859 /// This class is used to move downward extractelement transition. 5860 /// E.g., 5861 /// a = vector_op <2 x i32> 5862 /// b = extractelement <2 x i32> a, i32 0 5863 /// c = scalar_op b 5864 /// store c 5865 /// 5866 /// => 5867 /// a = vector_op <2 x i32> 5868 /// c = vector_op a (equivalent to scalar_op on the related lane) 5869 /// * d = extractelement <2 x i32> c, i32 0 5870 /// * store d 5871 /// Assuming both extractelement and store can be combine, we get rid of the 5872 /// transition. 5873 class VectorPromoteHelper { 5874 /// DataLayout associated with the current module. 5875 const DataLayout &DL; 5876 5877 /// Used to perform some checks on the legality of vector operations. 5878 const TargetLowering &TLI; 5879 5880 /// Used to estimated the cost of the promoted chain. 5881 const TargetTransformInfo &TTI; 5882 5883 /// The transition being moved downwards. 5884 Instruction *Transition; 5885 5886 /// The sequence of instructions to be promoted. 5887 SmallVector<Instruction *, 4> InstsToBePromoted; 5888 5889 /// Cost of combining a store and an extract. 5890 unsigned StoreExtractCombineCost; 5891 5892 /// Instruction that will be combined with the transition. 5893 Instruction *CombineInst = nullptr; 5894 5895 /// The instruction that represents the current end of the transition. 5896 /// Since we are faking the promotion until we reach the end of the chain 5897 /// of computation, we need a way to get the current end of the transition. 5898 Instruction *getEndOfTransition() const { 5899 if (InstsToBePromoted.empty()) 5900 return Transition; 5901 return InstsToBePromoted.back(); 5902 } 5903 5904 /// Return the index of the original value in the transition. 5905 /// E.g., for "extractelement <2 x i32> c, i32 1" the original value, 5906 /// c, is at index 0. 5907 unsigned getTransitionOriginalValueIdx() const { 5908 assert(isa<ExtractElementInst>(Transition) && 5909 "Other kind of transitions are not supported yet"); 5910 return 0; 5911 } 5912 5913 /// Return the index of the index in the transition. 5914 /// E.g., for "extractelement <2 x i32> c, i32 0" the index 5915 /// is at index 1. 5916 unsigned getTransitionIdx() const { 5917 assert(isa<ExtractElementInst>(Transition) && 5918 "Other kind of transitions are not supported yet"); 5919 return 1; 5920 } 5921 5922 /// Get the type of the transition. 5923 /// This is the type of the original value. 5924 /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the 5925 /// transition is <2 x i32>. 5926 Type *getTransitionType() const { 5927 return Transition->getOperand(getTransitionOriginalValueIdx())->getType(); 5928 } 5929 5930 /// Promote \p ToBePromoted by moving \p Def downward through. 5931 /// I.e., we have the following sequence: 5932 /// Def = Transition <ty1> a to <ty2> 5933 /// b = ToBePromoted <ty2> Def, ... 5934 /// => 5935 /// b = ToBePromoted <ty1> a, ... 5936 /// Def = Transition <ty1> ToBePromoted to <ty2> 5937 void promoteImpl(Instruction *ToBePromoted); 5938 5939 /// Check whether or not it is profitable to promote all the 5940 /// instructions enqueued to be promoted. 5941 bool isProfitableToPromote() { 5942 Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx()); 5943 unsigned Index = isa<ConstantInt>(ValIdx) 5944 ? cast<ConstantInt>(ValIdx)->getZExtValue() 5945 : -1; 5946 Type *PromotedType = getTransitionType(); 5947 5948 StoreInst *ST = cast<StoreInst>(CombineInst); 5949 unsigned AS = ST->getPointerAddressSpace(); 5950 unsigned Align = ST->getAlignment(); 5951 // Check if this store is supported. 5952 if (!TLI.allowsMisalignedMemoryAccesses( 5953 TLI.getValueType(DL, ST->getValueOperand()->getType()), AS, 5954 Align)) { 5955 // If this is not supported, there is no way we can combine 5956 // the extract with the store. 5957 return false; 5958 } 5959 5960 // The scalar chain of computation has to pay for the transition 5961 // scalar to vector. 5962 // The vector chain has to account for the combining cost. 5963 uint64_t ScalarCost = 5964 TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index); 5965 uint64_t VectorCost = StoreExtractCombineCost; 5966 for (const auto &Inst : InstsToBePromoted) { 5967 // Compute the cost. 5968 // By construction, all instructions being promoted are arithmetic ones. 5969 // Moreover, one argument is a constant that can be viewed as a splat 5970 // constant. 5971 Value *Arg0 = Inst->getOperand(0); 5972 bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) || 5973 isa<ConstantFP>(Arg0); 5974 TargetTransformInfo::OperandValueKind Arg0OVK = 5975 IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue 5976 : TargetTransformInfo::OK_AnyValue; 5977 TargetTransformInfo::OperandValueKind Arg1OVK = 5978 !IsArg0Constant ? TargetTransformInfo::OK_UniformConstantValue 5979 : TargetTransformInfo::OK_AnyValue; 5980 ScalarCost += TTI.getArithmeticInstrCost( 5981 Inst->getOpcode(), Inst->getType(), Arg0OVK, Arg1OVK); 5982 VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType, 5983 Arg0OVK, Arg1OVK); 5984 } 5985 LLVM_DEBUG( 5986 dbgs() << "Estimated cost of computation to be promoted:\nScalar: " 5987 << ScalarCost << "\nVector: " << VectorCost << '\n'); 5988 return ScalarCost > VectorCost; 5989 } 5990 5991 /// Generate a constant vector with \p Val with the same 5992 /// number of elements as the transition. 5993 /// \p UseSplat defines whether or not \p Val should be replicated 5994 /// across the whole vector. 5995 /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>, 5996 /// otherwise we generate a vector with as many undef as possible: 5997 /// <undef, ..., undef, Val, undef, ..., undef> where \p Val is only 5998 /// used at the index of the extract. 5999 Value *getConstantVector(Constant *Val, bool UseSplat) const { 6000 unsigned ExtractIdx = std::numeric_limits<unsigned>::max(); 6001 if (!UseSplat) { 6002 // If we cannot determine where the constant must be, we have to 6003 // use a splat constant. 6004 Value *ValExtractIdx = Transition->getOperand(getTransitionIdx()); 6005 if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx)) 6006 ExtractIdx = CstVal->getSExtValue(); 6007 else 6008 UseSplat = true; 6009 } 6010 6011 unsigned End = getTransitionType()->getVectorNumElements(); 6012 if (UseSplat) 6013 return ConstantVector::getSplat(End, Val); 6014 6015 SmallVector<Constant *, 4> ConstVec; 6016 UndefValue *UndefVal = UndefValue::get(Val->getType()); 6017 for (unsigned Idx = 0; Idx != End; ++Idx) { 6018 if (Idx == ExtractIdx) 6019 ConstVec.push_back(Val); 6020 else 6021 ConstVec.push_back(UndefVal); 6022 } 6023 return ConstantVector::get(ConstVec); 6024 } 6025 6026 /// Check if promoting to a vector type an operand at \p OperandIdx 6027 /// in \p Use can trigger undefined behavior. 6028 static bool canCauseUndefinedBehavior(const Instruction *Use, 6029 unsigned OperandIdx) { 6030 // This is not safe to introduce undef when the operand is on 6031 // the right hand side of a division-like instruction. 6032 if (OperandIdx != 1) 6033 return false; 6034 switch (Use->getOpcode()) { 6035 default: 6036 return false; 6037 case Instruction::SDiv: 6038 case Instruction::UDiv: 6039 case Instruction::SRem: 6040 case Instruction::URem: 6041 return true; 6042 case Instruction::FDiv: 6043 case Instruction::FRem: 6044 return !Use->hasNoNaNs(); 6045 } 6046 llvm_unreachable(nullptr); 6047 } 6048 6049 public: 6050 VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI, 6051 const TargetTransformInfo &TTI, Instruction *Transition, 6052 unsigned CombineCost) 6053 : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition), 6054 StoreExtractCombineCost(CombineCost) { 6055 assert(Transition && "Do not know how to promote null"); 6056 } 6057 6058 /// Check if we can promote \p ToBePromoted to \p Type. 6059 bool canPromote(const Instruction *ToBePromoted) const { 6060 // We could support CastInst too. 6061 return isa<BinaryOperator>(ToBePromoted); 6062 } 6063 6064 /// Check if it is profitable to promote \p ToBePromoted 6065 /// by moving downward the transition through. 6066 bool shouldPromote(const Instruction *ToBePromoted) const { 6067 // Promote only if all the operands can be statically expanded. 6068 // Indeed, we do not want to introduce any new kind of transitions. 6069 for (const Use &U : ToBePromoted->operands()) { 6070 const Value *Val = U.get(); 6071 if (Val == getEndOfTransition()) { 6072 // If the use is a division and the transition is on the rhs, 6073 // we cannot promote the operation, otherwise we may create a 6074 // division by zero. 6075 if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo())) 6076 return false; 6077 continue; 6078 } 6079 if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) && 6080 !isa<ConstantFP>(Val)) 6081 return false; 6082 } 6083 // Check that the resulting operation is legal. 6084 int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode()); 6085 if (!ISDOpcode) 6086 return false; 6087 return StressStoreExtract || 6088 TLI.isOperationLegalOrCustom( 6089 ISDOpcode, TLI.getValueType(DL, getTransitionType(), true)); 6090 } 6091 6092 /// Check whether or not \p Use can be combined 6093 /// with the transition. 6094 /// I.e., is it possible to do Use(Transition) => AnotherUse? 6095 bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); } 6096 6097 /// Record \p ToBePromoted as part of the chain to be promoted. 6098 void enqueueForPromotion(Instruction *ToBePromoted) { 6099 InstsToBePromoted.push_back(ToBePromoted); 6100 } 6101 6102 /// Set the instruction that will be combined with the transition. 6103 void recordCombineInstruction(Instruction *ToBeCombined) { 6104 assert(canCombine(ToBeCombined) && "Unsupported instruction to combine"); 6105 CombineInst = ToBeCombined; 6106 } 6107 6108 /// Promote all the instructions enqueued for promotion if it is 6109 /// is profitable. 6110 /// \return True if the promotion happened, false otherwise. 6111 bool promote() { 6112 // Check if there is something to promote. 6113 // Right now, if we do not have anything to combine with, 6114 // we assume the promotion is not profitable. 6115 if (InstsToBePromoted.empty() || !CombineInst) 6116 return false; 6117 6118 // Check cost. 6119 if (!StressStoreExtract && !isProfitableToPromote()) 6120 return false; 6121 6122 // Promote. 6123 for (auto &ToBePromoted : InstsToBePromoted) 6124 promoteImpl(ToBePromoted); 6125 InstsToBePromoted.clear(); 6126 return true; 6127 } 6128 }; 6129 6130 } // end anonymous namespace 6131 6132 void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) { 6133 // At this point, we know that all the operands of ToBePromoted but Def 6134 // can be statically promoted. 6135 // For Def, we need to use its parameter in ToBePromoted: 6136 // b = ToBePromoted ty1 a 6137 // Def = Transition ty1 b to ty2 6138 // Move the transition down. 6139 // 1. Replace all uses of the promoted operation by the transition. 6140 // = ... b => = ... Def. 6141 assert(ToBePromoted->getType() == Transition->getType() && 6142 "The type of the result of the transition does not match " 6143 "the final type"); 6144 ToBePromoted->replaceAllUsesWith(Transition); 6145 // 2. Update the type of the uses. 6146 // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def. 6147 Type *TransitionTy = getTransitionType(); 6148 ToBePromoted->mutateType(TransitionTy); 6149 // 3. Update all the operands of the promoted operation with promoted 6150 // operands. 6151 // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a. 6152 for (Use &U : ToBePromoted->operands()) { 6153 Value *Val = U.get(); 6154 Value *NewVal = nullptr; 6155 if (Val == Transition) 6156 NewVal = Transition->getOperand(getTransitionOriginalValueIdx()); 6157 else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) || 6158 isa<ConstantFP>(Val)) { 6159 // Use a splat constant if it is not safe to use undef. 6160 NewVal = getConstantVector( 6161 cast<Constant>(Val), 6162 isa<UndefValue>(Val) || 6163 canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo())); 6164 } else 6165 llvm_unreachable("Did you modified shouldPromote and forgot to update " 6166 "this?"); 6167 ToBePromoted->setOperand(U.getOperandNo(), NewVal); 6168 } 6169 Transition->moveAfter(ToBePromoted); 6170 Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted); 6171 } 6172 6173 /// Some targets can do store(extractelement) with one instruction. 6174 /// Try to push the extractelement towards the stores when the target 6175 /// has this feature and this is profitable. 6176 bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) { 6177 unsigned CombineCost = std::numeric_limits<unsigned>::max(); 6178 if (DisableStoreExtract || !TLI || 6179 (!StressStoreExtract && 6180 !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(), 6181 Inst->getOperand(1), CombineCost))) 6182 return false; 6183 6184 // At this point we know that Inst is a vector to scalar transition. 6185 // Try to move it down the def-use chain, until: 6186 // - We can combine the transition with its single use 6187 // => we got rid of the transition. 6188 // - We escape the current basic block 6189 // => we would need to check that we are moving it at a cheaper place and 6190 // we do not do that for now. 6191 BasicBlock *Parent = Inst->getParent(); 6192 LLVM_DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n'); 6193 VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost); 6194 // If the transition has more than one use, assume this is not going to be 6195 // beneficial. 6196 while (Inst->hasOneUse()) { 6197 Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin()); 6198 LLVM_DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n'); 6199 6200 if (ToBePromoted->getParent() != Parent) { 6201 LLVM_DEBUG(dbgs() << "Instruction to promote is in a different block (" 6202 << ToBePromoted->getParent()->getName() 6203 << ") than the transition (" << Parent->getName() 6204 << ").\n"); 6205 return false; 6206 } 6207 6208 if (VPH.canCombine(ToBePromoted)) { 6209 LLVM_DEBUG(dbgs() << "Assume " << *Inst << '\n' 6210 << "will be combined with: " << *ToBePromoted << '\n'); 6211 VPH.recordCombineInstruction(ToBePromoted); 6212 bool Changed = VPH.promote(); 6213 NumStoreExtractExposed += Changed; 6214 return Changed; 6215 } 6216 6217 LLVM_DEBUG(dbgs() << "Try promoting.\n"); 6218 if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted)) 6219 return false; 6220 6221 LLVM_DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n"); 6222 6223 VPH.enqueueForPromotion(ToBePromoted); 6224 Inst = ToBePromoted; 6225 } 6226 return false; 6227 } 6228 6229 /// For the instruction sequence of store below, F and I values 6230 /// are bundled together as an i64 value before being stored into memory. 6231 /// Sometimes it is more efficient to generate separate stores for F and I, 6232 /// which can remove the bitwise instructions or sink them to colder places. 6233 /// 6234 /// (store (or (zext (bitcast F to i32) to i64), 6235 /// (shl (zext I to i64), 32)), addr) --> 6236 /// (store F, addr) and (store I, addr+4) 6237 /// 6238 /// Similarly, splitting for other merged store can also be beneficial, like: 6239 /// For pair of {i32, i32}, i64 store --> two i32 stores. 6240 /// For pair of {i32, i16}, i64 store --> two i32 stores. 6241 /// For pair of {i16, i16}, i32 store --> two i16 stores. 6242 /// For pair of {i16, i8}, i32 store --> two i16 stores. 6243 /// For pair of {i8, i8}, i16 store --> two i8 stores. 6244 /// 6245 /// We allow each target to determine specifically which kind of splitting is 6246 /// supported. 6247 /// 6248 /// The store patterns are commonly seen from the simple code snippet below 6249 /// if only std::make_pair(...) is sroa transformed before inlined into hoo. 6250 /// void goo(const std::pair<int, float> &); 6251 /// hoo() { 6252 /// ... 6253 /// goo(std::make_pair(tmp, ftmp)); 6254 /// ... 6255 /// } 6256 /// 6257 /// Although we already have similar splitting in DAG Combine, we duplicate 6258 /// it in CodeGenPrepare to catch the case in which pattern is across 6259 /// multiple BBs. The logic in DAG Combine is kept to catch case generated 6260 /// during code expansion. 6261 static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, 6262 const TargetLowering &TLI) { 6263 // Handle simple but common cases only. 6264 Type *StoreType = SI.getValueOperand()->getType(); 6265 if (DL.getTypeStoreSizeInBits(StoreType) != DL.getTypeSizeInBits(StoreType) || 6266 DL.getTypeSizeInBits(StoreType) == 0) 6267 return false; 6268 6269 unsigned HalfValBitSize = DL.getTypeSizeInBits(StoreType) / 2; 6270 Type *SplitStoreType = Type::getIntNTy(SI.getContext(), HalfValBitSize); 6271 if (DL.getTypeStoreSizeInBits(SplitStoreType) != 6272 DL.getTypeSizeInBits(SplitStoreType)) 6273 return false; 6274 6275 // Match the following patterns: 6276 // (store (or (zext LValue to i64), 6277 // (shl (zext HValue to i64), 32)), HalfValBitSize) 6278 // or 6279 // (store (or (shl (zext HValue to i64), 32)), HalfValBitSize) 6280 // (zext LValue to i64), 6281 // Expect both operands of OR and the first operand of SHL have only 6282 // one use. 6283 Value *LValue, *HValue; 6284 if (!match(SI.getValueOperand(), 6285 m_c_Or(m_OneUse(m_ZExt(m_Value(LValue))), 6286 m_OneUse(m_Shl(m_OneUse(m_ZExt(m_Value(HValue))), 6287 m_SpecificInt(HalfValBitSize)))))) 6288 return false; 6289 6290 // Check LValue and HValue are int with size less or equal than 32. 6291 if (!LValue->getType()->isIntegerTy() || 6292 DL.getTypeSizeInBits(LValue->getType()) > HalfValBitSize || 6293 !HValue->getType()->isIntegerTy() || 6294 DL.getTypeSizeInBits(HValue->getType()) > HalfValBitSize) 6295 return false; 6296 6297 // If LValue/HValue is a bitcast instruction, use the EVT before bitcast 6298 // as the input of target query. 6299 auto *LBC = dyn_cast<BitCastInst>(LValue); 6300 auto *HBC = dyn_cast<BitCastInst>(HValue); 6301 EVT LowTy = LBC ? EVT::getEVT(LBC->getOperand(0)->getType()) 6302 : EVT::getEVT(LValue->getType()); 6303 EVT HighTy = HBC ? EVT::getEVT(HBC->getOperand(0)->getType()) 6304 : EVT::getEVT(HValue->getType()); 6305 if (!ForceSplitStore && !TLI.isMultiStoresCheaperThanBitsMerge(LowTy, HighTy)) 6306 return false; 6307 6308 // Start to split store. 6309 IRBuilder<> Builder(SI.getContext()); 6310 Builder.SetInsertPoint(&SI); 6311 6312 // If LValue/HValue is a bitcast in another BB, create a new one in current 6313 // BB so it may be merged with the splitted stores by dag combiner. 6314 if (LBC && LBC->getParent() != SI.getParent()) 6315 LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType()); 6316 if (HBC && HBC->getParent() != SI.getParent()) 6317 HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType()); 6318 6319 bool IsLE = SI.getModule()->getDataLayout().isLittleEndian(); 6320 auto CreateSplitStore = [&](Value *V, bool Upper) { 6321 V = Builder.CreateZExtOrBitCast(V, SplitStoreType); 6322 Value *Addr = Builder.CreateBitCast( 6323 SI.getOperand(1), 6324 SplitStoreType->getPointerTo(SI.getPointerAddressSpace())); 6325 if ((IsLE && Upper) || (!IsLE && !Upper)) 6326 Addr = Builder.CreateGEP( 6327 SplitStoreType, Addr, 6328 ConstantInt::get(Type::getInt32Ty(SI.getContext()), 1)); 6329 Builder.CreateAlignedStore( 6330 V, Addr, Upper ? SI.getAlignment() / 2 : SI.getAlignment()); 6331 }; 6332 6333 CreateSplitStore(LValue, false); 6334 CreateSplitStore(HValue, true); 6335 6336 // Delete the old store. 6337 SI.eraseFromParent(); 6338 return true; 6339 } 6340 6341 // Return true if the GEP has two operands, the first operand is of a sequential 6342 // type, and the second operand is a constant. 6343 static bool GEPSequentialConstIndexed(GetElementPtrInst *GEP) { 6344 gep_type_iterator I = gep_type_begin(*GEP); 6345 return GEP->getNumOperands() == 2 && 6346 I.isSequential() && 6347 isa<ConstantInt>(GEP->getOperand(1)); 6348 } 6349 6350 // Try unmerging GEPs to reduce liveness interference (register pressure) across 6351 // IndirectBr edges. Since IndirectBr edges tend to touch on many blocks, 6352 // reducing liveness interference across those edges benefits global register 6353 // allocation. Currently handles only certain cases. 6354 // 6355 // For example, unmerge %GEPI and %UGEPI as below. 6356 // 6357 // ---------- BEFORE ---------- 6358 // SrcBlock: 6359 // ... 6360 // %GEPIOp = ... 6361 // ... 6362 // %GEPI = gep %GEPIOp, Idx 6363 // ... 6364 // indirectbr ... [ label %DstB0, label %DstB1, ... label %DstBi ... ] 6365 // (* %GEPI is alive on the indirectbr edges due to other uses ahead) 6366 // (* %GEPIOp is alive on the indirectbr edges only because of it's used by 6367 // %UGEPI) 6368 // 6369 // DstB0: ... (there may be a gep similar to %UGEPI to be unmerged) 6370 // DstB1: ... (there may be a gep similar to %UGEPI to be unmerged) 6371 // ... 6372 // 6373 // DstBi: 6374 // ... 6375 // %UGEPI = gep %GEPIOp, UIdx 6376 // ... 6377 // --------------------------- 6378 // 6379 // ---------- AFTER ---------- 6380 // SrcBlock: 6381 // ... (same as above) 6382 // (* %GEPI is still alive on the indirectbr edges) 6383 // (* %GEPIOp is no longer alive on the indirectbr edges as a result of the 6384 // unmerging) 6385 // ... 6386 // 6387 // DstBi: 6388 // ... 6389 // %UGEPI = gep %GEPI, (UIdx-Idx) 6390 // ... 6391 // --------------------------- 6392 // 6393 // The register pressure on the IndirectBr edges is reduced because %GEPIOp is 6394 // no longer alive on them. 6395 // 6396 // We try to unmerge GEPs here in CodGenPrepare, as opposed to limiting merging 6397 // of GEPs in the first place in InstCombiner::visitGetElementPtrInst() so as 6398 // not to disable further simplications and optimizations as a result of GEP 6399 // merging. 6400 // 6401 // Note this unmerging may increase the length of the data flow critical path 6402 // (the path from %GEPIOp to %UGEPI would go through %GEPI), which is a tradeoff 6403 // between the register pressure and the length of data-flow critical 6404 // path. Restricting this to the uncommon IndirectBr case would minimize the 6405 // impact of potentially longer critical path, if any, and the impact on compile 6406 // time. 6407 static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI, 6408 const TargetTransformInfo *TTI) { 6409 BasicBlock *SrcBlock = GEPI->getParent(); 6410 // Check that SrcBlock ends with an IndirectBr. If not, give up. The common 6411 // (non-IndirectBr) cases exit early here. 6412 if (!isa<IndirectBrInst>(SrcBlock->getTerminator())) 6413 return false; 6414 // Check that GEPI is a simple gep with a single constant index. 6415 if (!GEPSequentialConstIndexed(GEPI)) 6416 return false; 6417 ConstantInt *GEPIIdx = cast<ConstantInt>(GEPI->getOperand(1)); 6418 // Check that GEPI is a cheap one. 6419 if (TTI->getIntImmCost(GEPIIdx->getValue(), GEPIIdx->getType()) 6420 > TargetTransformInfo::TCC_Basic) 6421 return false; 6422 Value *GEPIOp = GEPI->getOperand(0); 6423 // Check that GEPIOp is an instruction that's also defined in SrcBlock. 6424 if (!isa<Instruction>(GEPIOp)) 6425 return false; 6426 auto *GEPIOpI = cast<Instruction>(GEPIOp); 6427 if (GEPIOpI->getParent() != SrcBlock) 6428 return false; 6429 // Check that GEP is used outside the block, meaning it's alive on the 6430 // IndirectBr edge(s). 6431 if (find_if(GEPI->users(), [&](User *Usr) { 6432 if (auto *I = dyn_cast<Instruction>(Usr)) { 6433 if (I->getParent() != SrcBlock) { 6434 return true; 6435 } 6436 } 6437 return false; 6438 }) == GEPI->users().end()) 6439 return false; 6440 // The second elements of the GEP chains to be unmerged. 6441 std::vector<GetElementPtrInst *> UGEPIs; 6442 // Check each user of GEPIOp to check if unmerging would make GEPIOp not alive 6443 // on IndirectBr edges. 6444 for (User *Usr : GEPIOp->users()) { 6445 if (Usr == GEPI) continue; 6446 // Check if Usr is an Instruction. If not, give up. 6447 if (!isa<Instruction>(Usr)) 6448 return false; 6449 auto *UI = cast<Instruction>(Usr); 6450 // Check if Usr in the same block as GEPIOp, which is fine, skip. 6451 if (UI->getParent() == SrcBlock) 6452 continue; 6453 // Check if Usr is a GEP. If not, give up. 6454 if (!isa<GetElementPtrInst>(Usr)) 6455 return false; 6456 auto *UGEPI = cast<GetElementPtrInst>(Usr); 6457 // Check if UGEPI is a simple gep with a single constant index and GEPIOp is 6458 // the pointer operand to it. If so, record it in the vector. If not, give 6459 // up. 6460 if (!GEPSequentialConstIndexed(UGEPI)) 6461 return false; 6462 if (UGEPI->getOperand(0) != GEPIOp) 6463 return false; 6464 if (GEPIIdx->getType() != 6465 cast<ConstantInt>(UGEPI->getOperand(1))->getType()) 6466 return false; 6467 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1)); 6468 if (TTI->getIntImmCost(UGEPIIdx->getValue(), UGEPIIdx->getType()) 6469 > TargetTransformInfo::TCC_Basic) 6470 return false; 6471 UGEPIs.push_back(UGEPI); 6472 } 6473 if (UGEPIs.size() == 0) 6474 return false; 6475 // Check the materializing cost of (Uidx-Idx). 6476 for (GetElementPtrInst *UGEPI : UGEPIs) { 6477 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1)); 6478 APInt NewIdx = UGEPIIdx->getValue() - GEPIIdx->getValue(); 6479 unsigned ImmCost = TTI->getIntImmCost(NewIdx, GEPIIdx->getType()); 6480 if (ImmCost > TargetTransformInfo::TCC_Basic) 6481 return false; 6482 } 6483 // Now unmerge between GEPI and UGEPIs. 6484 for (GetElementPtrInst *UGEPI : UGEPIs) { 6485 UGEPI->setOperand(0, GEPI); 6486 ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1)); 6487 Constant *NewUGEPIIdx = 6488 ConstantInt::get(GEPIIdx->getType(), 6489 UGEPIIdx->getValue() - GEPIIdx->getValue()); 6490 UGEPI->setOperand(1, NewUGEPIIdx); 6491 // If GEPI is not inbounds but UGEPI is inbounds, change UGEPI to not 6492 // inbounds to avoid UB. 6493 if (!GEPI->isInBounds()) { 6494 UGEPI->setIsInBounds(false); 6495 } 6496 } 6497 // After unmerging, verify that GEPIOp is actually only used in SrcBlock (not 6498 // alive on IndirectBr edges). 6499 assert(find_if(GEPIOp->users(), [&](User *Usr) { 6500 return cast<Instruction>(Usr)->getParent() != SrcBlock; 6501 }) == GEPIOp->users().end() && "GEPIOp is used outside SrcBlock"); 6502 return true; 6503 } 6504 6505 bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) { 6506 // Bail out if we inserted the instruction to prevent optimizations from 6507 // stepping on each other's toes. 6508 if (InsertedInsts.count(I)) 6509 return false; 6510 6511 if (PHINode *P = dyn_cast<PHINode>(I)) { 6512 // It is possible for very late stage optimizations (such as SimplifyCFG) 6513 // to introduce PHI nodes too late to be cleaned up. If we detect such a 6514 // trivial PHI, go ahead and zap it here. 6515 if (Value *V = SimplifyInstruction(P, {*DL, TLInfo})) { 6516 P->replaceAllUsesWith(V); 6517 P->eraseFromParent(); 6518 ++NumPHIsElim; 6519 return true; 6520 } 6521 return false; 6522 } 6523 6524 if (CastInst *CI = dyn_cast<CastInst>(I)) { 6525 // If the source of the cast is a constant, then this should have 6526 // already been constant folded. The only reason NOT to constant fold 6527 // it is if something (e.g. LSR) was careful to place the constant 6528 // evaluation in a block other than then one that uses it (e.g. to hoist 6529 // the address of globals out of a loop). If this is the case, we don't 6530 // want to forward-subst the cast. 6531 if (isa<Constant>(CI->getOperand(0))) 6532 return false; 6533 6534 if (TLI && OptimizeNoopCopyExpression(CI, *TLI, *DL)) 6535 return true; 6536 6537 if (isa<ZExtInst>(I) || isa<SExtInst>(I)) { 6538 /// Sink a zext or sext into its user blocks if the target type doesn't 6539 /// fit in one register 6540 if (TLI && 6541 TLI->getTypeAction(CI->getContext(), 6542 TLI->getValueType(*DL, CI->getType())) == 6543 TargetLowering::TypeExpandInteger) { 6544 return SinkCast(CI); 6545 } else { 6546 bool MadeChange = optimizeExt(I); 6547 return MadeChange | optimizeExtUses(I); 6548 } 6549 } 6550 return false; 6551 } 6552 6553 if (CmpInst *CI = dyn_cast<CmpInst>(I)) 6554 if (!TLI || !TLI->hasMultipleConditionRegisters()) 6555 return OptimizeCmpExpression(CI, TLI); 6556 6557 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 6558 LI->setMetadata(LLVMContext::MD_invariant_group, nullptr); 6559 if (TLI) { 6560 bool Modified = optimizeLoadExt(LI); 6561 unsigned AS = LI->getPointerAddressSpace(); 6562 Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS); 6563 return Modified; 6564 } 6565 return false; 6566 } 6567 6568 if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 6569 if (TLI && splitMergedValStore(*SI, *DL, *TLI)) 6570 return true; 6571 SI->setMetadata(LLVMContext::MD_invariant_group, nullptr); 6572 if (TLI) { 6573 unsigned AS = SI->getPointerAddressSpace(); 6574 return optimizeMemoryInst(I, SI->getOperand(1), 6575 SI->getOperand(0)->getType(), AS); 6576 } 6577 return false; 6578 } 6579 6580 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) { 6581 unsigned AS = RMW->getPointerAddressSpace(); 6582 return optimizeMemoryInst(I, RMW->getPointerOperand(), 6583 RMW->getType(), AS); 6584 } 6585 6586 if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(I)) { 6587 unsigned AS = CmpX->getPointerAddressSpace(); 6588 return optimizeMemoryInst(I, CmpX->getPointerOperand(), 6589 CmpX->getCompareOperand()->getType(), AS); 6590 } 6591 6592 BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I); 6593 6594 if (BinOp && (BinOp->getOpcode() == Instruction::And) && 6595 EnableAndCmpSinking && TLI) 6596 return sinkAndCmp0Expression(BinOp, *TLI, InsertedInsts); 6597 6598 if (BinOp && (BinOp->getOpcode() == Instruction::AShr || 6599 BinOp->getOpcode() == Instruction::LShr)) { 6600 ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1)); 6601 if (TLI && CI && TLI->hasExtractBitsInsn()) 6602 return OptimizeExtractBits(BinOp, CI, *TLI, *DL); 6603 6604 return false; 6605 } 6606 6607 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) { 6608 if (GEPI->hasAllZeroIndices()) { 6609 /// The GEP operand must be a pointer, so must its result -> BitCast 6610 Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(), 6611 GEPI->getName(), GEPI); 6612 NC->setDebugLoc(GEPI->getDebugLoc()); 6613 GEPI->replaceAllUsesWith(NC); 6614 GEPI->eraseFromParent(); 6615 ++NumGEPsElim; 6616 optimizeInst(NC, ModifiedDT); 6617 return true; 6618 } 6619 if (tryUnmergingGEPsAcrossIndirectBr(GEPI, TTI)) { 6620 return true; 6621 } 6622 return false; 6623 } 6624 6625 if (CallInst *CI = dyn_cast<CallInst>(I)) 6626 return optimizeCallInst(CI, ModifiedDT); 6627 6628 if (SelectInst *SI = dyn_cast<SelectInst>(I)) 6629 return optimizeSelectInst(SI); 6630 6631 if (ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(I)) 6632 return optimizeShuffleVectorInst(SVI); 6633 6634 if (auto *Switch = dyn_cast<SwitchInst>(I)) 6635 return optimizeSwitchInst(Switch); 6636 6637 if (isa<ExtractElementInst>(I)) 6638 return optimizeExtractElementInst(I); 6639 6640 return false; 6641 } 6642 6643 /// Given an OR instruction, check to see if this is a bitreverse 6644 /// idiom. If so, insert the new intrinsic and return true. 6645 static bool makeBitReverse(Instruction &I, const DataLayout &DL, 6646 const TargetLowering &TLI) { 6647 if (!I.getType()->isIntegerTy() || 6648 !TLI.isOperationLegalOrCustom(ISD::BITREVERSE, 6649 TLI.getValueType(DL, I.getType(), true))) 6650 return false; 6651 6652 SmallVector<Instruction*, 4> Insts; 6653 if (!recognizeBSwapOrBitReverseIdiom(&I, false, true, Insts)) 6654 return false; 6655 Instruction *LastInst = Insts.back(); 6656 I.replaceAllUsesWith(LastInst); 6657 RecursivelyDeleteTriviallyDeadInstructions(&I); 6658 return true; 6659 } 6660 6661 // In this pass we look for GEP and cast instructions that are used 6662 // across basic blocks and rewrite them to improve basic-block-at-a-time 6663 // selection. 6664 bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, bool &ModifiedDT) { 6665 SunkAddrs.clear(); 6666 bool MadeChange = false; 6667 6668 CurInstIterator = BB.begin(); 6669 while (CurInstIterator != BB.end()) { 6670 MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT); 6671 if (ModifiedDT) 6672 return true; 6673 } 6674 6675 bool MadeBitReverse = true; 6676 while (TLI && MadeBitReverse) { 6677 MadeBitReverse = false; 6678 for (auto &I : reverse(BB)) { 6679 if (makeBitReverse(I, *DL, *TLI)) { 6680 MadeBitReverse = MadeChange = true; 6681 ModifiedDT = true; 6682 break; 6683 } 6684 } 6685 } 6686 MadeChange |= dupRetToEnableTailCallOpts(&BB); 6687 6688 return MadeChange; 6689 } 6690 6691 // llvm.dbg.value is far away from the value then iSel may not be able 6692 // handle it properly. iSel will drop llvm.dbg.value if it can not 6693 // find a node corresponding to the value. 6694 bool CodeGenPrepare::placeDbgValues(Function &F) { 6695 bool MadeChange = false; 6696 for (BasicBlock &BB : F) { 6697 Instruction *PrevNonDbgInst = nullptr; 6698 for (BasicBlock::iterator BI = BB.begin(), BE = BB.end(); BI != BE;) { 6699 Instruction *Insn = &*BI++; 6700 DbgValueInst *DVI = dyn_cast<DbgValueInst>(Insn); 6701 // Leave dbg.values that refer to an alloca alone. These 6702 // intrinsics describe the address of a variable (= the alloca) 6703 // being taken. They should not be moved next to the alloca 6704 // (and to the beginning of the scope), but rather stay close to 6705 // where said address is used. 6706 if (!DVI || (DVI->getValue() && isa<AllocaInst>(DVI->getValue()))) { 6707 PrevNonDbgInst = Insn; 6708 continue; 6709 } 6710 6711 Instruction *VI = dyn_cast_or_null<Instruction>(DVI->getValue()); 6712 if (VI && VI != PrevNonDbgInst && !VI->isTerminator()) { 6713 // If VI is a phi in a block with an EHPad terminator, we can't insert 6714 // after it. 6715 if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad()) 6716 continue; 6717 LLVM_DEBUG(dbgs() << "Moving Debug Value before :\n" 6718 << *DVI << ' ' << *VI); 6719 DVI->removeFromParent(); 6720 if (isa<PHINode>(VI)) 6721 DVI->insertBefore(&*VI->getParent()->getFirstInsertionPt()); 6722 else 6723 DVI->insertAfter(VI); 6724 MadeChange = true; 6725 ++NumDbgValueMoved; 6726 } 6727 } 6728 } 6729 return MadeChange; 6730 } 6731 6732 /// Scale down both weights to fit into uint32_t. 6733 static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) { 6734 uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse; 6735 uint32_t Scale = (NewMax / std::numeric_limits<uint32_t>::max()) + 1; 6736 NewTrue = NewTrue / Scale; 6737 NewFalse = NewFalse / Scale; 6738 } 6739 6740 /// Some targets prefer to split a conditional branch like: 6741 /// \code 6742 /// %0 = icmp ne i32 %a, 0 6743 /// %1 = icmp ne i32 %b, 0 6744 /// %or.cond = or i1 %0, %1 6745 /// br i1 %or.cond, label %TrueBB, label %FalseBB 6746 /// \endcode 6747 /// into multiple branch instructions like: 6748 /// \code 6749 /// bb1: 6750 /// %0 = icmp ne i32 %a, 0 6751 /// br i1 %0, label %TrueBB, label %bb2 6752 /// bb2: 6753 /// %1 = icmp ne i32 %b, 0 6754 /// br i1 %1, label %TrueBB, label %FalseBB 6755 /// \endcode 6756 /// This usually allows instruction selection to do even further optimizations 6757 /// and combine the compare with the branch instruction. Currently this is 6758 /// applied for targets which have "cheap" jump instructions. 6759 /// 6760 /// FIXME: Remove the (equivalent?) implementation in SelectionDAG. 6761 /// 6762 bool CodeGenPrepare::splitBranchCondition(Function &F) { 6763 if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive()) 6764 return false; 6765 6766 bool MadeChange = false; 6767 for (auto &BB : F) { 6768 // Does this BB end with the following? 6769 // %cond1 = icmp|fcmp|binary instruction ... 6770 // %cond2 = icmp|fcmp|binary instruction ... 6771 // %cond.or = or|and i1 %cond1, cond2 6772 // br i1 %cond.or label %dest1, label %dest2" 6773 BinaryOperator *LogicOp; 6774 BasicBlock *TBB, *FBB; 6775 if (!match(BB.getTerminator(), m_Br(m_OneUse(m_BinOp(LogicOp)), TBB, FBB))) 6776 continue; 6777 6778 auto *Br1 = cast<BranchInst>(BB.getTerminator()); 6779 if (Br1->getMetadata(LLVMContext::MD_unpredictable)) 6780 continue; 6781 6782 unsigned Opc; 6783 Value *Cond1, *Cond2; 6784 if (match(LogicOp, m_And(m_OneUse(m_Value(Cond1)), 6785 m_OneUse(m_Value(Cond2))))) 6786 Opc = Instruction::And; 6787 else if (match(LogicOp, m_Or(m_OneUse(m_Value(Cond1)), 6788 m_OneUse(m_Value(Cond2))))) 6789 Opc = Instruction::Or; 6790 else 6791 continue; 6792 6793 if (!match(Cond1, m_CombineOr(m_Cmp(), m_BinOp())) || 6794 !match(Cond2, m_CombineOr(m_Cmp(), m_BinOp())) ) 6795 continue; 6796 6797 LLVM_DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump()); 6798 6799 // Create a new BB. 6800 auto TmpBB = 6801 BasicBlock::Create(BB.getContext(), BB.getName() + ".cond.split", 6802 BB.getParent(), BB.getNextNode()); 6803 6804 // Update original basic block by using the first condition directly by the 6805 // branch instruction and removing the no longer needed and/or instruction. 6806 Br1->setCondition(Cond1); 6807 LogicOp->eraseFromParent(); 6808 6809 // Depending on the condition we have to either replace the true or the 6810 // false successor of the original branch instruction. 6811 if (Opc == Instruction::And) 6812 Br1->setSuccessor(0, TmpBB); 6813 else 6814 Br1->setSuccessor(1, TmpBB); 6815 6816 // Fill in the new basic block. 6817 auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB); 6818 if (auto *I = dyn_cast<Instruction>(Cond2)) { 6819 I->removeFromParent(); 6820 I->insertBefore(Br2); 6821 } 6822 6823 // Update PHI nodes in both successors. The original BB needs to be 6824 // replaced in one successor's PHI nodes, because the branch comes now from 6825 // the newly generated BB (NewBB). In the other successor we need to add one 6826 // incoming edge to the PHI nodes, because both branch instructions target 6827 // now the same successor. Depending on the original branch condition 6828 // (and/or) we have to swap the successors (TrueDest, FalseDest), so that 6829 // we perform the correct update for the PHI nodes. 6830 // This doesn't change the successor order of the just created branch 6831 // instruction (or any other instruction). 6832 if (Opc == Instruction::Or) 6833 std::swap(TBB, FBB); 6834 6835 // Replace the old BB with the new BB. 6836 for (PHINode &PN : TBB->phis()) { 6837 int i; 6838 while ((i = PN.getBasicBlockIndex(&BB)) >= 0) 6839 PN.setIncomingBlock(i, TmpBB); 6840 } 6841 6842 // Add another incoming edge form the new BB. 6843 for (PHINode &PN : FBB->phis()) { 6844 auto *Val = PN.getIncomingValueForBlock(&BB); 6845 PN.addIncoming(Val, TmpBB); 6846 } 6847 6848 // Update the branch weights (from SelectionDAGBuilder:: 6849 // FindMergedConditions). 6850 if (Opc == Instruction::Or) { 6851 // Codegen X | Y as: 6852 // BB1: 6853 // jmp_if_X TBB 6854 // jmp TmpBB 6855 // TmpBB: 6856 // jmp_if_Y TBB 6857 // jmp FBB 6858 // 6859 6860 // We have flexibility in setting Prob for BB1 and Prob for NewBB. 6861 // The requirement is that 6862 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) 6863 // = TrueProb for original BB. 6864 // Assuming the original weights are A and B, one choice is to set BB1's 6865 // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice 6866 // assumes that 6867 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. 6868 // Another choice is to assume TrueProb for BB1 equals to TrueProb for 6869 // TmpBB, but the math is more complicated. 6870 uint64_t TrueWeight, FalseWeight; 6871 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) { 6872 uint64_t NewTrueWeight = TrueWeight; 6873 uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight; 6874 scaleWeights(NewTrueWeight, NewFalseWeight); 6875 Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) 6876 .createBranchWeights(TrueWeight, FalseWeight)); 6877 6878 NewTrueWeight = TrueWeight; 6879 NewFalseWeight = 2 * FalseWeight; 6880 scaleWeights(NewTrueWeight, NewFalseWeight); 6881 Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) 6882 .createBranchWeights(TrueWeight, FalseWeight)); 6883 } 6884 } else { 6885 // Codegen X & Y as: 6886 // BB1: 6887 // jmp_if_X TmpBB 6888 // jmp FBB 6889 // TmpBB: 6890 // jmp_if_Y TBB 6891 // jmp FBB 6892 // 6893 // This requires creation of TmpBB after CurBB. 6894 6895 // We have flexibility in setting Prob for BB1 and Prob for TmpBB. 6896 // The requirement is that 6897 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) 6898 // = FalseProb for original BB. 6899 // Assuming the original weights are A and B, one choice is to set BB1's 6900 // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice 6901 // assumes that 6902 // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB. 6903 uint64_t TrueWeight, FalseWeight; 6904 if (Br1->extractProfMetadata(TrueWeight, FalseWeight)) { 6905 uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight; 6906 uint64_t NewFalseWeight = FalseWeight; 6907 scaleWeights(NewTrueWeight, NewFalseWeight); 6908 Br1->setMetadata(LLVMContext::MD_prof, MDBuilder(Br1->getContext()) 6909 .createBranchWeights(TrueWeight, FalseWeight)); 6910 6911 NewTrueWeight = 2 * TrueWeight; 6912 NewFalseWeight = FalseWeight; 6913 scaleWeights(NewTrueWeight, NewFalseWeight); 6914 Br2->setMetadata(LLVMContext::MD_prof, MDBuilder(Br2->getContext()) 6915 .createBranchWeights(TrueWeight, FalseWeight)); 6916 } 6917 } 6918 6919 // Note: No point in getting fancy here, since the DT info is never 6920 // available to CodeGenPrepare. 6921 ModifiedDT = true; 6922 6923 MadeChange = true; 6924 6925 LLVM_DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump(); 6926 TmpBB->dump()); 6927 } 6928 return MadeChange; 6929 } 6930