1 //===--- Scalarizer.cpp - Scalarize vector operations ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This pass converts vector operations into scalar operations, in order 11 // to expose optimization opportunities on the individual scalar operations. 12 // It is mainly intended for targets that do not have vector units, but it 13 // may also be useful for revectorizing code to different vector widths. 14 // 15 //===----------------------------------------------------------------------===// 16 17 #include "llvm/ADT/STLExtras.h" 18 #include "llvm/IR/IRBuilder.h" 19 #include "llvm/IR/InstVisitor.h" 20 #include "llvm/Pass.h" 21 #include "llvm/Support/CommandLine.h" 22 #include "llvm/Transforms/Scalar.h" 23 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 24 25 using namespace llvm; 26 27 #define DEBUG_TYPE "scalarizer" 28 29 namespace { 30 // Used to store the scattered form of a vector. 31 typedef SmallVector<Value *, 8> ValueVector; 32 33 // Used to map a vector Value to its scattered form. We use std::map 34 // because we want iterators to persist across insertion and because the 35 // values are relatively large. 36 typedef std::map<Value *, ValueVector> ScatterMap; 37 38 // Lists Instructions that have been replaced with scalar implementations, 39 // along with a pointer to their scattered forms. 40 typedef SmallVector<std::pair<Instruction *, ValueVector *>, 16> GatherList; 41 42 // Provides a very limited vector-like interface for lazily accessing one 43 // component of a scattered vector or vector pointer. 44 class Scatterer { 45 public: 46 Scatterer() {} 47 48 // Scatter V into Size components. If new instructions are needed, 49 // insert them before BBI in BB. If Cache is nonnull, use it to cache 50 // the results. 51 Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v, 52 ValueVector *cachePtr = nullptr); 53 54 // Return component I, creating a new Value for it if necessary. 55 Value *operator[](unsigned I); 56 57 // Return the number of components. 58 unsigned size() const { return Size; } 59 60 private: 61 BasicBlock *BB; 62 BasicBlock::iterator BBI; 63 Value *V; 64 ValueVector *CachePtr; 65 PointerType *PtrTy; 66 ValueVector Tmp; 67 unsigned Size; 68 }; 69 70 // FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp 71 // called Name that compares X and Y in the same way as FCI. 72 struct FCmpSplitter { 73 FCmpSplitter(FCmpInst &fci) : FCI(fci) {} 74 Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1, 75 const Twine &Name) const { 76 return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name); 77 } 78 FCmpInst &FCI; 79 }; 80 81 // ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp 82 // called Name that compares X and Y in the same way as ICI. 83 struct ICmpSplitter { 84 ICmpSplitter(ICmpInst &ici) : ICI(ici) {} 85 Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1, 86 const Twine &Name) const { 87 return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name); 88 } 89 ICmpInst &ICI; 90 }; 91 92 // BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create 93 // a binary operator like BO called Name with operands X and Y. 94 struct BinarySplitter { 95 BinarySplitter(BinaryOperator &bo) : BO(bo) {} 96 Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1, 97 const Twine &Name) const { 98 return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name); 99 } 100 BinaryOperator &BO; 101 }; 102 103 // Information about a load or store that we're scalarizing. 104 struct VectorLayout { 105 VectorLayout() : VecTy(nullptr), ElemTy(nullptr), VecAlign(0), ElemSize(0) {} 106 107 // Return the alignment of element I. 108 uint64_t getElemAlign(unsigned I) { 109 return MinAlign(VecAlign, I * ElemSize); 110 } 111 112 // The type of the vector. 113 VectorType *VecTy; 114 115 // The type of each element. 116 Type *ElemTy; 117 118 // The alignment of the vector. 119 uint64_t VecAlign; 120 121 // The size of each element. 122 uint64_t ElemSize; 123 }; 124 125 class Scalarizer : public FunctionPass, 126 public InstVisitor<Scalarizer, bool> { 127 public: 128 static char ID; 129 130 Scalarizer() : 131 FunctionPass(ID) { 132 initializeScalarizerPass(*PassRegistry::getPassRegistry()); 133 } 134 135 bool doInitialization(Module &M) override; 136 bool runOnFunction(Function &F) override; 137 138 // InstVisitor methods. They return true if the instruction was scalarized, 139 // false if nothing changed. 140 bool visitInstruction(Instruction &) { return false; } 141 bool visitSelectInst(SelectInst &SI); 142 bool visitICmpInst(ICmpInst &); 143 bool visitFCmpInst(FCmpInst &); 144 bool visitBinaryOperator(BinaryOperator &); 145 bool visitGetElementPtrInst(GetElementPtrInst &); 146 bool visitCastInst(CastInst &); 147 bool visitBitCastInst(BitCastInst &); 148 bool visitShuffleVectorInst(ShuffleVectorInst &); 149 bool visitPHINode(PHINode &); 150 bool visitLoadInst(LoadInst &); 151 bool visitStoreInst(StoreInst &); 152 153 static void registerOptions() { 154 // This is disabled by default because having separate loads and stores 155 // makes it more likely that the -combiner-alias-analysis limits will be 156 // reached. 157 OptionRegistry::registerOption<bool, Scalarizer, 158 &Scalarizer::ScalarizeLoadStore>( 159 "scalarize-load-store", 160 "Allow the scalarizer pass to scalarize loads and store", false); 161 } 162 163 private: 164 Scatterer scatter(Instruction *, Value *); 165 void gather(Instruction *, const ValueVector &); 166 bool canTransferMetadata(unsigned Kind); 167 void transferMetadata(Instruction *, const ValueVector &); 168 bool getVectorLayout(Type *, unsigned, VectorLayout &, const DataLayout &); 169 bool finish(); 170 171 template<typename T> bool splitBinary(Instruction &, const T &); 172 173 ScatterMap Scattered; 174 GatherList Gathered; 175 unsigned ParallelLoopAccessMDKind; 176 bool ScalarizeLoadStore; 177 }; 178 179 char Scalarizer::ID = 0; 180 } // end anonymous namespace 181 182 INITIALIZE_PASS_WITH_OPTIONS(Scalarizer, "scalarizer", 183 "Scalarize vector operations", false, false) 184 185 Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v, 186 ValueVector *cachePtr) 187 : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) { 188 Type *Ty = V->getType(); 189 PtrTy = dyn_cast<PointerType>(Ty); 190 if (PtrTy) 191 Ty = PtrTy->getElementType(); 192 Size = Ty->getVectorNumElements(); 193 if (!CachePtr) 194 Tmp.resize(Size, nullptr); 195 else if (CachePtr->empty()) 196 CachePtr->resize(Size, nullptr); 197 else 198 assert(Size == CachePtr->size() && "Inconsistent vector sizes"); 199 } 200 201 // Return component I, creating a new Value for it if necessary. 202 Value *Scatterer::operator[](unsigned I) { 203 ValueVector &CV = (CachePtr ? *CachePtr : Tmp); 204 // Try to reuse a previous value. 205 if (CV[I]) 206 return CV[I]; 207 IRBuilder<> Builder(BB, BBI); 208 if (PtrTy) { 209 if (!CV[0]) { 210 Type *Ty = 211 PointerType::get(PtrTy->getElementType()->getVectorElementType(), 212 PtrTy->getAddressSpace()); 213 CV[0] = Builder.CreateBitCast(V, Ty, V->getName() + ".i0"); 214 } 215 if (I != 0) 216 CV[I] = Builder.CreateConstGEP1_32(nullptr, CV[0], I, 217 V->getName() + ".i" + Twine(I)); 218 } else { 219 // Search through a chain of InsertElementInsts looking for element I. 220 // Record other elements in the cache. The new V is still suitable 221 // for all uncached indices. 222 for (;;) { 223 InsertElementInst *Insert = dyn_cast<InsertElementInst>(V); 224 if (!Insert) 225 break; 226 ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2)); 227 if (!Idx) 228 break; 229 unsigned J = Idx->getZExtValue(); 230 CV[J] = Insert->getOperand(1); 231 V = Insert->getOperand(0); 232 if (I == J) 233 return CV[J]; 234 } 235 CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I), 236 V->getName() + ".i" + Twine(I)); 237 } 238 return CV[I]; 239 } 240 241 bool Scalarizer::doInitialization(Module &M) { 242 ParallelLoopAccessMDKind = 243 M.getContext().getMDKindID("llvm.mem.parallel_loop_access"); 244 ScalarizeLoadStore = 245 M.getContext().getOption<bool, Scalarizer, &Scalarizer::ScalarizeLoadStore>(); 246 return false; 247 } 248 249 bool Scalarizer::runOnFunction(Function &F) { 250 for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) { 251 BasicBlock *BB = BBI; 252 for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) { 253 Instruction *I = II; 254 bool Done = visit(I); 255 ++II; 256 if (Done && I->getType()->isVoidTy()) 257 I->eraseFromParent(); 258 } 259 } 260 return finish(); 261 } 262 263 // Return a scattered form of V that can be accessed by Point. V must be a 264 // vector or a pointer to a vector. 265 Scatterer Scalarizer::scatter(Instruction *Point, Value *V) { 266 if (Argument *VArg = dyn_cast<Argument>(V)) { 267 // Put the scattered form of arguments in the entry block, 268 // so that it can be used everywhere. 269 Function *F = VArg->getParent(); 270 BasicBlock *BB = &F->getEntryBlock(); 271 return Scatterer(BB, BB->begin(), V, &Scattered[V]); 272 } 273 if (Instruction *VOp = dyn_cast<Instruction>(V)) { 274 // Put the scattered form of an instruction directly after the 275 // instruction. 276 BasicBlock *BB = VOp->getParent(); 277 return Scatterer(BB, std::next(BasicBlock::iterator(VOp)), 278 V, &Scattered[V]); 279 } 280 // In the fallback case, just put the scattered before Point and 281 // keep the result local to Point. 282 return Scatterer(Point->getParent(), Point, V); 283 } 284 285 // Replace Op with the gathered form of the components in CV. Defer the 286 // deletion of Op and creation of the gathered form to the end of the pass, 287 // so that we can avoid creating the gathered form if all uses of Op are 288 // replaced with uses of CV. 289 void Scalarizer::gather(Instruction *Op, const ValueVector &CV) { 290 // Since we're not deleting Op yet, stub out its operands, so that it 291 // doesn't make anything live unnecessarily. 292 for (unsigned I = 0, E = Op->getNumOperands(); I != E; ++I) 293 Op->setOperand(I, UndefValue::get(Op->getOperand(I)->getType())); 294 295 transferMetadata(Op, CV); 296 297 // If we already have a scattered form of Op (created from ExtractElements 298 // of Op itself), replace them with the new form. 299 ValueVector &SV = Scattered[Op]; 300 if (!SV.empty()) { 301 for (unsigned I = 0, E = SV.size(); I != E; ++I) { 302 Instruction *Old = cast<Instruction>(SV[I]); 303 CV[I]->takeName(Old); 304 Old->replaceAllUsesWith(CV[I]); 305 Old->eraseFromParent(); 306 } 307 } 308 SV = CV; 309 Gathered.push_back(GatherList::value_type(Op, &SV)); 310 } 311 312 // Return true if it is safe to transfer the given metadata tag from 313 // vector to scalar instructions. 314 bool Scalarizer::canTransferMetadata(unsigned Tag) { 315 return (Tag == LLVMContext::MD_tbaa 316 || Tag == LLVMContext::MD_fpmath 317 || Tag == LLVMContext::MD_tbaa_struct 318 || Tag == LLVMContext::MD_invariant_load 319 || Tag == LLVMContext::MD_alias_scope 320 || Tag == LLVMContext::MD_noalias 321 || Tag == ParallelLoopAccessMDKind); 322 } 323 324 // Transfer metadata from Op to the instructions in CV if it is known 325 // to be safe to do so. 326 void Scalarizer::transferMetadata(Instruction *Op, const ValueVector &CV) { 327 SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; 328 Op->getAllMetadataOtherThanDebugLoc(MDs); 329 for (unsigned I = 0, E = CV.size(); I != E; ++I) { 330 if (Instruction *New = dyn_cast<Instruction>(CV[I])) { 331 for (SmallVectorImpl<std::pair<unsigned, MDNode *>>::iterator 332 MI = MDs.begin(), 333 ME = MDs.end(); 334 MI != ME; ++MI) 335 if (canTransferMetadata(MI->first)) 336 New->setMetadata(MI->first, MI->second); 337 New->setDebugLoc(Op->getDebugLoc()); 338 } 339 } 340 } 341 342 // Try to fill in Layout from Ty, returning true on success. Alignment is 343 // the alignment of the vector, or 0 if the ABI default should be used. 344 bool Scalarizer::getVectorLayout(Type *Ty, unsigned Alignment, 345 VectorLayout &Layout, const DataLayout &DL) { 346 // Make sure we're dealing with a vector. 347 Layout.VecTy = dyn_cast<VectorType>(Ty); 348 if (!Layout.VecTy) 349 return false; 350 351 // Check that we're dealing with full-byte elements. 352 Layout.ElemTy = Layout.VecTy->getElementType(); 353 if (DL.getTypeSizeInBits(Layout.ElemTy) != 354 DL.getTypeStoreSizeInBits(Layout.ElemTy)) 355 return false; 356 357 if (Alignment) 358 Layout.VecAlign = Alignment; 359 else 360 Layout.VecAlign = DL.getABITypeAlignment(Layout.VecTy); 361 Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy); 362 return true; 363 } 364 365 // Scalarize two-operand instruction I, using Split(Builder, X, Y, Name) 366 // to create an instruction like I with operands X and Y and name Name. 367 template<typename Splitter> 368 bool Scalarizer::splitBinary(Instruction &I, const Splitter &Split) { 369 VectorType *VT = dyn_cast<VectorType>(I.getType()); 370 if (!VT) 371 return false; 372 373 unsigned NumElems = VT->getNumElements(); 374 IRBuilder<> Builder(I.getParent(), &I); 375 Scatterer Op0 = scatter(&I, I.getOperand(0)); 376 Scatterer Op1 = scatter(&I, I.getOperand(1)); 377 assert(Op0.size() == NumElems && "Mismatched binary operation"); 378 assert(Op1.size() == NumElems && "Mismatched binary operation"); 379 ValueVector Res; 380 Res.resize(NumElems); 381 for (unsigned Elem = 0; Elem < NumElems; ++Elem) 382 Res[Elem] = Split(Builder, Op0[Elem], Op1[Elem], 383 I.getName() + ".i" + Twine(Elem)); 384 gather(&I, Res); 385 return true; 386 } 387 388 bool Scalarizer::visitSelectInst(SelectInst &SI) { 389 VectorType *VT = dyn_cast<VectorType>(SI.getType()); 390 if (!VT) 391 return false; 392 393 unsigned NumElems = VT->getNumElements(); 394 IRBuilder<> Builder(SI.getParent(), &SI); 395 Scatterer Op1 = scatter(&SI, SI.getOperand(1)); 396 Scatterer Op2 = scatter(&SI, SI.getOperand(2)); 397 assert(Op1.size() == NumElems && "Mismatched select"); 398 assert(Op2.size() == NumElems && "Mismatched select"); 399 ValueVector Res; 400 Res.resize(NumElems); 401 402 if (SI.getOperand(0)->getType()->isVectorTy()) { 403 Scatterer Op0 = scatter(&SI, SI.getOperand(0)); 404 assert(Op0.size() == NumElems && "Mismatched select"); 405 for (unsigned I = 0; I < NumElems; ++I) 406 Res[I] = Builder.CreateSelect(Op0[I], Op1[I], Op2[I], 407 SI.getName() + ".i" + Twine(I)); 408 } else { 409 Value *Op0 = SI.getOperand(0); 410 for (unsigned I = 0; I < NumElems; ++I) 411 Res[I] = Builder.CreateSelect(Op0, Op1[I], Op2[I], 412 SI.getName() + ".i" + Twine(I)); 413 } 414 gather(&SI, Res); 415 return true; 416 } 417 418 bool Scalarizer::visitICmpInst(ICmpInst &ICI) { 419 return splitBinary(ICI, ICmpSplitter(ICI)); 420 } 421 422 bool Scalarizer::visitFCmpInst(FCmpInst &FCI) { 423 return splitBinary(FCI, FCmpSplitter(FCI)); 424 } 425 426 bool Scalarizer::visitBinaryOperator(BinaryOperator &BO) { 427 return splitBinary(BO, BinarySplitter(BO)); 428 } 429 430 bool Scalarizer::visitGetElementPtrInst(GetElementPtrInst &GEPI) { 431 VectorType *VT = dyn_cast<VectorType>(GEPI.getType()); 432 if (!VT) 433 return false; 434 435 IRBuilder<> Builder(GEPI.getParent(), &GEPI); 436 unsigned NumElems = VT->getNumElements(); 437 unsigned NumIndices = GEPI.getNumIndices(); 438 439 Scatterer Base = scatter(&GEPI, GEPI.getOperand(0)); 440 441 SmallVector<Scatterer, 8> Ops; 442 Ops.resize(NumIndices); 443 for (unsigned I = 0; I < NumIndices; ++I) 444 Ops[I] = scatter(&GEPI, GEPI.getOperand(I + 1)); 445 446 ValueVector Res; 447 Res.resize(NumElems); 448 for (unsigned I = 0; I < NumElems; ++I) { 449 SmallVector<Value *, 8> Indices; 450 Indices.resize(NumIndices); 451 for (unsigned J = 0; J < NumIndices; ++J) 452 Indices[J] = Ops[J][I]; 453 Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices, 454 GEPI.getName() + ".i" + Twine(I)); 455 if (GEPI.isInBounds()) 456 if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I])) 457 NewGEPI->setIsInBounds(); 458 } 459 gather(&GEPI, Res); 460 return true; 461 } 462 463 bool Scalarizer::visitCastInst(CastInst &CI) { 464 VectorType *VT = dyn_cast<VectorType>(CI.getDestTy()); 465 if (!VT) 466 return false; 467 468 unsigned NumElems = VT->getNumElements(); 469 IRBuilder<> Builder(CI.getParent(), &CI); 470 Scatterer Op0 = scatter(&CI, CI.getOperand(0)); 471 assert(Op0.size() == NumElems && "Mismatched cast"); 472 ValueVector Res; 473 Res.resize(NumElems); 474 for (unsigned I = 0; I < NumElems; ++I) 475 Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(), 476 CI.getName() + ".i" + Twine(I)); 477 gather(&CI, Res); 478 return true; 479 } 480 481 bool Scalarizer::visitBitCastInst(BitCastInst &BCI) { 482 VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy()); 483 VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy()); 484 if (!DstVT || !SrcVT) 485 return false; 486 487 unsigned DstNumElems = DstVT->getNumElements(); 488 unsigned SrcNumElems = SrcVT->getNumElements(); 489 IRBuilder<> Builder(BCI.getParent(), &BCI); 490 Scatterer Op0 = scatter(&BCI, BCI.getOperand(0)); 491 ValueVector Res; 492 Res.resize(DstNumElems); 493 494 if (DstNumElems == SrcNumElems) { 495 for (unsigned I = 0; I < DstNumElems; ++I) 496 Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(), 497 BCI.getName() + ".i" + Twine(I)); 498 } else if (DstNumElems > SrcNumElems) { 499 // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the 500 // individual elements to the destination. 501 unsigned FanOut = DstNumElems / SrcNumElems; 502 Type *MidTy = VectorType::get(DstVT->getElementType(), FanOut); 503 unsigned ResI = 0; 504 for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) { 505 Value *V = Op0[Op0I]; 506 Instruction *VI; 507 // Look through any existing bitcasts before converting to <N x t2>. 508 // In the best case, the resulting conversion might be a no-op. 509 while ((VI = dyn_cast<Instruction>(V)) && 510 VI->getOpcode() == Instruction::BitCast) 511 V = VI->getOperand(0); 512 V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast"); 513 Scatterer Mid = scatter(&BCI, V); 514 for (unsigned MidI = 0; MidI < FanOut; ++MidI) 515 Res[ResI++] = Mid[MidI]; 516 } 517 } else { 518 // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2. 519 unsigned FanIn = SrcNumElems / DstNumElems; 520 Type *MidTy = VectorType::get(SrcVT->getElementType(), FanIn); 521 unsigned Op0I = 0; 522 for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) { 523 Value *V = UndefValue::get(MidTy); 524 for (unsigned MidI = 0; MidI < FanIn; ++MidI) 525 V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI), 526 BCI.getName() + ".i" + Twine(ResI) 527 + ".upto" + Twine(MidI)); 528 Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(), 529 BCI.getName() + ".i" + Twine(ResI)); 530 } 531 } 532 gather(&BCI, Res); 533 return true; 534 } 535 536 bool Scalarizer::visitShuffleVectorInst(ShuffleVectorInst &SVI) { 537 VectorType *VT = dyn_cast<VectorType>(SVI.getType()); 538 if (!VT) 539 return false; 540 541 unsigned NumElems = VT->getNumElements(); 542 Scatterer Op0 = scatter(&SVI, SVI.getOperand(0)); 543 Scatterer Op1 = scatter(&SVI, SVI.getOperand(1)); 544 ValueVector Res; 545 Res.resize(NumElems); 546 547 for (unsigned I = 0; I < NumElems; ++I) { 548 int Selector = SVI.getMaskValue(I); 549 if (Selector < 0) 550 Res[I] = UndefValue::get(VT->getElementType()); 551 else if (unsigned(Selector) < Op0.size()) 552 Res[I] = Op0[Selector]; 553 else 554 Res[I] = Op1[Selector - Op0.size()]; 555 } 556 gather(&SVI, Res); 557 return true; 558 } 559 560 bool Scalarizer::visitPHINode(PHINode &PHI) { 561 VectorType *VT = dyn_cast<VectorType>(PHI.getType()); 562 if (!VT) 563 return false; 564 565 unsigned NumElems = VT->getNumElements(); 566 IRBuilder<> Builder(PHI.getParent(), &PHI); 567 ValueVector Res; 568 Res.resize(NumElems); 569 570 unsigned NumOps = PHI.getNumOperands(); 571 for (unsigned I = 0; I < NumElems; ++I) 572 Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps, 573 PHI.getName() + ".i" + Twine(I)); 574 575 for (unsigned I = 0; I < NumOps; ++I) { 576 Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I)); 577 BasicBlock *IncomingBlock = PHI.getIncomingBlock(I); 578 for (unsigned J = 0; J < NumElems; ++J) 579 cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock); 580 } 581 gather(&PHI, Res); 582 return true; 583 } 584 585 bool Scalarizer::visitLoadInst(LoadInst &LI) { 586 if (!ScalarizeLoadStore) 587 return false; 588 if (!LI.isSimple()) 589 return false; 590 591 VectorLayout Layout; 592 if (!getVectorLayout(LI.getType(), LI.getAlignment(), Layout, 593 LI.getModule()->getDataLayout())) 594 return false; 595 596 unsigned NumElems = Layout.VecTy->getNumElements(); 597 IRBuilder<> Builder(LI.getParent(), &LI); 598 Scatterer Ptr = scatter(&LI, LI.getPointerOperand()); 599 ValueVector Res; 600 Res.resize(NumElems); 601 602 for (unsigned I = 0; I < NumElems; ++I) 603 Res[I] = Builder.CreateAlignedLoad(Ptr[I], Layout.getElemAlign(I), 604 LI.getName() + ".i" + Twine(I)); 605 gather(&LI, Res); 606 return true; 607 } 608 609 bool Scalarizer::visitStoreInst(StoreInst &SI) { 610 if (!ScalarizeLoadStore) 611 return false; 612 if (!SI.isSimple()) 613 return false; 614 615 VectorLayout Layout; 616 Value *FullValue = SI.getValueOperand(); 617 if (!getVectorLayout(FullValue->getType(), SI.getAlignment(), Layout, 618 SI.getModule()->getDataLayout())) 619 return false; 620 621 unsigned NumElems = Layout.VecTy->getNumElements(); 622 IRBuilder<> Builder(SI.getParent(), &SI); 623 Scatterer Ptr = scatter(&SI, SI.getPointerOperand()); 624 Scatterer Val = scatter(&SI, FullValue); 625 626 ValueVector Stores; 627 Stores.resize(NumElems); 628 for (unsigned I = 0; I < NumElems; ++I) { 629 unsigned Align = Layout.getElemAlign(I); 630 Stores[I] = Builder.CreateAlignedStore(Val[I], Ptr[I], Align); 631 } 632 transferMetadata(&SI, Stores); 633 return true; 634 } 635 636 // Delete the instructions that we scalarized. If a full vector result 637 // is still needed, recreate it using InsertElements. 638 bool Scalarizer::finish() { 639 if (Gathered.empty()) 640 return false; 641 for (GatherList::iterator GMI = Gathered.begin(), GME = Gathered.end(); 642 GMI != GME; ++GMI) { 643 Instruction *Op = GMI->first; 644 ValueVector &CV = *GMI->second; 645 if (!Op->use_empty()) { 646 // The value is still needed, so recreate it using a series of 647 // InsertElements. 648 Type *Ty = Op->getType(); 649 Value *Res = UndefValue::get(Ty); 650 BasicBlock *BB = Op->getParent(); 651 unsigned Count = Ty->getVectorNumElements(); 652 IRBuilder<> Builder(BB, Op); 653 if (isa<PHINode>(Op)) 654 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); 655 for (unsigned I = 0; I < Count; ++I) 656 Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I), 657 Op->getName() + ".upto" + Twine(I)); 658 Res->takeName(Op); 659 Op->replaceAllUsesWith(Res); 660 } 661 Op->eraseFromParent(); 662 } 663 Gathered.clear(); 664 Scattered.clear(); 665 return true; 666 } 667 668 FunctionPass *llvm::createScalarizerPass() { 669 return new Scalarizer(); 670 } 671