Home | History | Annotate | Download | only in Analysis 
      1 //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 
     10 #include "llvm/Analysis/TargetTransformInfo.h"
     11 #include "llvm/Analysis/TargetTransformInfoImpl.h"
     12 #include "llvm/IR/CallSite.h"
     13 #include "llvm/IR/DataLayout.h"
     14 #include "llvm/IR/Instruction.h"
     15 #include "llvm/IR/Instructions.h"
     16 #include "llvm/IR/IntrinsicInst.h"
     17 #include "llvm/IR/Module.h"
     18 #include "llvm/IR/Operator.h"
     19 #include "llvm/IR/PatternMatch.h"
     20 #include "llvm/Support/CommandLine.h"
     21 #include "llvm/Support/ErrorHandling.h"
     22 #include <utility>
     23 
     24 using namespace llvm;
     25 using namespace PatternMatch;
     26 
     27 #define DEBUG_TYPE "tti"
     28 
     29 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
     30                                      cl::Hidden,
     31                                      cl::desc("Recognize reduction patterns."));
     32 
     33 namespace {
     34 /// No-op implementation of the TTI interface using the utility base
     35 /// classes.
     36 ///
     37 /// This is used when no target specific information is available.
     38 struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
     39   explicit NoTTIImpl(const DataLayout &DL)
     40       : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
     41 };
     42 }
     43 
     44 TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
     45     : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
     46 
     47 TargetTransformInfo::~TargetTransformInfo() {}
     48 
     49 TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
     50     : TTIImpl(std::move(Arg.TTIImpl)) {}
     51 
     52 TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
     53   TTIImpl = std::move(RHS.TTIImpl);
     54   return *this;
     55 }
     56 
     57 int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty,
     58                                           Type *OpTy) const {
     59   int Cost = TTIImpl->getOperationCost(Opcode, Ty, OpTy);
     60   assert(Cost >= 0 && "TTI should not produce negative costs!");
     61   return Cost;
     62 }
     63 
     64 int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs) const {
     65   int Cost = TTIImpl->getCallCost(FTy, NumArgs);
     66   assert(Cost >= 0 && "TTI should not produce negative costs!");
     67   return Cost;
     68 }
     69 
     70 int TargetTransformInfo::getCallCost(const Function *F,
     71                                      ArrayRef<const Value *> Arguments) const {
     72   int Cost = TTIImpl->getCallCost(F, Arguments);
     73   assert(Cost >= 0 && "TTI should not produce negative costs!");
     74   return Cost;
     75 }
     76 
     77 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
     78   return TTIImpl->getInliningThresholdMultiplier();
     79 }
     80 
     81 int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr,
     82                                     ArrayRef<const Value *> Operands) const {
     83   return TTIImpl->getGEPCost(PointeeType, Ptr, Operands);
     84 }
     85 
     86 int TargetTransformInfo::getExtCost(const Instruction *I,
     87                                     const Value *Src) const {
     88   return TTIImpl->getExtCost(I, Src);
     89 }
     90 
     91 int TargetTransformInfo::getIntrinsicCost(
     92     Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments) const {
     93   int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments);
     94   assert(Cost >= 0 && "TTI should not produce negative costs!");
     95   return Cost;
     96 }
     97 
     98 unsigned
     99 TargetTransformInfo::getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
    100                                                       unsigned &JTSize) const {
    101   return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize);
    102 }
    103 
    104 int TargetTransformInfo::getUserCost(const User *U,
    105     ArrayRef<const Value *> Operands) const {
    106   int Cost = TTIImpl->getUserCost(U, Operands);
    107   assert(Cost >= 0 && "TTI should not produce negative costs!");
    108   return Cost;
    109 }
    110 
    111 bool TargetTransformInfo::hasBranchDivergence() const {
    112   return TTIImpl->hasBranchDivergence();
    113 }
    114 
    115 bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
    116   return TTIImpl->isSourceOfDivergence(V);
    117 }
    118 
    119 bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
    120   return TTIImpl->isAlwaysUniform(V);
    121 }
    122 
    123 unsigned TargetTransformInfo::getFlatAddressSpace() const {
    124   return TTIImpl->getFlatAddressSpace();
    125 }
    126 
    127 bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
    128   return TTIImpl->isLoweredToCall(F);
    129 }
    130 
    131 void TargetTransformInfo::getUnrollingPreferences(
    132     Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const {
    133   return TTIImpl->getUnrollingPreferences(L, SE, UP);
    134 }
    135 
    136 bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
    137   return TTIImpl->isLegalAddImmediate(Imm);
    138 }
    139 
    140 bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
    141   return TTIImpl->isLegalICmpImmediate(Imm);
    142 }
    143 
    144 bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
    145                                                 int64_t BaseOffset,
    146                                                 bool HasBaseReg,
    147                                                 int64_t Scale,
    148                                                 unsigned AddrSpace,
    149                                                 Instruction *I) const {
    150   return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
    151                                         Scale, AddrSpace, I);
    152 }
    153 
    154 bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const {
    155   return TTIImpl->isLSRCostLess(C1, C2);
    156 }
    157 
    158 bool TargetTransformInfo::canMacroFuseCmp() const {
    159   return TTIImpl->canMacroFuseCmp();
    160 }
    161 
    162 bool TargetTransformInfo::shouldFavorPostInc() const {
    163   return TTIImpl->shouldFavorPostInc();
    164 }
    165 
    166 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const {
    167   return TTIImpl->isLegalMaskedStore(DataType);
    168 }
    169 
    170 bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const {
    171   return TTIImpl->isLegalMaskedLoad(DataType);
    172 }
    173 
    174 bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const {
    175   return TTIImpl->isLegalMaskedGather(DataType);
    176 }
    177 
    178 bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const {
    179   return TTIImpl->isLegalMaskedScatter(DataType);
    180 }
    181 
    182 bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
    183   return TTIImpl->hasDivRemOp(DataType, IsSigned);
    184 }
    185 
    186 bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
    187                                              unsigned AddrSpace) const {
    188   return TTIImpl->hasVolatileVariant(I, AddrSpace);
    189 }
    190 
    191 bool TargetTransformInfo::prefersVectorizedAddressing() const {
    192   return TTIImpl->prefersVectorizedAddressing();
    193 }
    194 
    195 int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
    196                                               int64_t BaseOffset,
    197                                               bool HasBaseReg,
    198                                               int64_t Scale,
    199                                               unsigned AddrSpace) const {
    200   int Cost = TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg,
    201                                            Scale, AddrSpace);
    202   assert(Cost >= 0 && "TTI should not produce negative costs!");
    203   return Cost;
    204 }
    205 
    206 bool TargetTransformInfo::LSRWithInstrQueries() const {
    207   return TTIImpl->LSRWithInstrQueries();
    208 }
    209 
    210 bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
    211   return TTIImpl->isTruncateFree(Ty1, Ty2);
    212 }
    213 
    214 bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
    215   return TTIImpl->isProfitableToHoist(I);
    216 }
    217 
    218 bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
    219 
    220 bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
    221   return TTIImpl->isTypeLegal(Ty);
    222 }
    223 
    224 unsigned TargetTransformInfo::getJumpBufAlignment() const {
    225   return TTIImpl->getJumpBufAlignment();
    226 }
    227 
    228 unsigned TargetTransformInfo::getJumpBufSize() const {
    229   return TTIImpl->getJumpBufSize();
    230 }
    231 
    232 bool TargetTransformInfo::shouldBuildLookupTables() const {
    233   return TTIImpl->shouldBuildLookupTables();
    234 }
    235 bool TargetTransformInfo::shouldBuildLookupTablesForConstant(Constant *C) const {
    236   return TTIImpl->shouldBuildLookupTablesForConstant(C);
    237 }
    238 
    239 bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
    240   return TTIImpl->useColdCCForColdCall(F);
    241 }
    242 
    243 unsigned TargetTransformInfo::
    244 getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const {
    245   return TTIImpl->getScalarizationOverhead(Ty, Insert, Extract);
    246 }
    247 
    248 unsigned TargetTransformInfo::
    249 getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
    250                                  unsigned VF) const {
    251   return TTIImpl->getOperandsScalarizationOverhead(Args, VF);
    252 }
    253 
    254 bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
    255   return TTIImpl->supportsEfficientVectorElementLoadStore();
    256 }
    257 
    258 bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const {
    259   return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
    260 }
    261 
    262 const TargetTransformInfo::MemCmpExpansionOptions *
    263 TargetTransformInfo::enableMemCmpExpansion(bool IsZeroCmp) const {
    264   return TTIImpl->enableMemCmpExpansion(IsZeroCmp);
    265 }
    266 
    267 bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
    268   return TTIImpl->enableInterleavedAccessVectorization();
    269 }
    270 
    271 bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
    272   return TTIImpl->isFPVectorizationPotentiallyUnsafe();
    273 }
    274 
    275 bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
    276                                                          unsigned BitWidth,
    277                                                          unsigned AddressSpace,
    278                                                          unsigned Alignment,
    279                                                          bool *Fast) const {
    280   return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace,
    281                                                  Alignment, Fast);
    282 }
    283 
    284 TargetTransformInfo::PopcntSupportKind
    285 TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
    286   return TTIImpl->getPopcntSupport(IntTyWidthInBit);
    287 }
    288 
    289 bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
    290   return TTIImpl->haveFastSqrt(Ty);
    291 }
    292 
    293 bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
    294   return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
    295 }
    296 
    297 int TargetTransformInfo::getFPOpCost(Type *Ty) const {
    298   int Cost = TTIImpl->getFPOpCost(Ty);
    299   assert(Cost >= 0 && "TTI should not produce negative costs!");
    300   return Cost;
    301 }
    302 
    303 int TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
    304                                                const APInt &Imm,
    305                                                Type *Ty) const {
    306   int Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
    307   assert(Cost >= 0 && "TTI should not produce negative costs!");
    308   return Cost;
    309 }
    310 
    311 int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const {
    312   int Cost = TTIImpl->getIntImmCost(Imm, Ty);
    313   assert(Cost >= 0 && "TTI should not produce negative costs!");
    314   return Cost;
    315 }
    316 
    317 int TargetTransformInfo::getIntImmCost(unsigned Opcode, unsigned Idx,
    318                                        const APInt &Imm, Type *Ty) const {
    319   int Cost = TTIImpl->getIntImmCost(Opcode, Idx, Imm, Ty);
    320   assert(Cost >= 0 && "TTI should not produce negative costs!");
    321   return Cost;
    322 }
    323 
    324 int TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
    325                                        const APInt &Imm, Type *Ty) const {
    326   int Cost = TTIImpl->getIntImmCost(IID, Idx, Imm, Ty);
    327   assert(Cost >= 0 && "TTI should not produce negative costs!");
    328   return Cost;
    329 }
    330 
    331 unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const {
    332   return TTIImpl->getNumberOfRegisters(Vector);
    333 }
    334 
    335 unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const {
    336   return TTIImpl->getRegisterBitWidth(Vector);
    337 }
    338 
    339 unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
    340   return TTIImpl->getMinVectorRegisterBitWidth();
    341 }
    342 
    343 bool TargetTransformInfo::shouldMaximizeVectorBandwidth(bool OptSize) const {
    344   return TTIImpl->shouldMaximizeVectorBandwidth(OptSize);
    345 }
    346 
    347 unsigned TargetTransformInfo::getMinimumVF(unsigned ElemWidth) const {
    348   return TTIImpl->getMinimumVF(ElemWidth);
    349 }
    350 
    351 bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
    352     const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
    353   return TTIImpl->shouldConsiderAddressTypePromotion(
    354       I, AllowPromotionWithoutCommonHeader);
    355 }
    356 
    357 unsigned TargetTransformInfo::getCacheLineSize() const {
    358   return TTIImpl->getCacheLineSize();
    359 }
    360 
    361 llvm::Optional<unsigned> TargetTransformInfo::getCacheSize(CacheLevel Level)
    362   const {
    363   return TTIImpl->getCacheSize(Level);
    364 }
    365 
    366 llvm::Optional<unsigned> TargetTransformInfo::getCacheAssociativity(
    367   CacheLevel Level) const {
    368   return TTIImpl->getCacheAssociativity(Level);
    369 }
    370 
    371 unsigned TargetTransformInfo::getPrefetchDistance() const {
    372   return TTIImpl->getPrefetchDistance();
    373 }
    374 
    375 unsigned TargetTransformInfo::getMinPrefetchStride() const {
    376   return TTIImpl->getMinPrefetchStride();
    377 }
    378 
    379 unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
    380   return TTIImpl->getMaxPrefetchIterationsAhead();
    381 }
    382 
    383 unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
    384   return TTIImpl->getMaxInterleaveFactor(VF);
    385 }
    386 
    387 int TargetTransformInfo::getArithmeticInstrCost(
    388     unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
    389     OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo,
    390     OperandValueProperties Opd2PropInfo,
    391     ArrayRef<const Value *> Args) const {
    392   int Cost = TTIImpl->getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
    393                                              Opd1PropInfo, Opd2PropInfo, Args);
    394   assert(Cost >= 0 && "TTI should not produce negative costs!");
    395   return Cost;
    396 }
    397 
    398 int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, Type *Ty, int Index,
    399                                         Type *SubTp) const {
    400   int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp);
    401   assert(Cost >= 0 && "TTI should not produce negative costs!");
    402   return Cost;
    403 }
    404 
    405 int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst,
    406                                  Type *Src, const Instruction *I) const {
    407   assert ((I == nullptr || I->getOpcode() == Opcode) &&
    408           "Opcode should reflect passed instruction.");
    409   int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, I);
    410   assert(Cost >= 0 && "TTI should not produce negative costs!");
    411   return Cost;
    412 }
    413 
    414 int TargetTransformInfo::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
    415                                                   VectorType *VecTy,
    416                                                   unsigned Index) const {
    417   int Cost = TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
    418   assert(Cost >= 0 && "TTI should not produce negative costs!");
    419   return Cost;
    420 }
    421 
    422 int TargetTransformInfo::getCFInstrCost(unsigned Opcode) const {
    423   int Cost = TTIImpl->getCFInstrCost(Opcode);
    424   assert(Cost >= 0 && "TTI should not produce negative costs!");
    425   return Cost;
    426 }
    427 
    428 int TargetTransformInfo::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
    429                                  Type *CondTy, const Instruction *I) const {
    430   assert ((I == nullptr || I->getOpcode() == Opcode) &&
    431           "Opcode should reflect passed instruction.");
    432   int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
    433   assert(Cost >= 0 && "TTI should not produce negative costs!");
    434   return Cost;
    435 }
    436 
    437 int TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val,
    438                                             unsigned Index) const {
    439   int Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index);
    440   assert(Cost >= 0 && "TTI should not produce negative costs!");
    441   return Cost;
    442 }
    443 
    444 int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src,
    445                                          unsigned Alignment,
    446                                          unsigned AddressSpace,
    447                                          const Instruction *I) const {
    448   assert ((I == nullptr || I->getOpcode() == Opcode) &&
    449           "Opcode should reflect passed instruction.");
    450   int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I);
    451   assert(Cost >= 0 && "TTI should not produce negative costs!");
    452   return Cost;
    453 }
    454 
    455 int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
    456                                                unsigned Alignment,
    457                                                unsigned AddressSpace) const {
    458   int Cost =
    459       TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
    460   assert(Cost >= 0 && "TTI should not produce negative costs!");
    461   return Cost;
    462 }
    463 
    464 int TargetTransformInfo::getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
    465                                                 Value *Ptr, bool VariableMask,
    466                                                 unsigned Alignment) const {
    467   int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
    468                                              Alignment);
    469   assert(Cost >= 0 && "TTI should not produce negative costs!");
    470   return Cost;
    471 }
    472 
    473 int TargetTransformInfo::getInterleavedMemoryOpCost(
    474     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
    475     unsigned Alignment, unsigned AddressSpace) const {
    476   int Cost = TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
    477                                                  Alignment, AddressSpace);
    478   assert(Cost >= 0 && "TTI should not produce negative costs!");
    479   return Cost;
    480 }
    481 
    482 int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
    483                                     ArrayRef<Type *> Tys, FastMathFlags FMF,
    484                                     unsigned ScalarizationCostPassed) const {
    485   int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys, FMF,
    486                                             ScalarizationCostPassed);
    487   assert(Cost >= 0 && "TTI should not produce negative costs!");
    488   return Cost;
    489 }
    490 
    491 int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
    492            ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF) const {
    493   int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF);
    494   assert(Cost >= 0 && "TTI should not produce negative costs!");
    495   return Cost;
    496 }
    497 
    498 int TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
    499                                           ArrayRef<Type *> Tys) const {
    500   int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys);
    501   assert(Cost >= 0 && "TTI should not produce negative costs!");
    502   return Cost;
    503 }
    504 
    505 unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
    506   return TTIImpl->getNumberOfParts(Tp);
    507 }
    508 
    509 int TargetTransformInfo::getAddressComputationCost(Type *Tp,
    510                                                    ScalarEvolution *SE,
    511                                                    const SCEV *Ptr) const {
    512   int Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
    513   assert(Cost >= 0 && "TTI should not produce negative costs!");
    514   return Cost;
    515 }
    516 
    517 int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty,
    518                                                     bool IsPairwiseForm) const {
    519   int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm);
    520   assert(Cost >= 0 && "TTI should not produce negative costs!");
    521   return Cost;
    522 }
    523 
    524 int TargetTransformInfo::getMinMaxReductionCost(Type *Ty, Type *CondTy,
    525                                                 bool IsPairwiseForm,
    526                                                 bool IsUnsigned) const {
    527   int Cost =
    528       TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned);
    529   assert(Cost >= 0 && "TTI should not produce negative costs!");
    530   return Cost;
    531 }
    532 
    533 unsigned
    534 TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
    535   return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
    536 }
    537 
    538 bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
    539                                              MemIntrinsicInfo &Info) const {
    540   return TTIImpl->getTgtMemIntrinsic(Inst, Info);
    541 }
    542 
    543 unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
    544   return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
    545 }
    546 
    547 Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
    548     IntrinsicInst *Inst, Type *ExpectedType) const {
    549   return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
    550 }
    551 
    552 Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context,
    553                                                      Value *Length,
    554                                                      unsigned SrcAlign,
    555                                                      unsigned DestAlign) const {
    556   return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign,
    557                                             DestAlign);
    558 }
    559 
    560 void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
    561     SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
    562     unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const {
    563   TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
    564                                              SrcAlign, DestAlign);
    565 }
    566 
    567 bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
    568                                               const Function *Callee) const {
    569   return TTIImpl->areInlineCompatible(Caller, Callee);
    570 }
    571 
    572 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
    573                                              Type *Ty) const {
    574   return TTIImpl->isIndexedLoadLegal(Mode, Ty);
    575 }
    576 
    577 bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
    578                                               Type *Ty) const {
    579   return TTIImpl->isIndexedStoreLegal(Mode, Ty);
    580 }
    581 
    582 unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
    583   return TTIImpl->getLoadStoreVecRegBitWidth(AS);
    584 }
    585 
    586 bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
    587   return TTIImpl->isLegalToVectorizeLoad(LI);
    588 }
    589 
    590 bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
    591   return TTIImpl->isLegalToVectorizeStore(SI);
    592 }
    593 
    594 bool TargetTransformInfo::isLegalToVectorizeLoadChain(
    595     unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
    596   return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
    597                                               AddrSpace);
    598 }
    599 
    600 bool TargetTransformInfo::isLegalToVectorizeStoreChain(
    601     unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
    602   return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
    603                                                AddrSpace);
    604 }
    605 
    606 unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
    607                                                   unsigned LoadSize,
    608                                                   unsigned ChainSizeInBytes,
    609                                                   VectorType *VecTy) const {
    610   return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
    611 }
    612 
    613 unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
    614                                                    unsigned StoreSize,
    615                                                    unsigned ChainSizeInBytes,
    616                                                    VectorType *VecTy) const {
    617   return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
    618 }
    619 
    620 bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode,
    621                                                 Type *Ty, ReductionFlags Flags) const {
    622   return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags);
    623 }
    624 
    625 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
    626   return TTIImpl->shouldExpandReduction(II);
    627 }
    628 
    629 int TargetTransformInfo::getInstructionLatency(const Instruction *I) const {
    630   return TTIImpl->getInstructionLatency(I);
    631 }
    632 
    633 static TargetTransformInfo::OperandValueKind
    634 getOperandInfo(Value *V, TargetTransformInfo::OperandValueProperties &OpProps) {
    635   TargetTransformInfo::OperandValueKind OpInfo =
    636       TargetTransformInfo::OK_AnyValue;
    637   OpProps = TargetTransformInfo::OP_None;
    638 
    639   if (auto *CI = dyn_cast<ConstantInt>(V)) {
    640     if (CI->getValue().isPowerOf2())
    641       OpProps = TargetTransformInfo::OP_PowerOf2;
    642     return TargetTransformInfo::OK_UniformConstantValue;
    643   }
    644 
    645   const Value *Splat = getSplatValue(V);
    646 
    647   // Check for a splat of a constant or for a non uniform vector of constants
    648   // and check if the constant(s) are all powers of two.
    649   if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {
    650     OpInfo = TargetTransformInfo::OK_NonUniformConstantValue;
    651     if (Splat) {
    652       OpInfo = TargetTransformInfo::OK_UniformConstantValue;
    653       if (auto *CI = dyn_cast<ConstantInt>(Splat))
    654         if (CI->getValue().isPowerOf2())
    655           OpProps = TargetTransformInfo::OP_PowerOf2;
    656     } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
    657       OpProps = TargetTransformInfo::OP_PowerOf2;
    658       for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
    659         if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I)))
    660           if (CI->getValue().isPowerOf2())
    661             continue;
    662         OpProps = TargetTransformInfo::OP_None;
    663         break;
    664       }
    665     }
    666   }
    667 
    668   // Check for a splat of a uniform value. This is not loop aware, so return
    669   // true only for the obviously uniform cases (argument, globalvalue)
    670   if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat)))
    671     OpInfo = TargetTransformInfo::OK_UniformValue;
    672 
    673   return OpInfo;
    674 }
    675 
    676 static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
    677                                      unsigned Level) {
    678   // We don't need a shuffle if we just want to have element 0 in position 0 of
    679   // the vector.
    680   if (!SI && Level == 0 && IsLeft)
    681     return true;
    682   else if (!SI)
    683     return false;
    684 
    685   SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1);
    686 
    687   // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether
    688   // we look at the left or right side.
    689   for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
    690     Mask[i] = val;
    691 
    692   SmallVector<int, 16> ActualMask = SI->getShuffleMask();
    693   return Mask == ActualMask;
    694 }
    695 
    696 namespace {
    697 /// Kind of the reduction data.
    698 enum ReductionKind {
    699   RK_None,           /// Not a reduction.
    700   RK_Arithmetic,     /// Binary reduction data.
    701   RK_MinMax,         /// Min/max reduction data.
    702   RK_UnsignedMinMax, /// Unsigned min/max reduction data.
    703 };
    704 /// Contains opcode + LHS/RHS parts of the reduction operations.
    705 struct ReductionData {
    706   ReductionData() = delete;
    707   ReductionData(ReductionKind Kind, unsigned Opcode, Value *LHS, Value *RHS)
    708       : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind) {
    709     assert(Kind != RK_None && "expected binary or min/max reduction only.");
    710   }
    711   unsigned Opcode = 0;
    712   Value *LHS = nullptr;
    713   Value *RHS = nullptr;
    714   ReductionKind Kind = RK_None;
    715   bool hasSameData(ReductionData &RD) const {
    716     return Kind == RD.Kind && Opcode == RD.Opcode;
    717   }
    718 };
    719 } // namespace
    720 
    721 static Optional<ReductionData> getReductionData(Instruction *I) {
    722   Value *L, *R;
    723   if (m_BinOp(m_Value(L), m_Value(R)).match(I))
    724     return ReductionData(RK_Arithmetic, I->getOpcode(), L, R);
    725   if (auto *SI = dyn_cast<SelectInst>(I)) {
    726     if (m_SMin(m_Value(L), m_Value(R)).match(SI) ||
    727         m_SMax(m_Value(L), m_Value(R)).match(SI) ||
    728         m_OrdFMin(m_Value(L), m_Value(R)).match(SI) ||
    729         m_OrdFMax(m_Value(L), m_Value(R)).match(SI) ||
    730         m_UnordFMin(m_Value(L), m_Value(R)).match(SI) ||
    731         m_UnordFMax(m_Value(L), m_Value(R)).match(SI)) {
    732       auto *CI = cast<CmpInst>(SI->getCondition());
    733       return ReductionData(RK_MinMax, CI->getOpcode(), L, R);
    734     }
    735     if (m_UMin(m_Value(L), m_Value(R)).match(SI) ||
    736         m_UMax(m_Value(L), m_Value(R)).match(SI)) {
    737       auto *CI = cast<CmpInst>(SI->getCondition());
    738       return ReductionData(RK_UnsignedMinMax, CI->getOpcode(), L, R);
    739     }
    740   }
    741   return llvm::None;
    742 }
    743 
    744 static ReductionKind matchPairwiseReductionAtLevel(Instruction *I,
    745                                                    unsigned Level,
    746                                                    unsigned NumLevels) {
    747   // Match one level of pairwise operations.
    748   // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
    749   //       <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
    750   // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
    751   //       <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
    752   // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
    753   if (!I)
    754     return RK_None;
    755 
    756   assert(I->getType()->isVectorTy() && "Expecting a vector type");
    757 
    758   Optional<ReductionData> RD = getReductionData(I);
    759   if (!RD)
    760     return RK_None;
    761 
    762   ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(RD->LHS);
    763   if (!LS && Level)
    764     return RK_None;
    765   ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(RD->RHS);
    766   if (!RS && Level)
    767     return RK_None;
    768 
    769   // On level 0 we can omit one shufflevector instruction.
    770   if (!Level && !RS && !LS)
    771     return RK_None;
    772 
    773   // Shuffle inputs must match.
    774   Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr;
    775   Value *NextLevelOpR = RS ? RS->getOperand(0) : nullptr;
    776   Value *NextLevelOp = nullptr;
    777   if (NextLevelOpR && NextLevelOpL) {
    778     // If we have two shuffles their operands must match.
    779     if (NextLevelOpL != NextLevelOpR)
    780       return RK_None;
    781 
    782     NextLevelOp = NextLevelOpL;
    783   } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) {
    784     // On the first level we can omit the shufflevector <0, undef,...>. So the
    785     // input to the other shufflevector <1, undef> must match with one of the
    786     // inputs to the current binary operation.
    787     // Example:
    788     //  %NextLevelOpL = shufflevector %R, <1, undef ...>
    789     //  %BinOp        = fadd          %NextLevelOpL, %R
    790     if (NextLevelOpL && NextLevelOpL != RD->RHS)
    791       return RK_None;
    792     else if (NextLevelOpR && NextLevelOpR != RD->LHS)
    793       return RK_None;
    794 
    795     NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS;
    796   } else
    797     return RK_None;
    798 
    799   // Check that the next levels binary operation exists and matches with the
    800   // current one.
    801   if (Level + 1 != NumLevels) {
    802     Optional<ReductionData> NextLevelRD =
    803         getReductionData(cast<Instruction>(NextLevelOp));
    804     if (!NextLevelRD || !RD->hasSameData(*NextLevelRD))
    805       return RK_None;
    806   }
    807 
    808   // Shuffle mask for pairwise operation must match.
    809   if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) {
    810     if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level))
    811       return RK_None;
    812   } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) {
    813     if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level))
    814       return RK_None;
    815   } else {
    816     return RK_None;
    817   }
    818 
    819   if (++Level == NumLevels)
    820     return RD->Kind;
    821 
    822   // Match next level.
    823   return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level,
    824                                        NumLevels);
    825 }
    826 
    827 static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot,
    828                                             unsigned &Opcode, Type *&Ty) {
    829   if (!EnableReduxCost)
    830     return RK_None;
    831 
    832   // Need to extract the first element.
    833   ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
    834   unsigned Idx = ~0u;
    835   if (CI)
    836     Idx = CI->getZExtValue();
    837   if (Idx != 0)
    838     return RK_None;
    839 
    840   auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0));
    841   if (!RdxStart)
    842     return RK_None;
    843   Optional<ReductionData> RD = getReductionData(RdxStart);
    844   if (!RD)
    845     return RK_None;
    846 
    847   Type *VecTy = RdxStart->getType();
    848   unsigned NumVecElems = VecTy->getVectorNumElements();
    849   if (!isPowerOf2_32(NumVecElems))
    850     return RK_None;
    851 
    852   // We look for a sequence of shuffle,shuffle,add triples like the following
    853   // that builds a pairwise reduction tree.
    854   //
    855   //  (X0, X1, X2, X3)
    856   //   (X0 + X1, X2 + X3, undef, undef)
    857   //    ((X0 + X1) + (X2 + X3), undef, undef, undef)
    858   //
    859   // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
    860   //       <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
    861   // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
    862   //       <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
    863   // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
    864   // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
    865   //       <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
    866   // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
    867   //       <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
    868   // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
    869   // %r = extractelement <4 x float> %bin.rdx8, i32 0
    870   if (matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)) ==
    871       RK_None)
    872     return RK_None;
    873 
    874   Opcode = RD->Opcode;
    875   Ty = VecTy;
    876 
    877   return RD->Kind;
    878 }
    879 
    880 static std::pair<Value *, ShuffleVectorInst *>
    881 getShuffleAndOtherOprd(Value *L, Value *R) {
    882   ShuffleVectorInst *S = nullptr;
    883 
    884   if ((S = dyn_cast<ShuffleVectorInst>(L)))
    885     return std::make_pair(R, S);
    886 
    887   S = dyn_cast<ShuffleVectorInst>(R);
    888   return std::make_pair(L, S);
    889 }
    890 
    891 static ReductionKind
    892 matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
    893                               unsigned &Opcode, Type *&Ty) {
    894   if (!EnableReduxCost)
    895     return RK_None;
    896 
    897   // Need to extract the first element.
    898   ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
    899   unsigned Idx = ~0u;
    900   if (CI)
    901     Idx = CI->getZExtValue();
    902   if (Idx != 0)
    903     return RK_None;
    904 
    905   auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0));
    906   if (!RdxStart)
    907     return RK_None;
    908   Optional<ReductionData> RD = getReductionData(RdxStart);
    909   if (!RD)
    910     return RK_None;
    911 
    912   Type *VecTy = ReduxRoot->getOperand(0)->getType();
    913   unsigned NumVecElems = VecTy->getVectorNumElements();
    914   if (!isPowerOf2_32(NumVecElems))
    915     return RK_None;
    916 
    917   // We look for a sequence of shuffles and adds like the following matching one
    918   // fadd, shuffle vector pair at a time.
    919   //
    920   // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef,
    921   //                           <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
    922   // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
    923   // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef,
    924   //                          <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
    925   // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
    926   // %r = extractelement <4 x float> %bin.rdx8, i32 0
    927 
    928   unsigned MaskStart = 1;
    929   Instruction *RdxOp = RdxStart;
    930   SmallVector<int, 32> ShuffleMask(NumVecElems, 0);
    931   unsigned NumVecElemsRemain = NumVecElems;
    932   while (NumVecElemsRemain - 1) {
    933     // Check for the right reduction operation.
    934     if (!RdxOp)
    935       return RK_None;
    936     Optional<ReductionData> RDLevel = getReductionData(RdxOp);
    937     if (!RDLevel || !RDLevel->hasSameData(*RD))
    938       return RK_None;
    939 
    940     Value *NextRdxOp;
    941     ShuffleVectorInst *Shuffle;
    942     std::tie(NextRdxOp, Shuffle) =
    943         getShuffleAndOtherOprd(RDLevel->LHS, RDLevel->RHS);
    944 
    945     // Check the current reduction operation and the shuffle use the same value.
    946     if (Shuffle == nullptr)
    947       return RK_None;
    948     if (Shuffle->getOperand(0) != NextRdxOp)
    949       return RK_None;
    950 
    951     // Check that shuffle masks matches.
    952     for (unsigned j = 0; j != MaskStart; ++j)
    953       ShuffleMask[j] = MaskStart + j;
    954     // Fill the rest of the mask with -1 for undef.
    955     std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
    956 
    957     SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
    958     if (ShuffleMask != Mask)
    959       return RK_None;
    960 
    961     RdxOp = dyn_cast<Instruction>(NextRdxOp);
    962     NumVecElemsRemain /= 2;
    963     MaskStart *= 2;
    964   }
    965 
    966   Opcode = RD->Opcode;
    967   Ty = VecTy;
    968   return RD->Kind;
    969 }
    970 
    971 int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const {
    972   switch (I->getOpcode()) {
    973   case Instruction::GetElementPtr:
    974     return getUserCost(I);
    975 
    976   case Instruction::Ret:
    977   case Instruction::PHI:
    978   case Instruction::Br: {
    979     return getCFInstrCost(I->getOpcode());
    980   }
    981   case Instruction::Add:
    982   case Instruction::FAdd:
    983   case Instruction::Sub:
    984   case Instruction::FSub:
    985   case Instruction::Mul:
    986   case Instruction::FMul:
    987   case Instruction::UDiv:
    988   case Instruction::SDiv:
    989   case Instruction::FDiv:
    990   case Instruction::URem:
    991   case Instruction::SRem:
    992   case Instruction::FRem:
    993   case Instruction::Shl:
    994   case Instruction::LShr:
    995   case Instruction::AShr:
    996   case Instruction::And:
    997   case Instruction::Or:
    998   case Instruction::Xor: {
    999     TargetTransformInfo::OperandValueKind Op1VK, Op2VK;
   1000     TargetTransformInfo::OperandValueProperties Op1VP, Op2VP;
   1001     Op1VK = getOperandInfo(I->getOperand(0), Op1VP);
   1002     Op2VK = getOperandInfo(I->getOperand(1), Op2VP);
   1003     SmallVector<const Value *, 2> Operands(I->operand_values());
   1004     return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK,
   1005                                   Op1VP, Op2VP, Operands);
   1006   }
   1007   case Instruction::Select: {
   1008     const SelectInst *SI = cast<SelectInst>(I);
   1009     Type *CondTy = SI->getCondition()->getType();
   1010     return getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy, I);
   1011   }
   1012   case Instruction::ICmp:
   1013   case Instruction::FCmp: {
   1014     Type *ValTy = I->getOperand(0)->getType();
   1015     return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), I);
   1016   }
   1017   case Instruction::Store: {
   1018     const StoreInst *SI = cast<StoreInst>(I);
   1019     Type *ValTy = SI->getValueOperand()->getType();
   1020     return getMemoryOpCost(I->getOpcode(), ValTy,
   1021                                 SI->getAlignment(),
   1022                                 SI->getPointerAddressSpace(), I);
   1023   }
   1024   case Instruction::Load: {
   1025     const LoadInst *LI = cast<LoadInst>(I);
   1026     return getMemoryOpCost(I->getOpcode(), I->getType(),
   1027                                 LI->getAlignment(),
   1028                                 LI->getPointerAddressSpace(), I);
   1029   }
   1030   case Instruction::ZExt:
   1031   case Instruction::SExt:
   1032   case Instruction::FPToUI:
   1033   case Instruction::FPToSI:
   1034   case Instruction::FPExt:
   1035   case Instruction::PtrToInt:
   1036   case Instruction::IntToPtr:
   1037   case Instruction::SIToFP:
   1038   case Instruction::UIToFP:
   1039   case Instruction::Trunc:
   1040   case Instruction::FPTrunc:
   1041   case Instruction::BitCast:
   1042   case Instruction::AddrSpaceCast: {
   1043     Type *SrcTy = I->getOperand(0)->getType();
   1044     return getCastInstrCost(I->getOpcode(), I->getType(), SrcTy, I);
   1045   }
   1046   case Instruction::ExtractElement: {
   1047     const ExtractElementInst * EEI = cast<ExtractElementInst>(I);
   1048     ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
   1049     unsigned Idx = -1;
   1050     if (CI)
   1051       Idx = CI->getZExtValue();
   1052 
   1053     // Try to match a reduction sequence (series of shufflevector and vector
   1054     // adds followed by a extractelement).
   1055     unsigned ReduxOpCode;
   1056     Type *ReduxType;
   1057 
   1058     switch (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType)) {
   1059     case RK_Arithmetic:
   1060       return getArithmeticReductionCost(ReduxOpCode, ReduxType,
   1061                                              /*IsPairwiseForm=*/false);
   1062     case RK_MinMax:
   1063       return getMinMaxReductionCost(
   1064           ReduxType, CmpInst::makeCmpResultType(ReduxType),
   1065           /*IsPairwiseForm=*/false, /*IsUnsigned=*/false);
   1066     case RK_UnsignedMinMax:
   1067       return getMinMaxReductionCost(
   1068           ReduxType, CmpInst::makeCmpResultType(ReduxType),
   1069           /*IsPairwiseForm=*/false, /*IsUnsigned=*/true);
   1070     case RK_None:
   1071       break;
   1072     }
   1073 
   1074     switch (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType)) {
   1075     case RK_Arithmetic:
   1076       return getArithmeticReductionCost(ReduxOpCode, ReduxType,
   1077                                              /*IsPairwiseForm=*/true);
   1078     case RK_MinMax:
   1079       return getMinMaxReductionCost(
   1080           ReduxType, CmpInst::makeCmpResultType(ReduxType),
   1081           /*IsPairwiseForm=*/true, /*IsUnsigned=*/false);
   1082     case RK_UnsignedMinMax:
   1083       return getMinMaxReductionCost(
   1084           ReduxType, CmpInst::makeCmpResultType(ReduxType),
   1085           /*IsPairwiseForm=*/true, /*IsUnsigned=*/true);
   1086     case RK_None:
   1087       break;
   1088     }
   1089 
   1090     return getVectorInstrCost(I->getOpcode(),
   1091                                    EEI->getOperand(0)->getType(), Idx);
   1092   }
   1093   case Instruction::InsertElement: {
   1094     const InsertElementInst * IE = cast<InsertElementInst>(I);
   1095     ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
   1096     unsigned Idx = -1;
   1097     if (CI)
   1098       Idx = CI->getZExtValue();
   1099     return getVectorInstrCost(I->getOpcode(),
   1100                                    IE->getType(), Idx);
   1101   }
   1102   case Instruction::ShuffleVector: {
   1103     const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
   1104     // TODO: Identify and add costs for insert/extract subvector, etc.
   1105     if (Shuffle->changesLength())
   1106       return -1;
   1107 
   1108     if (Shuffle->isIdentity())
   1109       return 0;
   1110 
   1111     Type *Ty = Shuffle->getType();
   1112     if (Shuffle->isReverse())
   1113       return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr);
   1114 
   1115     if (Shuffle->isSelect())
   1116       return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr);
   1117 
   1118     if (Shuffle->isTranspose())
   1119       return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr);
   1120 
   1121     if (Shuffle->isZeroEltSplat())
   1122       return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr);
   1123 
   1124     if (Shuffle->isSingleSource())
   1125       return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr);
   1126 
   1127     return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr);
   1128   }
   1129   case Instruction::Call:
   1130     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
   1131       SmallVector<Value *, 4> Args(II->arg_operands());
   1132 
   1133       FastMathFlags FMF;
   1134       if (auto *FPMO = dyn_cast<FPMathOperator>(II))
   1135         FMF = FPMO->getFastMathFlags();
   1136 
   1137       return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(),
   1138                                         Args, FMF);
   1139     }
   1140     return -1;
   1141   default:
   1142     // We don't have any information on this instruction.
   1143     return -1;
   1144   }
   1145 }
   1146 
   1147 TargetTransformInfo::Concept::~Concept() {}
   1148 
   1149 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
   1150 
   1151 TargetIRAnalysis::TargetIRAnalysis(
   1152     std::function<Result(const Function &)> TTICallback)
   1153     : TTICallback(std::move(TTICallback)) {}
   1154 
   1155 TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
   1156                                                FunctionAnalysisManager &) {
   1157   return TTICallback(F);
   1158 }
   1159 
   1160 AnalysisKey TargetIRAnalysis::Key;
   1161 
   1162 TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
   1163   return Result(F.getParent()->getDataLayout());
   1164 }
   1165 
   1166 // Register the basic pass.
   1167 INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
   1168                 "Target Transform Information", false, true)
   1169 char TargetTransformInfoWrapperPass::ID = 0;
   1170 
   1171 void TargetTransformInfoWrapperPass::anchor() {}
   1172 
   1173 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
   1174     : ImmutablePass(ID) {
   1175   initializeTargetTransformInfoWrapperPassPass(
   1176       *PassRegistry::getPassRegistry());
   1177 }
   1178 
   1179 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
   1180     TargetIRAnalysis TIRA)
   1181     : ImmutablePass(ID), TIRA(std::move(TIRA)) {
   1182   initializeTargetTransformInfoWrapperPassPass(
   1183       *PassRegistry::getPassRegistry());
   1184 }
   1185 
   1186 TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
   1187   FunctionAnalysisManager DummyFAM;
   1188   TTI = TIRA.run(F, DummyFAM);
   1189   return *TTI;
   1190 }
   1191 
   1192 ImmutablePass *
   1193 llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
   1194   return new TargetTransformInfoWrapperPass(std::move(TIRA));
   1195 }
   1196