Home | History | Annotate | Download | only in X86 
      1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
      2 //
      3 //                     The LLVM Compiler Infrastructure
      4 //
      5 // This file is distributed under the University of Illinois Open Source
      6 // License. See LICENSE.TXT for details.
      7 //
      8 //===----------------------------------------------------------------------===//
      9 //
     10 // Define several functions to decode x86 specific shuffle semantics using
     11 // constants from the constant pool.
     12 //
     13 //===----------------------------------------------------------------------===//
     14 
     15 #include "Utils/X86ShuffleDecode.h"
     16 #include "llvm/ADT/APInt.h"
     17 #include "llvm/IR/Constants.h"
     18 
     19 //===----------------------------------------------------------------------===//
     20 //  Vector Mask Decoding
     21 //===----------------------------------------------------------------------===//
     22 
     23 namespace llvm {
     24 
     25 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
     26                                 APInt &UndefElts,
     27                                 SmallVectorImpl<uint64_t> &RawMask) {
     28   // It is not an error for shuffle masks to not be a vector of
     29   // MaskEltSizeInBits because the constant pool uniques constants by their
     30   // bit representation.
     31   // e.g. the following take up the same space in the constant pool:
     32   //   i128 -170141183420855150465331762880109871104
     33   //
     34   //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
     35   //
     36   //   <4 x i32> <i32 -2147483648, i32 -2147483648,
     37   //              i32 -2147483648, i32 -2147483648>
     38   Type *CstTy = C->getType();
     39   if (!CstTy->isVectorTy())
     40     return false;
     41 
     42   Type *CstEltTy = CstTy->getVectorElementType();
     43   if (!CstEltTy->isIntegerTy())
     44     return false;
     45 
     46   unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
     47   unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
     48   unsigned NumCstElts = CstTy->getVectorNumElements();
     49 
     50   assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
     51          "Unaligned shuffle mask size");
     52 
     53   unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
     54   UndefElts = APInt(NumMaskElts, 0);
     55   RawMask.resize(NumMaskElts, 0);
     56 
     57   // Fast path - if the constants match the mask size then copy direct.
     58   if (MaskEltSizeInBits == CstEltSizeInBits) {
     59     assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
     60     for (unsigned i = 0; i != NumMaskElts; ++i) {
     61       Constant *COp = C->getAggregateElement(i);
     62       if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
     63         return false;
     64 
     65       if (isa<UndefValue>(COp)) {
     66         UndefElts.setBit(i);
     67         RawMask[i] = 0;
     68         continue;
     69       }
     70 
     71       auto *Elt = cast<ConstantInt>(COp);
     72       RawMask[i] = Elt->getValue().getZExtValue();
     73     }
     74     return true;
     75   }
     76 
     77   // Extract all the undef/constant element data and pack into single bitsets.
     78   APInt UndefBits(CstSizeInBits, 0);
     79   APInt MaskBits(CstSizeInBits, 0);
     80   for (unsigned i = 0; i != NumCstElts; ++i) {
     81     Constant *COp = C->getAggregateElement(i);
     82     if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
     83       return false;
     84 
     85     unsigned BitOffset = i * CstEltSizeInBits;
     86 
     87     if (isa<UndefValue>(COp)) {
     88       UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
     89       continue;
     90     }
     91 
     92     MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
     93   }
     94 
     95   // Now extract the undef/constant bit data into the raw shuffle masks.
     96   for (unsigned i = 0; i != NumMaskElts; ++i) {
     97     unsigned BitOffset = i * MaskEltSizeInBits;
     98     APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
     99 
    100     // Only treat the element as UNDEF if all bits are UNDEF, otherwise
    101     // treat it as zero.
    102     if (EltUndef.isAllOnesValue()) {
    103       UndefElts.setBit(i);
    104       RawMask[i] = 0;
    105       continue;
    106     }
    107 
    108     APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
    109     RawMask[i] = EltBits.getZExtValue();
    110   }
    111 
    112   return true;
    113 }
    114 
    115 void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
    116   Type *MaskTy = C->getType();
    117   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
    118   (void)MaskTySize;
    119   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
    120          "Unexpected vector size.");
    121 
    122   // The shuffle mask requires a byte vector.
    123   APInt UndefElts;
    124   SmallVector<uint64_t, 64> RawMask;
    125   if (!extractConstantMask(C, 8, UndefElts, RawMask))
    126     return;
    127 
    128   unsigned NumElts = RawMask.size();
    129   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
    130          "Unexpected number of vector elements.");
    131 
    132   for (unsigned i = 0; i != NumElts; ++i) {
    133     if (UndefElts[i]) {
    134       ShuffleMask.push_back(SM_SentinelUndef);
    135       continue;
    136     }
    137 
    138     uint64_t Element = RawMask[i];
    139     // If the high bit (7) of the byte is set, the element is zeroed.
    140     if (Element & (1 << 7))
    141       ShuffleMask.push_back(SM_SentinelZero);
    142     else {
    143       // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
    144       // lane of the vector we're inside.
    145       unsigned Base = i & ~0xf;
    146 
    147       // Only the least significant 4 bits of the byte are used.
    148       int Index = Base + (Element & 0xf);
    149       ShuffleMask.push_back(Index);
    150     }
    151   }
    152 }
    153 
    154 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
    155                         SmallVectorImpl<int> &ShuffleMask) {
    156   Type *MaskTy = C->getType();
    157   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
    158   (void)MaskTySize;
    159   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
    160          "Unexpected vector size.");
    161   assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
    162 
    163   // The shuffle mask requires elements the same size as the target.
    164   APInt UndefElts;
    165   SmallVector<uint64_t, 16> RawMask;
    166   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    167     return;
    168 
    169   unsigned NumElts = RawMask.size();
    170   unsigned NumEltsPerLane = 128 / ElSize;
    171   assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
    172          "Unexpected number of vector elements.");
    173 
    174   for (unsigned i = 0; i != NumElts; ++i) {
    175     if (UndefElts[i]) {
    176       ShuffleMask.push_back(SM_SentinelUndef);
    177       continue;
    178     }
    179 
    180     int Index = i & ~(NumEltsPerLane - 1);
    181     uint64_t Element = RawMask[i];
    182     if (ElSize == 64)
    183       Index += (Element >> 1) & 0x1;
    184     else
    185       Index += Element & 0x3;
    186 
    187     ShuffleMask.push_back(Index);
    188   }
    189 }
    190 
    191 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
    192                          SmallVectorImpl<int> &ShuffleMask) {
    193   Type *MaskTy = C->getType();
    194   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
    195   (void)MaskTySize;
    196   assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size.");
    197 
    198   // The shuffle mask requires elements the same size as the target.
    199   APInt UndefElts;
    200   SmallVector<uint64_t, 8> RawMask;
    201   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    202     return;
    203 
    204   unsigned NumElts = RawMask.size();
    205   unsigned NumEltsPerLane = 128 / ElSize;
    206   assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
    207          "Unexpected number of vector elements.");
    208 
    209   for (unsigned i = 0; i != NumElts; ++i) {
    210     if (UndefElts[i]) {
    211       ShuffleMask.push_back(SM_SentinelUndef);
    212       continue;
    213     }
    214 
    215     // VPERMIL2 Operation.
    216     // Bits[3] - Match Bit.
    217     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
    218     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
    219     uint64_t Selector = RawMask[i];
    220     unsigned MatchBit = (Selector >> 3) & 0x1;
    221 
    222     // M2Z[0:1]     MatchBit
    223     //   0Xb           X        Source selected by Selector index.
    224     //   10b           0        Source selected by Selector index.
    225     //   10b           1        Zero.
    226     //   11b           0        Zero.
    227     //   11b           1        Source selected by Selector index.
    228     if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
    229       ShuffleMask.push_back(SM_SentinelZero);
    230       continue;
    231     }
    232 
    233     int Index = i & ~(NumEltsPerLane - 1);
    234     if (ElSize == 64)
    235       Index += (Selector >> 1) & 0x1;
    236     else
    237       Index += Selector & 0x3;
    238 
    239     int Src = (Selector >> 2) & 0x1;
    240     Index += Src * NumElts;
    241     ShuffleMask.push_back(Index);
    242   }
    243 }
    244 
    245 void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
    246   assert(C->getType()->getPrimitiveSizeInBits() == 128 &&
    247          "Unexpected vector size.");
    248 
    249   // The shuffle mask requires a byte vector.
    250   APInt UndefElts;
    251   SmallVector<uint64_t, 16> RawMask;
    252   if (!extractConstantMask(C, 8, UndefElts, RawMask))
    253     return;
    254 
    255   unsigned NumElts = RawMask.size();
    256   assert(NumElts == 16 && "Unexpected number of vector elements.");
    257 
    258   for (unsigned i = 0; i != NumElts; ++i) {
    259     if (UndefElts[i]) {
    260       ShuffleMask.push_back(SM_SentinelUndef);
    261       continue;
    262     }
    263 
    264     // VPPERM Operation
    265     // Bits[4:0] - Byte Index (0 - 31)
    266     // Bits[7:5] - Permute Operation
    267     //
    268     // Permute Operation:
    269     // 0 - Source byte (no logical operation).
    270     // 1 - Invert source byte.
    271     // 2 - Bit reverse of source byte.
    272     // 3 - Bit reverse of inverted source byte.
    273     // 4 - 00h (zero - fill).
    274     // 5 - FFh (ones - fill).
    275     // 6 - Most significant bit of source byte replicated in all bit positions.
    276     // 7 - Invert most significant bit of source byte and replicate in all bit
    277     // positions.
    278     uint64_t Element = RawMask[i];
    279     uint64_t Index = Element & 0x1F;
    280     uint64_t PermuteOp = (Element >> 5) & 0x7;
    281 
    282     if (PermuteOp == 4) {
    283       ShuffleMask.push_back(SM_SentinelZero);
    284       continue;
    285     }
    286     if (PermuteOp != 0) {
    287       ShuffleMask.clear();
    288       return;
    289     }
    290     ShuffleMask.push_back((int)Index);
    291   }
    292 }
    293 
    294 void DecodeVPERMVMask(const Constant *C, unsigned ElSize,
    295                       SmallVectorImpl<int> &ShuffleMask) {
    296   Type *MaskTy = C->getType();
    297   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
    298   (void)MaskTySize;
    299   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
    300          "Unexpected vector size.");
    301   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
    302          "Unexpected vector element size.");
    303 
    304   // The shuffle mask requires elements the same size as the target.
    305   APInt UndefElts;
    306   SmallVector<uint64_t, 64> RawMask;
    307   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    308     return;
    309 
    310   unsigned NumElts = RawMask.size();
    311 
    312   for (unsigned i = 0; i != NumElts; ++i) {
    313     if (UndefElts[i]) {
    314       ShuffleMask.push_back(SM_SentinelUndef);
    315       continue;
    316     }
    317     int Index = RawMask[i] & (NumElts - 1);
    318     ShuffleMask.push_back(Index);
    319   }
    320 }
    321 
    322 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize,
    323                        SmallVectorImpl<int> &ShuffleMask) {
    324   Type *MaskTy = C->getType();
    325   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
    326   (void)MaskTySize;
    327   assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
    328          "Unexpected vector size.");
    329   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
    330          "Unexpected vector element size.");
    331 
    332   // The shuffle mask requires elements the same size as the target.
    333   APInt UndefElts;
    334   SmallVector<uint64_t, 64> RawMask;
    335   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    336     return;
    337 
    338   unsigned NumElts = RawMask.size();
    339 
    340   for (unsigned i = 0; i != NumElts; ++i) {
    341     if (UndefElts[i]) {
    342       ShuffleMask.push_back(SM_SentinelUndef);
    343       continue;
    344     }
    345     int Index = RawMask[i] & (NumElts*2 - 1);
    346     ShuffleMask.push_back(Index);
    347   }
    348 }
    349 } // llvm namespace
    350