Lines Matching refs:Zeroable
4979 const APInt &Zeroable,
4985 if (Zeroable[i])
6071 /// (not just zeroable) from their inputs.
6589 // Find all zeroable elements.
6590 std::bitset<4> Zeroable;
6593 Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
6595 assert(Zeroable.size() - Zeroable.count() > 1 &&
6599 // zeroable or extract_vector_elt with constant index.
6603 if (Zeroable[i])
6628 if (Zeroable[EltIdx]) {
6660 if (Zeroable[i])
6680 unsigned ZMask = Zeroable.to_ulong();
9082 // Merges a general DAG shuffle mask and zeroable bit mask into a target shuffle
9085 const APInt &Zeroable) {
9087 assert(NumElts == (int)Zeroable.getBitWidth() && "Mismatch mask sizes");
9095 TargetMask[i] = (Zeroable[i] ? SM_SentinelZero : M);
9145 /// Compute whether each element of a shuffle is zeroable.
9147 /// A "zeroable" vector shuffle element is one which can be lowered to zero.
9155 APInt Zeroable(Mask.size(), 0);
9170 Zeroable.setBit(i);
9188 Zeroable.setBit(i);
9194 Zeroable.setBit(i);
9200 Zeroable.setBit(i);
9215 Zeroable.setBit(i);
9220 return Zeroable;
9225 // Each Zeroable's element correspond to a particular Mask's element.
9230 static bool isNonZeroElementsInOrder(const APInt &Zeroable,
9240 if (Zeroable[i])
9259 const APInt &Zeroable,
9282 if (Zeroable[i / NumEltBytes]) {
9316 const APInt &Zeroable,
9321 if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
9324 unsigned VEXPANDMask = (~Zeroable).getZExtValue();
9357 "Zeroable shuffle detected");
9616 /// one of the inputs being zeroable.
9619 const APInt &Zeroable,
9628 if (Zeroable[i])
9640 return SDValue(); // No non-zeroable elements!
9741 const APInt &Zeroable,
9744 SmallVector<int, 64> Mask = createTargetShuffleMask(Original, Zeroable);
9827 lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG))
10144 /// right with zeroable elements 'shifted in'. It handles both the strictly
10165 const APInt &Zeroable,
10173 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
10210 // and that the shifted in elements are all zeroable.
10227 const APInt &Zeroable,
10239 ShiftVT, Opcode, VT.getScalarSizeInBits(), Mask, 0, Zeroable, Subtarget);
10245 Mask, Size, Zeroable, Subtarget);
10264 uint64_t &BitIdx, const APInt &Zeroable) {
10268 assert(!Zeroable.isAllOnesValue() && "Fully zeroable shuffle mask");
10275 // lower half that isn't zeroable.
10278 if (!Zeroable[Len - 1])
10280 assert(Len > 0 && "Zeroable shuffle mask");
10386 const APInt &Zeroable,
10389 if (matchVectorShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
10572 const APInt &Zeroable, const X86Subtarget &Subtarget,
10594 // Each of the extended elements need to be zeroable.
10595 if (!Zeroable[i])
10667 if (!Zeroable[i])
10728 const APInt &Zeroable, const X86Subtarget &Subtarget,
10738 if (i != V2Index && !Zeroable[i]) {
10775 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!");
11091 // elements are zeroable.
11094 const APInt &Zeroable,
11112 // Synthesize a zero mask from the zeroable elements (includes undefs).
11113 if (Zeroable[i]) {
11124 // We can only insert a single non-zeroable element.
11137 // Don't bother if we have no (non-zeroable) element for insertion.
11183 const APInt &Zeroable,
11190 if (!matchVectorShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
11318 const APInt &Zeroable,
11357 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
11364 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
11380 Zeroable, Subtarget, DAG))
11400 const APInt &Zeroable,
11433 Zeroable, Subtarget, DAG))
11439 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
11445 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
11453 Zeroable, Subtarget, DAG))
11605 const APInt &Zeroable,
11658 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
11663 Zeroable, Subtarget, DAG))
11668 lowerVectorShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
11700 const APInt &Zeroable,
11712 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
11741 Zeroable, Subtarget, DAG))
11747 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
11755 Zeroable, Subtarget, DAG))
11759 Zeroable, DAG))
12324 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse,
12343 if (Zeroable[i / Scale])
12385 const APInt &Zeroable,
12396 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
12409 Zeroable, Subtarget, DAG))
12440 Zeroable, Subtarget, DAG))
12446 Zeroable, DAG))
12452 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
12460 Zeroable, Subtarget, DAG))
12464 Zeroable, DAG))
12496 Zeroable, DAG, V1InUse, V2InUse);
12592 const APInt &Zeroable,
12602 Zeroable, Subtarget, DAG))
12617 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
12623 Zeroable, DAG))
12726 Zeroable, DAG))
12752 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
12760 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
12786 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
12890 const APInt &Zeroable,
12895 return lowerV2I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
12897 return lowerV2F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
12899 return lowerV4I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
12901 return lowerV4F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
12903 return lowerV8I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
12905 return lowerV16I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
13154 const APInt &Zeroable,
13162 if (!canWidenShuffleElements(Mask, Zeroable, WidenedMask))
13165 bool IsLowZero = (Zeroable & 0x3) == 0x3;
13166 bool IsHighZero = (Zeroable & 0xc) == 0xc;
13184 Zeroable, Subtarget, DAG))
13692 const APInt &Zeroable,
13701 Zeroable, Subtarget, DAG))
13745 Zeroable, Subtarget, DAG))
13770 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask,
13789 const APInt &Zeroable,
13799 Zeroable, Subtarget, DAG))
13803 Zeroable, Subtarget, DAG))
13833 Zeroable, Subtarget, DAG))
13842 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask,
13883 const APInt &Zeroable,
13892 Zeroable, Subtarget, DAG))
13955 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask,
13982 const APInt &Zeroable,
13995 DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
14008 Zeroable, Subtarget, DAG))
14036 Zeroable, Subtarget, DAG))
14045 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask,
14095 const APInt &Zeroable,
14108 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14117 Zeroable, Subtarget, DAG))
14132 Zeroable, Subtarget, DAG))
14164 DL, MVT::v16i16, Mask, V1, V2, Zeroable, Subtarget, DAG))
14186 const APInt &Zeroable,
14199 DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14208 Zeroable, Subtarget, DAG))
14223 Zeroable, Subtarget, DAG))
14244 DL, MVT::v32i8, Mask, V1, V2, Zeroable, Subtarget, DAG))
14268 const APInt &Zeroable,
14278 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
14298 lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG))
14314 return lowerV4F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14316 return lowerV4I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14318 return lowerV8F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14320 return lowerV8I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14322 return lowerV16I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14324 return lowerV32I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14334 const APInt &Zeroable,
14345 // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
14351 if (WidenedMask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
14352 (WidenedMask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
14353 unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
14435 const APInt &Zeroable,
14466 lowerV4X128VectorShuffle(DL, MVT::v8f64, Mask, Zeroable, V1, V2,
14479 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1,
14484 Zeroable, Subtarget, DAG))
14492 const APInt &Zeroable,
14522 Zeroable, Subtarget, DAG))
14538 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
14547 const APInt &Zeroable,
14577 lowerV4X128VectorShuffle(DL, MVT::v8i64, Mask, Zeroable,
14583 Zeroable, Subtarget, DAG))
14600 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1,
14605 Zeroable, Subtarget, DAG))
14613 const APInt &Zeroable,
14625 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
14648 Zeroable, Subtarget, DAG))
14672 if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask,
14677 Zeroable, Subtarget, DAG))
14684 const APInt &Zeroable,
14697 DL, MVT::v32i16, V1, V2, Mask, Zeroable
14707 Zeroable, Subtarget, DAG))
14727 Zeroable, Subtarget, DAG))
14731 DL, MVT::v32i16, Mask, V1, V2, Zeroable, Subtarget, DAG))
14739 const APInt &Zeroable,
14752 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14762 Zeroable, Subtarget, DAG))
14771 DL, MVT::v64i8, Mask, V1, V2, Zeroable, Subtarget, DAG))
14785 Zeroable, Subtarget, DAG))
14799 const APInt &Zeroable,
14812 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
14831 return lowerV8F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14833 return lowerV16F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14835 return lowerV8I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14837 return lowerV16I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14839 return lowerV32I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14841 return lowerV64I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14854 const APInt &Zeroable,
14872 // Make sure the number of zeroable bits in the top at least covers the bits
14874 if (Zeroable.countLeadingOnes() >= (NumElts - SubvecElts)) {
15049 APInt Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
15050 if (Zeroable.isAllOnesValue())
15055 // Create an alternative mask with info about zeroable elements.
15056 // Here we do not set undef elements as zeroable.
15059 assert(!Zeroable.isNullValue() && "V2's non-undef elements are used?!");
15061 if (Mask[i] != SM_SentinelUndef && Zeroable[i])
15113 return lower128BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
15117 return lower256BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
15121 return lower512BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
15125 return lower1BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
29274 const APInt &Zeroable,
29382 0, Zeroable, Subtarget);
29462 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
29544 if (Zeroable.getBoolValue() &&
29545 matchVectorShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
29572 // referencing one of the vectors, is zeroable or entirely undef.
29727 // Determine zeroable mask elements.
29728 APInt Zeroable(NumMaskElts, 0);
29731 Zeroable.setBit(i);
29759 if (matchUnaryPermuteVectorShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
29789 MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, NewV1,
29809 Zeroable)) {
