Lines Matching refs:V2
2950 SDValue V2) {
2956 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
3853 SDValue V1, SDValue V2, SelectionDAG &DAG) {
3865 return DAG.getNode(Opc, dl, VT, V1, V2);
4661 static SDValue concat128BitVectors(SDValue V1, SDValue V2, EVT VT,
4665 return insert128BitVector(V, V2, NumElems / 2, DAG, dl);
4668 static SDValue concat256BitVectors(SDValue V1, SDValue V2, EVT VT,
4672 return insert256BitVector(V, V2, NumElems / 2, DAG, dl);
4698 SDValue V1, SDValue V2) {
4706 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
4711 SDValue V1, SDValue V2) {
4719 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
4723 /// This produces a shuffle where the low element of V2 is swizzled into the
4726 static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
4730 MVT VT = V2.getSimpleValueType();
4732 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
4736 // If this is the insertion idx, put the low elt of V2 here.
4738 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
5099 SDValue V2 = IsUnary ? V1 : Ops[1];
5102 V2 = peekThroughBitcasts(V2);
5112 SDValue V = M < Size ? V1 : V2;
5432 SDValue V2 = Elt.getOperand(0);
5455 if (V2.getSimpleValueType() != MVT::v4f32)
5456 V2 = DAG.getBitcast(MVT::v4f32, V2);
5464 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
6778 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
6780 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
6908 SDValue V2 = Op.getOperand(1);
6911 return concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
6919 concat128BitVectors(V1, V2, HalfVT, NumElems / 2, DAG, dl),
6923 return concat256BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
6971 SDValue V2 = Op.getOperand(1);
6973 assert(V1.getValueType() == V2.getValueType() &&
6981 bool IsZeroV2 = ISD::isBuildVectorAllZeros(V2.getNode());
6987 if (V2.isUndef())
6994 V2 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Undef, V2, IdxVal);
6997 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, ZeroVec, V2, IdxVal);
7000 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, V1, V2, IdxVal);
7146 static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
7156 auto *BV2 = dyn_cast<BuildVectorSDNode>(V2);
7235 SDValue V1, SDValue V2) {
7238 V2 = peekThroughBitcasts(V2);
7241 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
7256 SDValue V = M < Size ? V1 : V2;
7302 /// This is only possible if V2 is unused (at all, or only for zero elements).
7305 SDValue V2,
7317 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7356 SDValue V2, SelectionDAG &DAG) {
7370 if (isShuffleEquivalent(V1, V2, Mask, Unpckl))
7371 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
7372 if (isShuffleEquivalent(V1, V2, Mask, Unpckh))
7373 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
7377 if (isShuffleEquivalent(V1, V2, Mask, Unpckl))
7378 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
7381 if (isShuffleEquivalent(V1, V2, Mask, Unpckh))
7382 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
7392 SDValue V2, ArrayRef<int> Mask,
7405 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7413 V = Mask[i] < Size ? V1 : V2;
7414 else if (V != (Mask[i] < Size ? V1 : V2))
7435 SDValue V2, ArrayRef<int> Mask,
7452 // We have to cast V2 around.
7454 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::ANDNP, DL, MaskVT,
7456 DAG.getBitcast(MaskVT, V2)));
7457 return DAG.getNode(ISD::OR, DL, VT, V1, V2);
7467 SDValue V2, ArrayRef<int> Original,
7471 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
7473 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7510 V2 = getZeroVector(VT, Subtarget, DAG, DL);
7526 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
7543 V2 = DAG.getBitcast(BlendVT, V2);
7545 VT, DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
7555 V2 = DAG.getBitcast(MVT::v8i16, V2);
7557 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
7571 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
7582 if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, DAG))
7612 V2 = DAG.getBitcast(BlendVT, V2);
7615 DAG.getBuildVector(BlendVT, DL, VSELECTMask), V1, V2));
7629 SDValue V1, SDValue V2,
7651 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
7664 SDValue V2,
7667 // Shuffle the input elements into the desired positions in V1 and V2 and
7688 lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))
7692 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
7693 return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
7708 /// Essentially it concatenates V1 and V2, shifts right by some number of
7713 SDValue V1, SDValue V2,
7761 SDValue MaskV = Mask[l + i] < NumElts ? V1 : V2;
7849 SDValue V2, ArrayRef<int> Mask,
7852 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7910 for (SDValue V : {V1, V2})
7920 SDValue V2, ArrayRef<int> Mask,
7922 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
7951 SDValue &V = (M < Size ? V1 : V2);
7994 Base = V2;
8009 Insert = V2;
8020 } else if ((!Base || (Base == V2)) &&
8023 Base = V2;
8222 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
8224 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8257 SDValue V = M < NumElements ? V1 : V2;
8324 return V2;
8379 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
8381 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8400 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
8415 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
8444 ExtVT, V1, V2);
8451 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
8453 V2 = DAG.getBitcast(VT, V2);
8463 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
8465 V2 = DAG.getBitcast(MVT::v16i8, V2);
8466 V2 = DAG.getNode(
8467 X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
8471 V2 = DAG.getBitcast(VT, V2);
8474 return V2;
8537 SDValue V1, SDValue V2,
8556 if (isShuffleEquivalent(V1, V2, Mask, BroadcastMask)) {
8697 // perform INSERTPS if a single V1 element is out of place and all V2
8699 static bool matchVectorShuffleAsInsertPS(SDValue &V1, SDValue &V2,
8705 assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
8733 // V2 input for insertion.
8746 // If we have a V1 input out of place, we use V1 as the V2 element insertion
8747 // and don't use the original V2 at all.
8750 V2 = V1;
8756 // the zero mask and the V2 insertion - so remove V1 dependency.
8760 // Insert the V2 element into the desired position.
8767 SDValue V2, ArrayRef<int> Mask,
8770 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
8771 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
8775 if (!matchVectorShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
8778 // Insert the V2 element into the desired position.
8779 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
8793 SDValue V1, SDValue V2,
8800 assert(!V2.isUndef() &&
8844 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
8848 V2 = DAG.getBitcast(UnpackVT, V2);
8853 UnpackVT, V1, V2));
8892 DL, VT, V1, V2),
8907 SDValue V1, SDValue V2,
8911 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
8912 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
8914 if (V2.isUndef()) {
8917 DL, MVT::v2f64, V1, V2, Mask, Subtarget, DAG))
8940 DL, MVT::v2f64, V1, V2, Mask, Subtarget, DAG))
8942 // Try inverting the insertion since for v2 masks it is easy to do and we
8947 DL, MVT::v2f64, V2, V1, InverseMask, Subtarget, DAG))
8953 if (isShuffleEquivalent(V1, V2, Mask, {0, 3}) ||
8954 isShuffleEquivalent(V1, V2, Mask, {1, 3}))
8960 DL, MVT::v2f64, V2,
8964 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
8970 lowerVectorShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
8974 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
8985 SDValue V1, SDValue V2,
8989 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
8990 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
8992 if (V2.isUndef()) {
8995 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
9010 assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!");
9011 assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!");
9013 assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
9024 if (SDValue V2Pack = GetPackNode(V2)) {
9036 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
9043 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
9045 // Try inverting the insertion since for v2 masks it is easy to do and we
9049 DL, MVT::v2i64, V2, V1, InverseMask, Subtarget, DAG))
9056 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
9062 lowerVectorShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
9069 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
9075 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2,
9083 V2 = DAG.getBitcast(MVT::v2f64, V2);
9085 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
9117 SDValue V2, SelectionDAG &DAG) {
9118 SDValue LowV = V1, HighV = V2;
9133 // Handles all the cases where we have a single V2 element and an undef.
9140 // Handle the case where the V2 element ends up adjacent to a V1 element.
9144 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
9150 LowV = V2;
9153 HighV = V2;
9155 NewMask[V1Index] = 2; // We put the V1 element in V2[2].
9156 NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
9160 // Handle the easy case where we have V1 in the low lanes and V2 in the
9171 LowV = V2;
9173 // We have a mixture of V1 and V2 in both low and high lanes. Rather than
9178 // V2.
9183 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
9205 SDValue V1, SDValue V2,
9209 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
9217 DL, MVT::v4f32, V1, V2, Mask, Subtarget, DAG))
9222 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2}))
9224 if (isShuffleEquivalent(V1, V2, Mask, {1, 1, 3, 3}))
9244 // when the V2 input is targeting element 0 of the mask -- that is the fast
9247 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v4f32, V1, V2,
9252 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
9257 if (SDValue V = lowerVectorShuffleAsInsertPS(DL, V1, V2, Mask, DAG))
9262 DL, MVT::v4f32, V1, V2, Mask, DAG))
9267 if (isShuffleEquivalent(V1, V2, Mask, {0, 1, 4, 5}))
9268 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
9269 if (isShuffleEquivalent(V1, V2, Mask, {2, 3, 6, 7}))
9270 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
9274 lowerVectorShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
9278 return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
9286 SDValue V1, SDValue V2,
9290 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
9296 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2,
9305 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG))
9315 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 1, 1}))
9317 else if (isShuffleEquivalent(V1, V2, Mask, {2, 2, 3, 3}))
9325 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
9331 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v4i32, V1, V2,
9339 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
9344 lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG))
9349 lowerVectorShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
9356 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG))
9362 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2,
9367 V2, Mask, DAG))
9378 DAG.getBitcast(MVT::v4f32, V2), Mask));
9852 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
9854 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
9886 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
9887 DAG.getBitcast(MVT::v16i8, V2),
9893 V = DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2);
9895 V = V1InUse ? V1 : V2;
9914 SDValue V1, SDValue V2,
9918 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
9924 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
9932 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
9942 lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
9962 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
9968 if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask, DAG))
9973 V2,
9981 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
9986 lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG))
9991 lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
9996 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
10000 lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
10004 V2, Mask, DAG))
10011 return lowerVectorShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask, DAG,
10017 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
10095 SDValue V1, SDValue V2,
10099 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
10103 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
10109 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
10114 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
10119 if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, DAG))
10128 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
10223 lowerVectorShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask, DAG))
10228 lowerVectorShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
10249 DL, MVT::v16i8, V1, V2, Mask, DAG, V1InUse, V2InUse);
10251 // If both V1 and V2 are in use and we can use a direct blend or an unpack,
10256 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2,
10269 DL, MVT::v16i8, V1, V2, Mask, DAG))
10278 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v16i8, V1, V2,
10283 lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
10293 bool IsSingleInput = V2.isUndef();
10309 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask);
10313 V2 = IsSingleInput ? V1 : DAG.getBitcast(MVT::v8i16, V2);
10314 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
10325 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2,
10384 MVT VT, SDValue V1, SDValue V2,
10389 return lowerV2I64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
10391 return lowerV2F64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
10393 return lowerV4I32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
10395 return lowerV4F32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
10397 return lowerV8I16VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
10399 return lowerV16I8VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
10468 SDValue V2, ArrayRef<int> Mask,
10473 assert(V2.getSimpleValueType() == VT && "Bad operand type!");
10518 std::tie(LoV2, HiV2) = SplitVector(V2);
10549 // First try just blending the halves of V1 or V2.
10572 // We only use half of V2 so map the usage down into the final blend mask.
10594 SDValue V1, SDValue V2,
10597 assert(!V2.isUndef() && "This routine must not be used to lower single-input "
10621 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
10636 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
10640 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, DAG);
10653 SDValue V1, SDValue V2,
10669 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
10671 assert(V2.isUndef() &&
10695 SDValue V2, ArrayRef<int> Mask,
10703 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask,
10708 bool IsV2Zero = ISD::isBuildVectorAllZeros(V2.getNode());
10715 bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask, {0, 1, 0, 1});
10716 if (OnlyUsesV1 || isShuffleEquivalent(V1, V2, Mask, {0, 1, 4, 5})) {
10718 if (Subtarget.hasAVX2() && V2.isUndef())
10726 OnlyUsesV1 ? V1 : V2,
10757 // Shuffle mask values >= 4 are selecting elements of V2.
10768 V2 = DAG.getUNDEF(VT);
10775 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
10792 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
10794 assert(!V2.isUndef() && "This is only useful with multiple inputs.");
10840 V2 = DAG.getBitcast(LaneVT, V2);
10841 SDValue LaneShuffle = DAG.getVectorShuffle(LaneVT, DL, V1, V2, LaneMask);
10861 SDValue V1, SDValue V2,
10910 // i.e. 0 = Lower V1, 1 = Upper V1, 2 = Lower V2, 3 = Upper V2.
10964 SDValue V = (HalfIdx < 2 ? V1 : V2);
10997 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11035 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
11105 SDValue RepeatedShuffle = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
11123 SDValue V2, SelectionDAG &DAG) {
11144 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
11147 return DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
11157 SDValue V1, SDValue V2,
11161 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
11166 if (SDValue V = lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask,
11170 if (V2.isUndef()) {
11173 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
11177 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2}))
11197 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
11201 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask,
11207 lowerVectorShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
11210 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
11216 lowerVectorShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG))
11222 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
11232 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
11238 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2,
11242 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG);
11250 SDValue V1, SDValue V2,
11254 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
11260 if (SDValue V = lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask,
11264 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
11269 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i64, V1, V2,
11273 if (V2.isUndef()) {
11294 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
11300 lowerVectorShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
11310 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
11314 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2,
11323 SDValue V1, SDValue V2,
11327 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
11330 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
11335 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8f32, V1, V2,
11347 if (isShuffleEquivalent(V1, V2, RepeatedMask, {0, 0, 2, 2}))
11349 if (isShuffleEquivalent(V1, V2, RepeatedMask, {1, 1, 3, 3}))
11352 if (V2.isUndef())
11358 lowerVectorShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
11363 return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
11369 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
11374 if (V2.isUndef()) {
11388 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask,
11395 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
11401 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2,
11405 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG);
11413 SDValue V1, SDValue V2,
11417 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
11424 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2,
11428 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
11433 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i32, V1, V2,
11443 if (V2.isUndef())
11449 lowerVectorShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
11454 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
11460 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11466 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11471 if (V2.isUndef()) {
11483 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
11487 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2,
11496 SDValue V1, SDValue V2,
11500 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
11507 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2,
11512 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i16, V1, V2,
11516 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
11522 lowerVectorShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
11526 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
11532 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11538 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11541 if (V2.isUndef()) {
11545 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2,
11559 V2, Subtarget, DAG))
11565 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
11569 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG);
11577 SDValue V1, SDValue V2,
11581 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
11588 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2,
11593 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v32i8, V1, V2,
11597 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
11603 lowerVectorShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
11607 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
11613 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11619 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11624 if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask))
11625 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, Mask,
11629 V2, Subtarget, DAG))
11635 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
11639 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG);
11648 MVT VT, SDValue V1, SDValue V2,
11658 DL, VT, V1, V2, Mask, Subtarget, DAG))
11663 lowerVectorShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
11677 if (SDValue V = lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, DAG))
11679 if (SDValue V = lowerVectorShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
11681 return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
11687 V2 = DAG.getBitcast(FpVT, V2);
11688 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
11693 return lowerV4F64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
11695 return lowerV4I64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
11697 return lowerV8F32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
11699 return lowerV8I32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
11701 return lowerV16I16VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
11703 return lowerV32I8VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
11713 SDValue V2, SelectionDAG &DAG) {
11734 SDValue Op = WidenedMask[i] > MaxOp1Index ? V2 : V1;
11760 SDValue V2, SelectionDAG &DAG) {
11768 if (V2.isUndef())
11771 return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2);
11776 SDValue V1, SDValue V2,
11780 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
11783 if (V2.isUndef()) {
11785 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2, 4, 4, 6, 6}))
11806 lowerV4X128VectorShuffle(DL, MVT::v8f64, Mask, V1, V2, DAG))
11810 lowerVectorShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
11815 lowerVectorShuffleWithSHUFPD(DL, MVT::v8f64, Mask, V1, V2, DAG))
11818 return lowerVectorShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG);
11823 SDValue V1, SDValue V2,
11827 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
11837 if (isShuffleEquivalent(V1, V2, RepeatedMask, {0, 0, 2, 2}))
11839 if (isShuffleEquivalent(V1, V2, RepeatedMask, {1, 1, 3, 3}))
11842 if (V2.isUndef())
11848 lowerVectorShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
11852 return lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
11855 return lowerVectorShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG);
11860 SDValue V1, SDValue V2,
11864 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
11868 lowerV4X128VectorShuffle(DL, MVT::v8i64, Mask, V1, V2, DAG))
11871 if (V2.isUndef()) {
11893 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
11898 lowerVectorShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
11901 return lowerVectorShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG);
11906 SDValue V1, SDValue V2,
11910 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
11919 if (V2.isUndef())
11925 lowerVectorShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
11930 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
11937 DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
11940 return lowerVectorShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
11945 SDValue V1, SDValue V2,
11949 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
11955 lowerVectorShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
11959 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
11965 DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
11968 if (V2.isUndef()) {
11979 return lowerVectorShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG);
11984 SDValue V1, SDValue V2,
11988 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
11994 lowerVectorShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
11998 if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
12004 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
12008 V2, Subtarget, DAG))
12012 return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
12021 MVT VT, SDValue V1, SDValue V2,
12029 lowerVectorShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))
12038 return lowerV8F64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
12040 return lowerV16F32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
12042 return lowerV8I64VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
12044 return lowerV16I32VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
12046 return lowerV32I16VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
12048 return lowerV64I8VectorShuffle(DL, Mask, V1, V2, Subtarget, DAG);
12060 MVT VT, SDValue V1, SDValue V2,
12096 if (V2.isUndef())
12097 V2 = DAG.getUNDEF(ExtVT);
12098 else if (ISD::isBuildVectorAllZeros(V2.getNode()))
12099 V2 = getZeroVector(ExtVT, Subtarget, DAG, DL);
12100 else if (ISD::isBuildVectorAllOnes(V2.getNode()))
12101 V2 = getOnesVector(ExtVT, Subtarget, DAG, DL);
12103 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
12105 DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask));
12119 SDValue V2 = Op.getOperand(1);
12129 bool V2IsUndef = V2.isUndef();
12149 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
12155 SmallBitVector Zeroable = computeZeroableShuffleElements(Mask, V1, V2);
12174 V2 = DAG.getBitcast(NewVT, V2);
12176 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
12190 // V2. This allows us to match the shuffle pattern strictly on how many
12196 assert((NumV2Elements > 0 || V2IsUndef) && "V2 not undef, but not used");
12198 // When the number of V1 and V2 elements are the same, try to minimize the
12199 // number of uses of V2 in the low half of the vector. When that is tied,
12201 // indices for V2. When those are equal, try to ensure that the number of odd
12202 // indices for V1 is lower than the number of odd indices for V2.
12236 return lower128BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG);
12239 return lower256BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG);
12242 return lower512BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG);
12245 return lower1BitVectorShuffle(DL, Mask, VT, V1, V2, Subtarget, DAG);
24678 SDValue V2 = SVOp->getOperand(1);
24683 V2.getOpcode() == ISD::CONCAT_VECTORS) {
24694 if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR ||
24695 !V2.getOperand(1).isUndef() || !V1.getOperand(1).isUndef())
24698 if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()))
25203 SDValue V1 = Input, V2 = Input;
25204 if (Zeroable.any() && matchVectorShuffleAsInsertPS(V1, V2, InsertPSMask,
25210 V2 = DAG.getBitcast(MVT::v4f32, V2);
25211 DCI.AddToWorklist(V2.getNode());
25212 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
26008 SDValue V2 = N->getOperand(1);
26012 if (V1.getOpcode() == ISD::FADD && V2.getOpcode() == ISD::FSUB) {
26014 std::swap(V1, V2);
26015 } else if (V1.getOpcode() != ISD::FSUB || V2.getOpcode() != ISD::FADD)
26019 if (!V1->hasOneUse() || !V2->hasOneUse())
26025 if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
26026 (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
26031 if (!(isShuffleEquivalent(V1, V2, Mask, {0, 3}) ||
26032 isShuffleEquivalent(V1, V2, Mask, {0, 5, 2, 7}) ||
26033 isShuffleEquivalent(V1, V2, Mask, {0, 9, 2, 11, 4, 13, 6, 15})))
27984 SDValue V2 = N01->getOperand(1);
27988 !ISD::isBuildVectorAllOnes(V2.getNode()))
