Lines Matching refs:VT
58 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
68 EVT VT = Vec.getValueType();
69 assert(VT.is256BitVector() && "Unexpected vector size!");
70 EVT ElVT = VT.getVectorElementType();
71 unsigned Factor = VT.getSizeInBits()/128;
73 VT.getVectorNumElements()/Factor);
112 EVT VT = Vec.getValueType();
113 assert(VT.is128BitVector() && "Unexpected vector size!");
115 EVT ElVT = VT.getVectorElementType();
135 static SDValue Concat128BitVectors(SDValue V1, SDValue V2, EVT VT,
138 SDValue V = Insert128BitVector(DAG.getUNDEF(VT), V1, 0, DAG, dl);
362 MVT VT = IntVTs[i];
363 setOperationAction(ISD::MULHS, VT, Expand);
364 setOperationAction(ISD::MULHU, VT, Expand);
365 setOperationAction(ISD::SDIV, VT, Expand);
366 setOperationAction(ISD::UDIV, VT, Expand);
367 setOperationAction(ISD::SREM, VT, Expand);
368 setOperationAction(ISD::UREM, VT, Expand);
371 setOperationAction(ISD::ADDC, VT, Custom);
372 setOperationAction(ISD::ADDE, VT, Custom);
373 setOperationAction(ISD::SUBC, VT, Custom);
374 setOperationAction(ISD::SUBE, VT, Custom);
523 MVT VT = IntVTs[i];
524 setOperationAction(ISD::ATOMIC_CMP_SWAP, VT, Custom);
525 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
526 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
751 MVT VT = (MVT::SimpleValueType)i;
752 setOperationAction(ISD::ADD , VT
753 setOperationAction(ISD::SUB , VT, Expand);
754 setOperationAction(ISD::FADD, VT, Expand);
755 setOperationAction(ISD::FNEG, VT, Expand);
756 setOperationAction(ISD::FSUB, VT, Expand);
757 setOperationAction(ISD::MUL , VT, Expand);
758 setOperationAction(ISD::FMUL, VT, Expand);
759 setOperationAction(ISD::SDIV, VT, Expand);
760 setOperationAction(ISD::UDIV, VT, Expand);
761 setOperationAction(ISD::FDIV, VT, Expand);
762 setOperationAction(ISD::SREM, VT, Expand);
763 setOperationAction(ISD::UREM, VT, Expand);
764 setOperationAction(ISD::LOAD, VT, Expand);
765 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
766 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
767 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
768 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
769 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
770 setOperationAction(ISD::FABS, VT, Expand);
771 setOperationAction(ISD::FSIN, VT, Expand);
772 setOperationAction(ISD::FSINCOS, VT, Expand);
773 setOperationAction(ISD::FCOS, VT, Expand);
774 setOperationAction(ISD::FSINCOS, VT, Expand);
775 setOperationAction(ISD::FREM, VT, Expand);
776 setOperationAction(ISD::FMA, VT, Expand);
777 setOperationAction(ISD::FPOWI, VT, Expand);
778 setOperationAction(ISD::FSQRT, VT, Expand);
779 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
780 setOperationAction(ISD::FFLOOR, VT, Expand);
781 setOperationAction(ISD::FCEIL, VT, Expand);
782 setOperationAction(ISD::FTRUNC, VT, Expand);
783 setOperationAction(ISD::FRINT, VT, Expand);
784 setOperationAction(ISD::FNEARBYINT, VT, Expand);
785 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
786 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
787 setOperationAction(ISD::SDIVREM, VT, Expand);
788 setOperationAction(ISD::UDIVREM, VT, Expand);
789 setOperationAction(ISD::FPOW, VT, Expand);
790 setOperationAction(ISD::CTPOP, VT, Expand);
791 setOperationAction(ISD::CTTZ, VT, Expand);
792 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
793 setOperationAction(ISD::CTLZ, VT, Expand);
794 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
795 setOperationAction(ISD::SHL, VT, Expand);
796 setOperationAction(ISD::SRA, VT, Expand);
797 setOperationAction(ISD::SRL, VT, Expand);
798 setOperationAction(ISD::ROTL, VT, Expand);
799 setOperationAction(ISD::ROTR, VT, Expand);
800 setOperationAction(ISD::BSWAP, VT, Expand);
801 setOperationAction(ISD::SETCC, VT, Expand);
802 setOperationAction(ISD::FLOG, VT, Expand);
803 setOperationAction(ISD::FLOG2, VT, Expand);
804 setOperationAction(ISD::FLOG10, VT, Expand);
805 setOperationAction(ISD::FEXP, VT, Expand);
806 setOperationAction(ISD::FEXP2, VT, Expand);
807 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
808 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
809 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
810 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
811 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
812 setOperationAction(ISD::TRUNCATE, VT, Expand);
813 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
814 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
815 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
816 setOperationAction(ISD::VSELECT, VT, Expand);
819 setTruncStoreAction(VT,
821 setLoadExtAction(ISD::SEXTLOAD, VT, Expand);
822 setLoadExtAction(ISD::ZEXTLOAD, VT, Expand);
823 setLoadExtAction(ISD::EXTLOAD, VT, Expand);
924 MVT VT = (MVT::SimpleValueType)i;
926 if (!isPowerOf2_32(VT.getVectorNumElements()))
929 if (!VT.is128BitVector())
931 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
932 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
933 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
950 MVT VT = (MVT::SimpleValueType)i;
953 if (!VT.is128BitVector())
956 setOperationAction(ISD::AND, VT, Promote);
957 AddPromotedToType (ISD::AND, VT, MVT::v2i64);
958 setOperationAction(ISD::OR, VT, Promote);
959 AddPromotedToType (ISD::OR, VT, MVT::v2i64);
960 setOperationAction(ISD::XOR, VT, Promote);
961 AddPromotedToType (ISD::XOR, VT, MVT::v2i64);
962 setOperationAction(ISD::LOAD, VT, Promote);
963 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64);
964 setOperationAction(ISD::SELECT, VT, Promote);
965 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64);
1226 MVT VT = (MVT::SimpleValueType)i;
1230 if (VT.is128BitVector())
1231 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1234 if (!VT.is256BitVector())
1237 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1238 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1239 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1240 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1241 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1242 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1243 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1248 MVT VT = (MVT::SimpleValueType)i;
1251 if (!VT.is256BitVector())
1254 setOperationAction(ISD::AND, VT, Promote);
1255 AddPromotedToType (ISD::AND, VT, MVT::v4i64);
1256 setOperationAction(ISD::OR, VT, Promote);
1257 AddPromotedToType (ISD::OR, VT, MVT::v4i64);
1258 setOperationAction(ISD::XOR, VT, Promote);
1259 AddPromotedToType (ISD::XOR, VT, MVT::v4i64);
1260 setOperationAction(ISD::LOAD, VT, Promote);
1261 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64);
1262 setOperationAction(ISD::SELECT, VT, Promote);
1263 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64);
1269 for (int VT = MVT::FIRST_VECTOR_VALUETYPE;
1270 VT != MVT::LAST_VECTOR_VALUETYPE; VT++) {
1271 setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT,
1287 MVT VT = IntVTs[i];
1288 setOperationAction(ISD::SADDO, VT, Custom);
1289 setOperationAction(ISD::UADDO, VT, Custom);
1290 setOperationAction(ISD::SSUBO, VT, Custom);
1291 setOperationAction(ISD::USUBO, VT, Custom);
1292 setOperationAction(ISD::SMULO, VT, Custom);
1293 setOperationAction(ISD::UMULO, VT, Custom);
1367 EVT X86TargetLowering::getSetCCResultType(EVT VT) const {
1368 if (!VT.isVector()) return MVT::i8;
1369 return VT.changeVectorElementTypeToInteger();
1464 bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
1465 if (VT == MVT::f32)
1467 else if (VT == MVT::f64)
1473 X86TargetLowering::allowsUnalignedMemoryAccesses(EVT VT, bool *Fast) const {
1532 X86TargetLowering::findRepresentativeClass(MVT VT) const{
1535 switch (VT.SimpleTy) {
1537 return TargetLowering::findRepresentativeClass(VT);
1744 X86TargetLowering::getTypeForExtArgOrReturn(MVT VT,
1748 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND)
1754 return VT.bitsLT(MinVT) ? MinVT : VT;
2248 EVT VT = getPointerTy();
2252 OutRetAddr = DAG.getLoad(VT
3063 static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
3070 return DAG.getNode(Opc, dl, VT, V1);
3074 static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
3084 return DAG.getNode(Opc, dl, VT, V1, DAG.getConstant(TargetMask, MVT::i8));
3088 static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
3096 return DAG.getNode(Opc, dl, VT, V1, V2,
3101 static SDValue getTargetShuffleNode(unsigned Opc, DebugLoc dl, EVT VT,
3114 return DAG.getNode(Opc, dl, VT, V1, V2);
3299 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
3333 static bool isPSHUFDMask(ArrayRef<int> Mask, EVT VT) {
3334 if (VT == MVT::v4f32 || VT == MVT::v4i32 )
3336 if (VT == MVT::v2f64 || VT == MVT::v2i64)
3343 static bool isPSHUFHWMask(ArrayRef<int> Mask, EVT VT, bool HasInt256) {
3344 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
3356 if (VT == MVT::v16i16) {
3372 static bool isPSHUFLWMask(ArrayRef<int> Mask, EVT VT, bool HasInt256) {
3373 if (VT != MVT::v8i16 && (!HasInt256 || VT != MVT::v16i16))
3385 if (VT == MVT::v16i16) {
3401 static bool isPALIGNRMask(ArrayRef<int> Mask, EVT VT,
3403 if ((VT.is128BitVector() && !Subtarget->hasSSSE3()) ||
3404 (VT.is256BitVector() && !Subtarget->hasInt256()))
3407 unsigned NumElts = VT.getVectorNumElements();
3408 unsigned NumLanes = VT.getSizeInBits()/128;
3491 static bool isSHUFPMask(ArrayRef<int> Mask, EVT VT, bool HasFp256,
3493 if (!HasFp256 && VT.is256BitVector())
3496 unsigned NumElems = VT.getVectorNumElements();
3497 unsigned NumLanes = VT.getSizeInBits()/128;
3544 static bool isMOVHLPSMask(ArrayRef<int> Mask, EVT VT) {
3545 if (!VT.is128BitVector())
3548 unsigned NumElems = VT.getVectorNumElements();
3563 static bool isMOVHLPS_v_undef_Mask(ArrayRef<int> Mask, EVT VT) {
3564 if (!VT.is128BitVector())
3567 unsigned NumElems = VT.getVectorNumElements();
3580 static bool isMOVLPMask(ArrayRef<int> Mask, EVT VT) {
3581 if (!VT.is128BitVector())
3584 unsigned NumElems = VT.getVectorNumElements();
3602 static bool isMOVLHPSMask(ArrayRef<int> Mask, EVT VT) {
3603 if (!VT.is128BitVector())
3606 unsigned NumElems = VT.getVectorNumElements();
3628 MVT VT = SVOp->getValueType(0).getSimpleVT();
3631 if (VT != MVT::v8i32 && VT != MVT::v8f32)
3651 SDValue UndefNode = DAG.getNode(ISD::UNDEF, dl, VT);
3659 Op1 = DAG.getVectorShuffle(VT, dl, Op1, UndefNode, ShiftRightMask);
3663 Op0 = DAG.getVectorShuffle(VT, dl, Op0, UndefNode, ShiftLeftMask);
3666 return DAG.getVectorShuffle(VT, dl, Op0, Op1, BlendMask);
3671 static bool isUNPCKLMask(ArrayRef<int> Mask, EVT VT,
3673 unsigned NumElts = VT.getVectorNumElements();
3675 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3678 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
3684 unsigned NumLanes = VT.getSizeInBits()/128;
3710 static bool isUNPCKHMask(ArrayRef<int> Mask, EVT VT,
3712 unsigned NumElts = VT.getVectorNumElements();
3714 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3717 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
3723 unsigned NumLanes = VT.getSizeInBits()/128;
3748 static bool isUNPCKL_v_undef_Mask(ArrayRef<int> Mask, EVT VT, bool HasInt256) {
3749 unsigned NumElts = VT.getVectorNumElements();
3750 bool Is256BitVec = VT.is256BitVector();
3752 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3768 unsigned NumLanes = VT.getSizeInBits()/128;
3791 static bool isUNPCKH_v_undef_Mask(ArrayRef<int> Mask, EVT VT, bool HasInt256) {
3792 unsigned NumElts = VT.getVectorNumElements();
3794 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3797 if (VT.is256BitVector() && NumElts != 4 && NumElts != 8 &&
3803 unsigned NumLanes = VT.getSizeInBits()/128;
3823 static bool isMOVLMask(ArrayRef<int> Mask, EVT VT) {
3824 if (VT.getVectorElementType().getSizeInBits() < 32)
3826 if (!VT.is128BitVector())
3829 unsigned NumElts = VT.getVectorNumElements();
3847 static bool isVPERM2X128Mask(ArrayRef<int> Mask, EVT VT, bool HasFp256) {
3848 if (!HasFp256 || !VT.is256BitVector())
3854 unsigned HalfSize = VT.getVectorNumElements()/2;
3879 MVT VT = SVOp->getValueType(0).getSimpleVT();
3881 unsigned HalfSize = VT.getVectorNumElements()/2;
3907 static bool isVPERMILPMask(ArrayRef<int> Mask, EVT VT, bool HasFp256) {
3911 unsigned NumElts = VT.getVectorNumElements();
3913 if (!VT.is256BitVector() || (NumElts != 4 && NumElts != 8))
3916 unsigned NumLanes = VT.getSizeInBits()/128;
3938 static bool isCommutedMOVLMask(ArrayRef<int> Mask, EVT VT,
3940 if (!VT.is128BitVector())
3943 unsigned NumOps = VT.getVectorNumElements();
3962 static bool isMOVSHDUPMask(ArrayRef<int> Mask, EVT VT,
3967 unsigned NumElems = VT.getVectorNumElements();
3969 if ((VT.is128BitVector() && NumElems != 4) ||
3970 (VT.is256BitVector() && NumElems != 8))
3985 static bool isMOVSLDUPMask(ArrayRef<int> Mask, EVT VT,
3990 unsigned NumElems = VT.getVectorNumElements();
3992 if ((VT.is128BitVector() && NumElems != 4) ||
3993 (VT.is256BitVector() && NumElems != 8))
4008 static bool isMOVDDUPYMask(ArrayRef<int> Mask, EVT VT, bool HasFp256) {
4009 if (!HasFp256 || !VT.is256BitVector())
4012 unsigned NumElts = VT.getVectorNumElements();
4028 static bool isMOVDDUPMask(ArrayRef<int> Mask, EVT VT) {
4029 if (!VT.is128BitVector())
4032 unsigned e = VT.getVectorNumElements() / 2;
4053 MVT VT = N->getValueType(0).getSimpleVT();
4054 unsigned ElSize = VT.getVectorElementType().getSizeInBits();
4071 MVT VT = N->getValueType(0).getSimpleVT();
4072 unsigned ElSize = VT.getVectorElementType().getSizeInBits();
4082 MVT VT = N->getValueType(0).getSimpleVT();
4084 assert((VT.is128BitVector() || VT.is256BitVector()) &&
4089 unsigned NumElts = VT.getVectorNumElements();
4090 unsigned NumLanes = VT.getSizeInBits()/128;
4112 MVT VT = N->getValueType(0).getSimpleVT();
4114 assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
4117 unsigned NumElts = VT.getVectorNumElements();
4136 MVT VT = N->getValueType(0).getSimpleVT();
4138 assert((VT == MVT::v8i16 || VT == MVT::v16i16) &&
4141 unsigned NumElts = VT.getVectorNumElements();
4160 MVT VT = SVOp->getValueType(0).getSimpleVT();
4161 unsigned EltSize = VT.getVectorElementType().getSizeInBits() >> 3;
4163 unsigned NumElts = VT.getVectorNumElements();
4164 unsigned NumLanes = VT.getSizeInBits()/128;
4219 MVT VT = N->getValueType(0).getSimpleVT();
4221 unsigned NumElts = VT.getVectorNumElements();
4223 assert((VT.is256BitVector() && NumElts == 4) &&
4250 MVT VT = SVOp->getValueType(0).getSimpleVT();
4251 unsigned NumElems = VT.getVectorNumElements();
4264 return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1),
4272 static bool ShouldXformToMOVHLPS(ArrayRef<int> Mask, EVT VT) {
4273 if (!VT.is128BitVector())
4275 if (VT.getVectorNumElements() != 4)
4329 ArrayRef<int> Mask, EVT VT) {
4330 if (!VT.is128BitVector())
4340 unsigned NumElems = VT.getVectorNumElements();
4396 static SDValue getZeroVector(EVT VT, const X86Subtarget *Subtarget,
4398 assert(VT.isVector() && "Expected a vector type");
4403 if (VT.is128BitVector()) { // SSE
4411 } else if (VT.is256BitVector()) { // AVX
4426 return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
4433 static SDValue getOnesVector(MVT VT, bool HasInt256, SelectionDAG &DAG,
4435 assert(VT.isVector() && "Expected a vector type");
4439 if (VT.is256BitVector()) {
4447 } else if (VT.is128BitVector()) {
4452 return DAG.getNode(ISD::BITCAST, dl, VT, Vec);
4467 static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
4469 unsigned NumElems = VT.getVectorNumElements();
4474 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
4478 static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
4480 unsigned NumElems = VT.getVectorNumElements();
4486 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
4490 static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
4492 unsigned NumElems = VT.getVectorNumElements();
4498 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
4506 EVT VT = V.getValueType();
4507 int NumElems = VT.getVectorNumElements();
4512 V = getUnpackl(DAG, dl, VT, V, V);
4514 V = getUnpackh(DAG, dl, VT, V, V);
4524 EVT VT = V.getValueType();
4527 if (VT.is128BitVector()) {
4532 } else if (VT.is256BitVector()) {
4545 return DAG.getNode(ISD::BITCAST, dl, VT, V);
4595 EVT VT = V2.getValueType();
4597 ? getZeroVector(VT, Subtarget, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT);
4598 unsigned NumElems = VT.getVectorNumElements();
4603 return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]);
4609 static bool getTargetShuffleMask(SDNode *N, MVT VT,
4611 unsigned NumElems = VT.getVectorNumElements();
4618 DecodeSHUFPMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
4621 DecodeUNPCKHMask(VT, Mask);
4624 DecodeUNPCKLMask(VT, Mask);
4634 DecodePALIGNRMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
4639 DecodePSHUFMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
4644 DecodePSHUFHWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
4649 DecodePSHUFLWMask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
4670 DecodeVPERM2X128Mask(VT, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
4695 EVT VT = V.getValueType();
4703 return DAG.getUNDEF(VT.getVectorElementType());
4705 unsigned NumElems = VT.getVectorNumElements();
4735 unsigned NumElems = VT.getVectorNumElements();
4743 : DAG.getUNDEF(VT.getVectorElementType());
4963 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
4966 assert(VT.is128BitVector() && "Unknown type for VShift");
4970 return DAG.getNode(ISD::BITCAST, dl, VT,
4977 VT, DebugLoc dl,
5007 unsigned RequiredAlign = VT.getSizeInBits()/8;
5034 unsigned NumElems = VT.getVectorNumElements();
5052 /// vector of type 'VT', see if the elements can be replaced by a single large
5060 static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts,
5062 EVT EltVT = VT.getVectorElementType();
5098 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
5102 return DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
5129 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
5146 MVT VT = Op.getValueType().getSimpleVT();
5149 assert((VT.is128BitVector() || VT.is256BitVector()) &&
5172 if (!ConstSplatVal && !Ld->hasNUsesOfValue(VT.getVectorNumElements(), 0))
5193 if (VT.is256BitVector())
5195 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc);
5211 bool Is256 = VT.is256BitVector();
5236 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5246 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5253 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5259 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5268 EVT VT = Op.getValueType();
5271 if (!isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
5301 if (ExtractedFromVec.getValueType() != VT)
5329 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
5330 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]);
5333 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
5344 MVT VT = Op.getValueType().getSimpleVT();
5345 MVT ExtVT = VT.getVectorElementType();
5352 if (VT == MVT::v4i32 || VT == MVT::v8i32)
5355 return getZeroVector(VT, Subtarget, DAG, dl);
5362 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256()))
5365 return getOnesVector(VT, Subtarget->hasInt256(), DAG, dl);
5397 return DAG.getUNDEF(VT);
5413 assert(VT == MVT::v2i64 && "Expected an SSE value type!");
5433 return DAG.getNode(ISD::BITCAST, dl, VT, Item);
5443 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
5447 if (VT.is256BitVector()) {
5448 SDValue ZeroVec = getZeroVector(VT, Subtarget, DAG, dl);
5449 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec,
5452 assert(VT.is128BitVector() && "Expected an SSE value type!");
5453 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
5461 if (VT.is256BitVector()) {
5465 assert(VT.is128BitVector() && "Expected an SSE value type!");
5468 return DAG.getNode(ISD::BITCAST, dl, VT, Item);
5476 unsigned NumBits = VT.getSizeInBits();
5477 return getVShift(true, VT,
5479 VT, Op.getOperand(1)),
5492 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
5499 return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]);
5513 return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
5525 if (VT.is256BitVector()) {
5538 return Concat128BitVectors(Lower, Upper, VT, NumElems, DAG, dl);
5546 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
5553 // If element VT is < 32 bits, convert it to inserts into a zero vector.
5566 // If element VT is == 32 bits, turn it into a number of shuffles.
5572 V[i] = getZeroVector(VT, Subtarget, DAG, dl);
5574 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
5584 V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]);
5587 V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]);
5590 V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]);
5603 return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]);
5606 if (Values.size() > 1 && VT.is128BitVector()) {
5612 SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG);
5625 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
5627 Result = DAG.getUNDEF(VT);
5631 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
5642 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
5644 V[i] = DAG.getUNDEF(VT);
5663 V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + EltStride]);
5702 MVT VT = SVOp->getValueType(0).getSimpleVT();
5703 MVT EltVT = VT.getVectorElementType();
5704 unsigned NumElems = VT.getVectorNumElements();
5708 if (!Subtarget->hasInt256() && VT == MVT::v16i16)
5738 MVT BlendVT = VT;
5742 V1 = DAG.getNode(ISD::BITCAST, dl, VT, V1);
5743 V2 = DAG.getNode(ISD::BITCAST, dl, VT, V2);
5748 return DAG.getNode(ISD::BITCAST, dl, VT, Ret);
6143 MVT VT = SVOp->getValueType(0).getSimpleVT();
6157 if (VT != MVT::v32i8 || !Subtarget->hasInt256() ||
6191 MVT VT = SVOp->getValueType(0).getSimpleVT();
6193 unsigned NumElems = VT.getVectorNumElements();
6196 switch (VT.SimpleTy) {
6228 static SDValue getVZextMovL(MVT VT, EVT OpVT,
6231 if (VT == MVT::v2f64 || VT == MVT::v4f32) {
6245 return DAG.getNode(ISD::BITCAST, dl, VT,
6255 return DAG.getNode(ISD::BITCAST, dl, VT,
6270 MVT VT = SVOp->getValueType(0).getSimpleVT();
6272 unsigned NumElems = VT.getVectorNumElements();
6276 MVT EltVT = VT.getVectorElementType();
6372 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Output[0], Output[1]);
6382 MVT VT = SVOp->getValueType(0).getSimpleVT();
6384 assert(VT.is128BitVector() && "Unsupported vector size");
6416 V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
6427 return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]);
6458 V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
6465 return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
6476 return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]);
6508 SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]);
6509 SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]);
6514 return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]);
6533 EVT VT = Op.getValueType();
6537 return DAG.getNode(ISD::BITCAST, dl, VT,
6547 EVT VT = Op.getValueType();
6549 assert(VT != MVT::v2i64 && "unsupported shuffle type");
6551 if (HasSSE2 && VT == MVT::v2f64)
6552 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
6555 return DAG.getNode(ISD::BITCAST, dl, VT,
6565 EVT VT = Op.getValueType();
6567 assert((VT == MVT::v4i32 || VT == MVT::v4f32) &&
6574 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
6581 EVT VT = Op.getValueType();
6582 unsigned NumElems = VT.getVectorNumElements();
6605 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
6610 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG);
6622 if (NumElems == 2 || !isMOVLMask(SVOp->getMask(), VT))
6623 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
6624 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
6627 assert(VT != MVT::v4i32 && "unsupported shuffle type");
6630 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1,
6641 EVT VT = Op.getValueType();
6644 if (!Subtarget->hasInt256() && VT.is256BitVector())
6651 unsigned NumElems = VT.getVectorNumElements();
6655 if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() ||
6656 VT.getVectorElementType() == MVT::i64)
6681 unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift;
6719 return DAG.getNode(ISD::BITCAST, DL, VT,
6726 MVT VT = Op.getValueType().getSimpleVT();
6732 return getZeroVector(VT, Subtarget, DAG, dl);
6749 if (VT == MVT::v8i16 || VT == MVT::v16i8 ||
6750 VT == MVT::v16i16 || VT == MVT::v32i8) {
6753 return DAG.getNode(ISD::BITCAST, dl, VT, NewOp);
6754 } else if ((VT == MVT::v4i32 ||
6755 (VT == MVT::v4f32 && Subtarget->hasSSE2()))) {
6764 return getVZextMovL(VT, NewVT, NewOp.getOperand(0),
6772 return getVZextMovL(VT, NewVT, NewOp.getOperand(1),
6785 MVT VT = Op.getValueType().getSimpleVT();
6787 unsigned NumElems = VT.getVectorNumElements();
6799 assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
6802 return DAG.getUNDEF(VT);
6831 if (OptForSize && isUNPCKL_v_undef_Mask(M, VT, HasInt256))
6832 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
6833 if (OptForSize && isUNPCKH_v_undef_Mask(M, VT, HasInt256))
6834 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
6836 if (isMOVDDUPMask(M, VT) && Subtarget->hasSSE3() &&
6840 if (isMOVHLPS_v_undef_Mask(M, VT))
6844 if (HasSSE2 && isUNPCKHMask(M, VT, HasInt256) && V2IsUndef &&
6845 (VT == MVT::v2f64 || VT == MVT::v2i64))
6846 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
6848 if (isPSHUFDMask(M, VT)) {
6852 if (isMOVDDUPMask(M, VT) && ((VT == MVT::v4f32 || VT == MVT::v2i64)))
6853 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG);
6857 if (HasSSE2 && (VT == MVT::v4f32 || VT == MVT::v4i32))
6858 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG);
6860 if (HasFp256 && (VT == MVT::v4f32 || VT == MVT::v2f64))
6861 return getTargetShuffleNode(X86ISD::VPERMILP, dl, VT, V1, TargetMask,
6864 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V1,
6876 MVT EltVT = VT.getVectorElementType();
6878 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
6881 if (isMOVLMask(M, VT)) {
6883 return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl);
6884 if (!isMOVLPMask(M, VT)) {
6885 if (HasSSE2 && (VT == MVT::v2i64 || VT == MVT::v2f64))
6886 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
6888 if (VT == MVT::v4i32 || VT == MVT::v4f32)
6889 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
6894 if (isMOVLHPSMask(M, VT) && !isUNPCKLMask(M, VT, HasInt256))
6897 if (isMOVHLPSMask(M, VT))
6900 if (V2IsUndef && isMOVSHDUPMask(M, VT, Subtarget))
6901 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG);
6903 if (V2IsUndef && isMOVSLDUPMask(M, VT, Subtarget))
6904 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG);
6906 if (isMOVLPMask(M, VT))
6909 if (ShouldXformToMOVHLPS(M, VT) ||
6910 ShouldXformToMOVLP(V1.getNode(), V2.getNode(), M, VT))
6915 MVT EltVT = VT.getVectorElementType();
6917 return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
6934 if (isCommutedMOVLMask(M, VT, V2IsSplat, V2IsUndef)) {
6941 return getMOVL(DAG, dl, VT, V2, V1);
6944 if (isUNPCKLMask(M, VT, HasInt256))
6945 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
6947 if (isUNPCKHMask(M, VT, HasInt256))
6948 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
6956 if (isUNPCKLMask(NewMask, VT, HasInt256, true))
6957 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
6958 if (isUNPCKHMask(NewMask, VT, HasInt256, true))
6959 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
6970 if (isUNPCKLMask(M, VT, HasInt256))
6971 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
6973 if (isUNPCKHMask(M, VT, HasInt256))
6974 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
6978 if (!V2IsUndef && (isSHUFPMask(M, VT, HasFp256, /* Commuted */ true)))
6985 if (isPALIGNRMask(M, VT, Subtarget))
6986 return getTargetShuffleNode(X86ISD::PALIGNR, dl, VT, V1, V2,
6990 if (ShuffleVectorSDNode::isSplatMask(&M[0], VT) &&
6992 if (VT == MVT::v2f64 || VT == MVT::v2i64)
6993 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
6996 if (isPSHUFHWMask(M, VT, HasInt256))
6997 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1,
7001 if (isPSHUFLWMask(M, VT, HasInt256))
7002 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1,
7006 if (isSHUFPMask(M, VT, HasFp256))
7007 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V2,
7010 if (isUNPCKL_v_undef_Mask(M, VT, HasInt256))
7011 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
7012 if (isUNPCKH_v_undef_Mask(M, VT, HasInt256))
7013 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
7021 if (V2IsUndef && isMOVDDUPYMask(M, VT, HasFp256))
7022 return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG);
7025 if (isVPERMILPMask(M, VT, HasFp256)) {
7026 if (HasInt256 && VT == MVT::v8i32)
7027 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1,
7029 return getTargetShuffleNode(X86ISD::VPERMILP, dl, VT, V1,
7034 if (isVPERM2X128Mask(M, VT, HasFp256))
7035 return getTargetShuffleNode(X86ISD::VPERM2X128, dl, VT, V1,
7042 if (V2IsUndef && HasInt256 && (VT == MVT::v8i32 || VT == MVT::v8f32)) {
7050 return DAG.getNode(X86ISD::VPERMV, dl, VT,
7051 DAG.getNode(ISD::BITCAST, dl, VT, Mask), V1);
7054 if (V2IsUndef && HasInt256 && (VT == MVT::v4i64 || VT == MVT::v4f64))
7055 return getTargetShuffleNode(X86ISD::VPERMI, dl, VT, V1,
7065 if (VT == MVT::v8i16) {
7071 if (VT == MVT::v16i8) {
7077 if (VT == MVT::v32i8) {
7085 if (NumElems == 4 && VT.is128BitVector())
7089 if (VT.is256BitVector())
7096 MVT VT = Op.getValueType().getSimpleVT();
7102 if (VT.getSizeInBits() == 8) {
7106 DAG.getValueType(VT));
7107 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
7110 if (VT.getSizeInBits() == 16) {
7123 DAG.getValueType(VT));
7124 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
7127 if (VT == MVT::f32) {
7149 if (VT == MVT::i32 || VT == MVT::i64) {
7191 MVT VT = Op.getValueType().getSimpleVT();
7194 if (VT.getSizeInBits() == 16) {
7208 DAG.getValueType(VT));
7209 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
7212 if (VT.getSizeInBits() == 32) {
7222 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
7226 if (VT.getSizeInBits() == 64) {
7241 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
7249 MVT VT = Op.getValueType().getSimpleVT();
7250 MVT EltVT = VT.getVectorElementType();
7257 if (!VT.is128BitVector())
7263 if (VT == MVT::v8i16)
7265 else if (VT == MVT::v16i8)
7276 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
7291 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
7303 MVT VT = Op.getValueType().getSimpleVT();
7304 MVT EltVT = VT.getVectorElementType();
7313 if (VT.is256BitVector()) {
7318 unsigned NumElems = VT.getVectorNumElements();
7344 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
7909 EVT VT = Op.getValueType();
7910 unsigned VTBits = VT.getSizeInBits();
7916 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
7918 : DAG.getConstant(0, VT);
7922 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
7923 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
7925 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
7926 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt);
7940 Hi = DAG.getNode(X86ISD::CMOV, dl, VT
7941 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
7943 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
7944 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
8359 MVT VT = Op->getValueType(0).getSimpleVT();
8377 if (((VT != MVT::v8i32) || (InVT != MVT::v8i16)) &&
8378 ((VT != MVT::v4i64) || (InVT != MVT::v4i32)))
8382 return DAG.getNode(X86ISD::VZEXT_MOVL, dl, VT, In);
8390 MVT HVT = MVT::getVectorVT(VT.getVectorElementType(),
8391 VT.getVectorNumElements()/2);
8396 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
8412 MVT VT = Op.getValueType().getSimpleVT();
8422 if (!VT.is256BitVector() || !SVT.is128BitVector() ||
8423 VT.getVectorNumElements() != SVT.getVectorNumElements())
8430 return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
8444 MVT VT = Op.getValueType().getSimpleVT();
8448 if ((VT == MVT::v4i32) && (SVT == MVT::v4i64)) {
8455 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
8470 SDValue Undef = DAG.getUNDEF(VT);
8471 OpLo = DAG.getVectorShuffle(VT, DL, OpLo, Undef, ShufMask1);
8472 OpHi = DAG.getVectorShuffle(VT, DL, OpHi, Undef, ShufMask1);
8476 return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask2);
8479 if ((VT == MVT::v8i16) && (SVT == MVT::v8i32)) {
8507 return DAG.getNode(ISD::BITCAST, DL, VT, In);
8537 if (!VT.is128BitVector() || !SVT.is256BitVector())
8540 assert(VT.getVectorNumElements() != SVT.getVectorNumElements() &&
8544 unsigned NumElems = VT.getVectorNumElements();
8545 EVT NVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
8555 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V,
8561 MVT VT = Op.getValueType().getSimpleVT();
8562 if (VT.isVector()) {
8563 if (VT == MVT::v8i16)
8564 return DAG.getNode(ISD::TRUNCATE, Op.getDebugLoc(), VT,
8605 MVT VT = Op.getValueType().getSimpleVT();
8611 return DAG.getNode(X86ISD::VFPEXT, DL, VT,
8619 MVT VT = Op.getValueType().getSimpleVT();
8620 MVT EltVT = VT;
8621 unsigned NumElts = VT == MVT::f64 ? 2 : 4;
8622 if (VT.isVector()) {
8623 EltVT = VT.getVectorElementType();
8624 NumElts = VT.getVectorNumElements();
8636 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
8639 if (VT.isVector()) {
8640 MVT ANDVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
8641 return DAG.getNode(ISD::BITCAST, dl, VT,
8647 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
8653 MVT VT = Op.getValueType().getSimpleVT();
8654 MVT EltVT = VT;
8655 unsigned NumElts = VT == MVT::f64 ? 2 : 4;
8656 if (VT.isVector()) {
8657 EltVT = VT.getVectorElementType();
8658 NumElts = VT.getVectorNumElements();
8670 SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
8673 if (VT.isVector()) {
8674 MVT XORVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
8675 return DAG.getNode(ISD::BITCAST, dl, VT,
8682 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
8690 MVT VT = Op.getValueType().getSimpleVT();
8694 if (SrcVT.bitsLT(VT)) {
8695 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1);
8696 SrcVT = VT;
8699 if (SrcVT.bitsGT(VT)) {
8700 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
8701 SrcVT = VT;
8728 if (SrcVT.bitsGT(VT)) {
8740 if (VT == MVT::f64) {
8755 SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
8758 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2);
8761 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
8767 MVT VT = Op.getValueType().getSimpleVT();
8770 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
8771 DAG.getConstant(1, VT));
8772 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT));
8792 EVT VT = MVT::Other;
8822 VT = ExtractedFromVec.getValueType();
8824 if (!VT.is128BitVector() && !VT.is256BitVector())
8828 VT != VecInMap.begin()->first.getValueType())
8835 assert((VT.is128BitVector() || VT.is256BitVector()) &&
8838 unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U;
8849 EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
9038 EVT VT = Op.getValueType();
9057 SDValue V0 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(0));
9058 SDValue V1 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(1));
9059 Op = DAG.getNode(ConvertedOp, dl, VT, V0, V1);
9213 MVT VT = Op.getValueType().getSimpleVT();
9215 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&
9218 unsigned NumElems = VT.getVectorNumElements();
9233 MVT EltVT = VT.getVectorElementType();
9235 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
9246 MVT VT = Op.getValueType().getSimpleVT();
9306 SDValue Cmp0 = DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1,
9308 SDValue Cmp1 = DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1,
9310 return DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
9313 return DAG.getNode(X86ISD::CMPP, dl, VT, Op0, Op1,
9318 if (VT.is256BitVector() && !Subtarget->hasInt256())
9345 if (VT == MVT::v2i64) {
9368 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
9375 EVT EltVT = VT.getVectorElementType();
9378 std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit);
9379 SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0],
9381 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec);
9382 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec);
9385 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
9389 Result = DAG.getNOT(dl, Result, VT);
9396 MVT VT = Op.getValueType().getSimpleVT();
9398 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
9400 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
9573 MVT VT = Op.getValueType().getSimpleVT();
9576 if (VT.isFloatingPoint() && !VT.isVector() &&
9577 !isScalarFPTypeInSSEReg(VT)) // FPStack?
9684 MVT VT = Op->getValueType(0).getSimpleVT();
9689 if ((VT != MVT::v4i64 || InVT != MVT::v4i32) &&
9690 (VT != MVT::v8i32 || InVT != MVT::v8i16))
9694 return DAG.getNode(X86ISD::VSEXT_MOVL, dl, VT, In);
9703 // concat the vectors to original VT
9720 MVT HalfVT = MVT::getVectorVT(VT.getScalarType(),
9721 VT.getVectorNumElements()/2);
9726 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
10231 static SDValue getTargetVShiftNode(unsigned Opc, DebugLoc dl, EVT VT,
10244 return DAG.getNode(Opc, dl, VT, SrcOp,
10267 MVT EltVT = VT.getVectorElementType().getSimpleVT();
10271 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
10979 EVT VT = Op.getValueType();
10983 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
10985 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
11213 EVT VT = Op.getValueType();
11252 return DAG.getNode((VT.getSizeInBits() < 16 ?
11253 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
11257 EVT VT = Op.getValueType();
11258 EVT OpVT = VT;
11259 unsigned NumBits = VT.getSizeInBits();
11263 if (VT == MVT::i8) {
11285 if (VT == MVT::i8)
11291 EVT VT = Op.getValueType();
11292 EVT OpVT = VT;
11293 unsigned NumBits = VT.getSizeInBits();
11297 if (VT == MVT::i8) {
11310 if (VT == MVT::i8)
11316 EVT VT = Op.getValueType();
11317 unsigned NumBits = VT.getSizeInBits();
11322 VT, MVT::i32);
11328 DAG.getConstant(NumBits, VT),
11332 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops, array_lengthof(Ops));
11338 EVT VT = Op.getValueType();
11340 assert(VT.is256BitVector() && VT.isInteger() &&
11343 unsigned NumElems = VT.getVectorNumElements();
11356 MVT EltVT = VT.getVectorElementType().getSimpleVT();
11359 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
11381 EVT VT = Op.getValueType();
11384 if (VT.is256BitVector() && !Subtarget->hasInt256())
11391 if (VT == MVT::v4i32) {
11397 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
11398 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
11405 Evens = DAG.getNode(ISD::BITCAST, dl, VT, Evens);
11406 Odds = DAG.getNode(ISD::BITCAST, dl, VT, Odds);
11411 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
11414 assert((VT == MVT::v2i64 || VT == MVT::v4i64) &&
11430 SDValue Ahi = DAG.getNode(X86ISD::VSRLI, dl, VT, A, ShAmt);
11431 SDValue Bhi = DAG.getNode(X86ISD::VSRLI, dl, VT, B, ShAmt);
11434 EVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 : MVT::v8i32;
11440 SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
11441 SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
11442 SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
11444 AloBhi = DAG.getNode(X86ISD::VSHLI, dl, VT, AloBhi, ShAmt);
11445 AhiBlo = DAG.getNode(X86ISD::VSHLI, dl, VT, AhiBlo, ShAmt);
11447 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
11448 return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
11452 EVT VT = Op.getValueType();
11453 EVT EltTy = VT.getVectorElementType();
11454 unsigned NumElts = VT.getVectorNumElements();
11474 SDValue SGN = getTargetVShiftNode(X86ISD::VSRAI, dl, VT, N0, Sz, DAG);
11477 SDValue SRL = getTargetVShiftNode(X86ISD::VSRLI, dl, VT, SGN, Amt, DAG);
11478 SDValue ADD = DAG.getNode(ISD::ADD, dl, VT, N0, SRL);
11480 SDValue SRA = getTargetVShiftNode(X86ISD::VSRAI, dl, VT, ADD, Lg2Amt, DAG);
11488 SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], NumElts);
11489 return DAG.getNode(ISD::SUB, dl, VT, Zero, SRA);
11496 EVT VT = Op.getValueType();
11510 if (VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 ||
11512 (VT == MVT::v4i64 || VT == MVT::v8i32 || VT == MVT::v16i16))) {
11514 return DAG.getNode(X86ISD::VSHLI, dl, VT, R,
11517 return DAG.getNode(X86ISD::VSRLI, dl, VT, R,
11519 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64)
11520 return DAG.getNode(X86ISD::VSRAI, dl, VT, R,
11524 if (VT == MVT::v16i8) {
11529 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
11534 return DAG.getNode(ISD::AND, dl, VT, SHL,
11535 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 16));
11541 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
11546 return DAG.getNode(ISD::AND, dl, VT, SRL,
11547 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 16));
11552 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
11553 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
11557 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
11560 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 16);
11561 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
11562 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
11568 if (Subtarget->hasInt256() && VT == MVT::v32i8) {
11573 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL);
11578 return DAG.getNode(ISD::AND, dl, VT, SHL,
11579 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 32));
11585 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL);
11590 return DAG.getNode(ISD::AND, dl, VT, SRL,
11591 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 32));
11596 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
11597 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
11601 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
11604 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &V[0], 32);
11605 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
11606 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
11615 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
11616 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT));
11618 Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT));
11620 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
11621 return DAG.getNode(ISD::MUL, dl, VT, Op, R);
11623 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
11627 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT));
11628 Op = DAG.getNode(ISD::BITCAST, dl, VT, Op);
11631 SDValue VSelM = DAG.getConstant(0x80, VT);
11632 SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
11633 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
11635 SDValue CM1 = DAG.getConstant(0x0f, VT);
11636 SDValue CM2 = DAG.getConstant(0x3f, VT);
11639 SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1);
11642 M = DAG.getNode(ISD::BITCAST, dl, VT, M);
11643 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
11646 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
11647 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
11648 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
11651 M = DAG.getNode(ISD::AND, dl, VT, R, CM2);
11654 M = DAG.getNode(ISD::BITCAST, dl, VT, M);
11655 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R);
11658 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
11659 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
11660 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
11663 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel,
11664 DAG.getNode(ISD::ADD, dl, VT, R, R), R);
11669 if (VT.is256BitVector()) {
11670 unsigned NumElems = VT.getVectorNumElements();
11671 MVT EltVT = VT.getVectorElementType().getSimpleVT();
11704 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2);
11789 EVT VT = Op.getValueType();
11791 if (!Subtarget->hasSSE2() || !VT.isVector())
11794 unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
11798 switch (VT.getSimpleVT().SimpleTy) {
11806 unsigned NumElems = VT.getVectorNumElements();
11813 MVT EltVT = VT.getVectorElementType().getSimpleVT();
11825 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, LHS1, LHS2);
11830 SDValue Tmp1 = getTargetVShiftNode(X86ISD::VSHLI, dl, VT,
11832 return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, Tmp1, ShAmt, DAG);
12011 EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
12019 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
12034 EVT VT = Op.getNode()->getValueType(0);
12037 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
12040 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
12186 EVT VT = cast<AtomicSDNode>(Node)->getMemoryVT();
12192 SDValue Zero = DAG.getConstant(0, VT);
12193 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, dl, VT,
12254 EVT VT = N->getValueType(0);
12257 Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot,
12685 EVT VT) const {
12687 if (VT.getSizeInBits() == 64)
12691 return (VT.getVectorNumElements() == 2 ||
12692 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
12693 isMOVLMask(M, VT) ||
12694 isSHUFPMask(M, VT, Subtarget->hasFp256()) ||
12695 isPSHUFDMask(M, VT) ||
12696 isPSHUFHWMask(M, VT, Subtarget->hasInt256()) ||
12697 isPSHUFLWMask(M, VT, Subtarget->hasInt256()) ||
12698 isPALIGNRMask(M, VT, Subtarget) ||
12699 isUNPCKLMask(M, VT, Subtarget->hasInt256()) ||
12700 isUNPCKHMask(M, VT, Subtarget->hasInt256()) ||
12701 isUNPCKL_v_undef_Mask(M, VT, Subtarget->hasInt256()) ||
12702 isUNPCKH_v_undef_Mask(M, VT, Subtarget->hasInt256()));
12707 EVT VT) const {
12708 unsigned NumElts = VT.getVectorNumElements();
12712 if (NumElts == 4 && VT.is128BitVector()) {
12713 return (isMOVLMask(Mask, VT) ||
12714 isCommutedMOVLMask(Mask, VT, true) ||
12715 isSHUFPMask(Mask, VT, Subtarget->hasFp256()) ||
12716 isSHUFPMask(Mask, VT, Subtarget->hasFp256(), /* Commuted */ true));
12782 static unsigned getCmpXChgOpcode(EVT VT) {
12783 switch (VT.getSimpleVT().SimpleTy) {
12795 static unsigned getLoadOpcode(EVT VT) {
12796 switch (VT.getSimpleVT().SimpleTy) {
12888 static unsigned getPseudoCMOVOpc(EVT VT) {
12889 switch (VT.getSimpleVT().SimpleTy) {
12954 MVT::SimpleValueType VT = *RC->vt_begin();
12959 unsigned PhyReg = getX86SubSuperRegister(X86::EAX, VT);
12961 unsigned LCMPXCHGOpc = getCmpXChgOpcode(VT);
12962 unsigned LOADOpc = getLoadOpcode(VT);
13077 if (VT != MVT::i8) {
13112 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
13114 unsigned SelOpc = getPseudoCMOVOpc(VT);
14750 EVT VT = SVOp->getValueType(0);
14751 unsigned NumElems = VT.getVectorNumElements();
14766 EVT VT = SVOp->getValueType(0);
14767 unsigned NumElems = VT.getVectorNumElements();
14786 EVT VT = SVOp->getValueType(0);
14787 unsigned NumElems = VT.getVectorNumElements();
14841 return DAG.getNode(ISD::BITCAST, dl, VT, ResNode);
14847 SDValue Zeros = getZeroVector(VT, Subtarget, DAG, dl);
14859 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, 0, DAG, dl);
14866 SDValue InsV = Insert128BitVector(DAG.getUNDEF(VT), V, NumElems/2, DAG, dl);
14878 EVT VT = N->getValueType(0);
14882 if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(VT.getVectorElementType()))
14886 if (Subtarget->hasFp256() && VT.is256BitVector() &&
14891 if (!VT.is128BitVector())
14898 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
14901 return EltsFromConsecutiveLoads(VT, Elts, dl, DAG);
14928 EVT VT = InVec.getValueType();
14936 if (BCVT.getVectorNumElements() != VT.getVectorNumElements())
14951 if (!getTargetShuffleMask(InVec.getNode(), VT.getSimpleVT(), ShuffleMask,
14956 unsigned NumElems = VT.getVectorNumElements();
14988 getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
14990 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VT))
14999 SDValue Shuffle = (UnaryShuffle) ? DAG.getUNDEF(VT) : InVec.getOperand(1);
15003 Shuffle = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
15105 static unsigned matchIntegerMINMAX(SDValue Cond, EVT VT, SDValue LHS,
15108 if (!VT.isVector())
15111 switch (VT.getSimpleVT().SimpleTy) {
15127 (Subtarget->hasSSE2() && VT == MVT::v16i8);
15129 (Subtarget->hasSSE2() && VT == MVT::v8i16);
15184 EVT VT = LHS.getValueType();
15190 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
15191 VT != MVT::f80 && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
15193 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) {
15449 return DAG.getNode(ISD::SELECT, DL, VT, Cond, LHS, RHS);
15458 ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) ||
15459 (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) {
15482 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
15490 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS,
15491 DAG.getConstant(-A, VT));
15504 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
15512 if (unsigned Op = matchIntegerMINMAX(Cond, VT, LHS, RHS, DAG, Subtarget))
15521 !DCI.isBeforeLegalize() && TLI.isOperationLegal(ISD::VSELECT, VT)) {
15820 EVT VT = N->getValueType(0);
15821 if (VT != MVT::i64)
15856 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
15859 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
15860 DAG.getConstant(MulAmt1, VT));
15863 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
15866 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
15867 DAG.getConstant(MulAmt2, VT));
15879 EVT VT = N0.getValueType();
15883 if (VT.isInteger() && !VT.isVector() &&
15895 return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
15896 N00, DAG.getConstant(Mask, VT));
15911 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, N0, N0);
15923 EVT VT = N->getValueType(0);
15936 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 &&
15938 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16)))
15942 EVT EltVT = VT.getVectorElementType();
15946 unsigned NumElts = VT.getVectorNumElements();
16010 switch (VT.getSimpleVT().SimpleTy) {
16018 return getTargetVShiftNode(X86ISD::VSHLI, DL, VT, ValOp, BaseShAmt, DAG);
16021 switch (VT.getSimpleVT().SimpleTy) {
16027 return getTargetVShiftNode(X86ISD::VSRAI, DL, VT, ValOp, BaseShAmt, DAG);
16030 switch (VT.getSimpleVT().SimpleTy) {
16038 return getTargetVShiftNode(X86ISD::VSRLI, DL, VT, ValOp, BaseShAmt, DAG);
16066 EVT VT = CMP00.getValueType();
16068 if (VT == MVT::f32 || VT == MVT::f64) {
16123 EVT VT = N->getValueType(0);
16135 if (VT.is256BitVector() &&
16157 EVT VT = N->getValueType(0);
16158 if (!VT.is256BitVector())
16185 if (WideVT != VT)
16220 Mask = Mask.zext(VT.getScalarType().getSizeInBits());
16221 return DAG.getNode(ISD::AND, DL, VT,
16222 Op, DAG.getConstant(Mask, VT));
16225 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
16235 EVT VT = N->getValueType(0);
16246 if (Subtarget->hasBMI() && (VT == MVT::i32 || VT == MVT::i64)) {
16254 return DAG.getNode(X86ISD::BLSI, DL, VT, N1);
16259 return DAG.getNode(X86ISD::BLSI, DL, VT, N0);
16264 return DAG.getNode(X86ISD::BLSR, DL, VT, N1);
16269 return DAG.getNode(X86ISD::BLSR, DL, VT, N0);
16278 if (VT != MVT::v2i64 && VT != MVT::v4i64)
16289 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
16295 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
16303 EVT VT = N->getValueType(0);
16315 if (VT == MVT::v2i64 || VT == MVT::v4i64) {
16317 (VT == MVT::v4i64 && !Subtarget->hasInt256()))
16370 "Unsupported VT for PSIGN");
16372 return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
16378 EVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8;
16384 return DAG.getNode(ISD::BITCAST, DL, VT, Mask);
16388 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
16420 unsigned Bits = VT.getSizeInBits();
16428 return DAG.getNode(Opc, DL, VT,
16437 return DAG.getNode(Opc, DL, VT,
16448 EVT VT = N->getValueType(0);
16452 if (VT.isInteger() && VT.getSizeInBits() == 8)
16461 if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
16467 if (Y1C->getAPIntValue() == VT.getSizeInBits()-1) {
16469 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
16470 DAG.getConstant(0, VT), N0.getOperand(0));
16475 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue),
16485 EVT VT = N->getValueType(0);
16499 if (VT != MVT::i32 && VT != MVT::i64)
16511 return DAG.getNode(X86ISD::BLSMSK, DL, VT, N1);
16515 return DAG.getNode(X86ISD::BLSMSK, DL, VT, N0);
16721 EVT VT = St->getValue().getValueType();
16727 bool IsAligned = Alignment == 0 || Alignment == VT.getSizeInBits()/8;
16733 if (VT.is256BitVector() && !Subtarget->hasInt256() &&
16734 StVT == VT && !IsAligned) {
16735 unsigned NumElems = VT.getVectorNumElements();
16759 if (St->isTruncatingStore() && VT.isVector()) {
16761 unsigned NumElems = VT.getVectorNumElements();
16762 assert(StVT != VT && "Cannot truncate to the same type");
16763 unsigned FromSz = VT.getVectorElementType().getSizeInBits();
16774 assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
16780 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
16813 StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());
16814 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
16843 if (VT.getSizeInBits() != 64)
16851 if ((VT.isVector() ||
16852 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
16883 if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))
16980 EVT VT = LHS.getValueType();
16982 assert((VT.is128BitVector() || VT.is256BitVector()) &&
16987 unsigned NumElts = VT.getVectorNumElements();
16988 unsigned NumLanes = VT.getSizeInBits()/128;
16999 // type VT.
17078 EVT VT = N->getValueType(0);
17083 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
17084 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
17086 return DAG.getNode(X86ISD::FHADD, N->getDebugLoc(), VT, LHS, RHS);
17093 EVT VT = N->getValueType(0);
17098 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
17099 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
17101 return DAG.getNode(X86ISD::FHSUB, N->getDebugLoc(), VT, LHS, RHS);
17178 EVT VT = N->getValueType(0), OpVT = Op.getValueType();
17180 VT.getVectorElementType().getSizeInBits() ==
17182 return DAG.getNode(ISD::BITCAST, N->getDebugLoc(), VT, Op);
17189 EVT VT = N->getValueType(0);
17190 if (!VT.isVector())
17203 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
17231 EVT VT = N->getValueType(0);
17232 if (VT.isVector() && VT.getSizeInBits() == 256) {
17244 EVT VT = N->getValueType(0);
17247 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
17250 EVT ScalarVT = VT.getScalarType();
17278 return DAG.getNode(Opcode, dl, VT, A, B, C);
17290 EVT VT = N->getValueType(0);
17300 return DAG.getNode(ISD::AND, dl, VT,
17301 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
17303 DAG.getConstant(1, VT));
17307 if (VT.is256BitVector()) {
17435 EVT VT = Ld->getValueType(0);
17439 !DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
17461 EVT VT = N->getValueType(0);
17463 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,
17464 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
17467 DAG.getConstant(1, VT));
17517 EVT VT = N->getValueType(0);
17522 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
17523 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
17525 return DAG.getNode(X86ISD::HADD, N->getDebugLoc(), VT, Op0, Op1);
17544 EVT VT = Op0.getValueType();
17545 SDValue NewXor = DAG.getNode(ISD::XOR, Op1.getDebugLoc(), VT,
17547 DAG.getConstant(~XorC, VT));
17548 return DAG.getNode(ISD::ADD, N->getDebugLoc(), VT, NewXor,
17549 DAG.getConstant(C->getAPIntValue()+1, VT));
17554 EVT VT = N->getValueType(0);
17555 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) ||
17556 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) &&
17558 return DAG.getNode(X86ISD::HSUB, N->getDebugLoc(), VT, Op0, Op1);
17644 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
17645 if (!isTypeLegal(VT))
17647 if (VT
17673 EVT VT = Op.getValueType();
17674 if (VT != MVT::i16)
18150 EVT VT) const {
18162 if (VT == MVT::i32 || VT == MVT::f32)
18164 if (VT == MVT::i16)
18166 if (VT == MVT::i8 || VT == MVT::i1)
18168 if (VT == MVT::i64 || VT == MVT::f64)
18174 if (VT == MVT::i32 || VT == MVT::f32)
18176 if (VT == MVT::i16)
18178 if (VT == MVT::i8 || VT == MVT::i1)
18180 if (VT == MVT::i64)
18185 if (VT == MVT::i8 || VT == MVT::i1)
18187 if (VT == MVT::i16)
18189 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit())
18193 if (VT == MVT::i8 || VT == MVT::i1)
18195 if (VT == MVT::i16)
18197 if (VT == MVT::i32 || !Subtarget->is64Bit())
18201 // If SSE is enabled for this VT, use f80 to ensure the isel moves the
18203 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
18205 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
18217 switch (VT.getSimpleVT().SimpleTy) {
18250 Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
18294 if (Res.second->hasType(VT))
18302 if (VT == MVT::i8 || VT == MVT::i1) {
18315 } else if (VT == MVT::i32 || VT == MVT::f32) {
18332 } else if (VT == MVT::i64 || VT == MVT::f64) {
18358 if (VT == MVT::f32 || VT == MVT::i32)
18360 else if (VT == MVT::f64 || VT == MVT::i64)
18362 else if (X86::VR128RegClass.hasType(VT))
18364 else if (X86::VR256RegClass.hasType(VT))
