Lines Matching refs:X86ISD
1913 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
2216 X86ISD::NodeType opcode = X86ISD::RET_FLAG;
2218 opcode = X86ISD::IRET;
2240 if (UI->getOpcode() != X86ISD::RET_FLAG)
2660 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
2799 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
3173 unsigned(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
3318 Callee = DAG.getNode(X86ISD::WrapperRIP, dl,
3395 return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
3398 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
3793 case X86ISD::BLENDI:
3794 case X86ISD::PSHUFB:
3795 case X86ISD::PSHUFD:
3796 case X86ISD::PSHUFHW:
3797 case X86ISD::PSHUFLW:
3798 case X86ISD::SHUFP:
3799 case X86ISD::INSERTPS:
3800 case X86ISD::PALIGNR:
3801 case X86ISD::VSHLDQ:
3802 case X86ISD::VSRLDQ:
3803 case X86ISD::MOVLHPS:
3804 case X86ISD::MOVLHPD:
3805 case X86ISD::MOVHLPS:
3806 case X86ISD::MOVLPS:
3807 case X86ISD::MOVLPD:
3808 case X86ISD::MOVSHDUP:
3809 case X86ISD::MOVSLDUP:
3810 case X86ISD::MOVDDUP:
3811 case X86ISD::MOVSS:
3812 case X86ISD::MOVSD:
3813 case X86ISD::UNPCKL:
3814 case X86ISD::UNPCKH:
3815 case X86ISD::VPERMILPI:
3816 case X86ISD::VPERMILPV:
3817 case X86ISD::VPERM2X128:
3818 case X86ISD::VPERMIL2:
3819 case X86ISD::VPERMI:
3820 case X86ISD::VPPERM:
3821 case X86ISD::VPERMV:
3822 case X86ISD::VPERMV3:
3823 case X86ISD::VZEXT_MOVL:
3831 case X86ISD::PSHUFB:
3832 case X86ISD::VPERMILPV:
3842 case X86ISD::PSHUFD:
3843 case X86ISD::PSHUFHW:
3844 case X86ISD::PSHUFLW:
3845 case X86ISD::VPERMILPI:
3846 case X86ISD::VPERMI:
3856 case X86ISD::MOVLHPS:
3857 case X86ISD::MOVLHPD:
3858 case X86ISD::MOVHLPS:
3859 case X86ISD::MOVLPS:
3860 case X86ISD::MOVLPD:
3861 case X86ISD::MOVSS:
3862 case X86ISD::MOVSD:
3863 case X86ISD::UNPCKL:
3864 case X86ISD::UNPCKH:
4141 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
4522 return DAG.getNode(X86ISD::BLENDI, dl, ResultVT, Result, Vec256, Mask);
4540 Vec256 = DAG.getNode(X86ISD::BLENDI, dl, CastVT, Result, Vec256, Mask);
4605 WideSubVec = DAG.getNode(X86ISD::VSHLI, dl, WideOpVT, WideSubVec, ShiftBits);
4614 Vec = DAG.getNode(X86ISD::VSHLI, dl, WideOpVT, WideSubVec,
4616 Vec = ShiftRight ? DAG.getNode(X86ISD::VSRLI, dl, WideOpVT, Vec,
4625 Vec = DAG.getNode(X86ISD::VSRLI, dl, WideOpVT, Vec, ShiftBits);
4626 Vec = DAG.getNode(X86ISD::VSHLI, dl, WideOpVT, Vec, ShiftBits);
4638 WideSubVec = DAG.getNode(X86ISD::VSHLI, dl, WideOpVT, WideSubVec,
4642 Vec = DAG.getNode(X86ISD::VSHLI, dl, WideOpVT, Vec, ShiftBits);
4643 Vec = DAG.getNode(X86ISD::VSRLI, dl, WideOpVT, Vec, ShiftBits);
4768 if (MaskNode.getOpcode() == X86ISD::VBROADCAST) {
4783 if (MaskNode.getOpcode() == X86ISD::VZEXT_MOVL &&
4834 if (Ptr->getOpcode() == X86ISD::Wrapper ||
4835 Ptr->getOpcode() == X86ISD::WrapperRIP)
4864 case X86ISD::BLENDI:
4868 case X86ISD::SHUFP:
4873 case X86ISD::INSERTPS:
4878 case X86ISD::UNPCKH:
4882 case X86ISD::UNPCKL:
4886 case X86ISD::MOVHLPS:
4890 case X86ISD::MOVLHPS:
4894 case X86ISD::PALIGNR:
4899 case X86ISD::VSHLDQ:
4905 case X86ISD::VSRLDQ:
4911 case X86ISD::PSHUFD:
4912 case X86ISD::VPERMILPI:
4917 case X86ISD::PSHUFHW:
4922 case X86ISD::PSHUFLW:
4927 case X86ISD::VZEXT_MOVL:
4931 case X86ISD::VPERMILPV: {
4946 case X86ISD::PSHUFB: {
4960 case X86ISD::VPERMI:
4965 case X86ISD::MOVSS:
4966 case X86ISD::MOVSD:
4969 case X86ISD::VPERM2X128:
4974 case X86ISD::MOVSLDUP:
4978 case X86ISD::MOVSHDUP:
4982 case X86ISD::MOVDDUP:
4986 case X86ISD::MOVLHPD:
4987 case X86ISD::MOVLPD:
4988 case X86ISD::MOVLPS:
4991 case X86ISD::VPERMIL2: {
5010 case X86ISD::VPPERM: {
5024 case X86ISD::VPERMV: {
5041 case X86ISD::VPERMV3: {
5464 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
5475 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
5698 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT,
5729 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, VecVT, V);
5805 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc);
5869 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5878 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5886 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5892 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6265 /// Try to fold a build_vector that performs an 'addsub' to an X86ISD::ADDSUB
6362 return DAG.getNode(X86ISD::ADDSUB, DL, VT, InVec0, InVec1);
6396 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6399 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6403 return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
6406 return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
6420 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6426 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6437 X86Opcode = X86ISD::HADD;
6442 X86Opcode = X86ISD::HSUB;
6470 X86Opcode = X86ISD::HADD;
6472 X86Opcode = X86ISD::HSUB;
6474 X86Opcode = X86ISD::FHADD;
6476 X86Opcode = X86ISD::FHSUB;
7348 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V1),
7371 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
7373 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
7378 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
7382 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
7424 ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND,
7454 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::ANDNP, DL, MaskVT,
7463 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
7526 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
7545 VT, DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
7557 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
7571 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
7802 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
7817 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
7819 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
7878 unsigned OpCode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI)
7879 : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI);
7973 return DAG.getNode(X86ISD::EXTRQI, DL, VT, Src,
8034 return DAG.getNode(X86ISD::INSERTQI, DL, VT, Base, Insert,
8106 InputV = DAG.getNode(X86ISD::VZEXT, DL, ExtVT, InputV);
8118 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
8125 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
8129 unsigned OddEvenOp = (Offset & 1 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW);
8144 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
8154 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
8158 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
8174 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
8191 unsigned UnpackLoHi = X86ISD::UNPCKL;
8193 UnpackLoHi = X86ISD::UNPCKH;
8330 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
8443 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL,
8451 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
8467 X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
8526 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
8549 unsigned Opcode = VT == MVT::v2f64 ? X86ISD::MOVDDUP : X86ISD::VBROADCAST;
8550 bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2();
8635 Opcode = (BroadcastVT.is128BitVector() ? X86ISD::MOVDDUP : Opcode);
8672 if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector())
8779 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
8852 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
8891 VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL,
8927 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
8931 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V1,
8959 isShuffleFoldableLoad(V1S) ? X86ISD::MOVLPD : X86ISD::MOVSD,
8974 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
9007 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
9021 return V.getOpcode() == X86ISD::PACKUS ? V : SDValue();
9028 DAG.getNode(X86ISD::PACKUS, DL, PackVT,
9144 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
9183 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
9195 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
9223 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
9225 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
9231 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
9237 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
9268 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
9270 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
9320 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
9438 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
9444 return SplatHalfs(LToLInputs[0], LToHInputs[0], X86ISD::PSHUFLW, 0);
9446 return SplatHalfs(HToLInputs[0], HToHInputs[0], X86ISD::PSHUFHW, 2);
9552 V = DAG.getNode(FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
9579 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
9815 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
9818 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
9823 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
9835 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
9843 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
9882 V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
9886 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
10199 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
10314 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
10317 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
10368 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
10370 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
10376 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
10688 SDValue Flipped = DAG.getNode(X86ISD::VPERM2X128, DL, VT, DAG.getUNDEF(VT),
10775 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
11144 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
11147 return DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
11178 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
11185 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
11191 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
11282 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
11289 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
11348 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
11350 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
11353 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
11380 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1,
11384 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32,
11444 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
11476 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32,
11754 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
11769 return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1);
11771 return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2);
11786 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
11795 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
11801 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
11838 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
11840 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
11843 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
11881 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
11888 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
11920 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
12328 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
12343 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
12412 Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec,
12414 Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec,
12416 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec,
12443 SDValue Mask = DAG.getNode(X86ISD::VINSERT, dl, MaskVT,
12446 SDValue Perm = DAG.getNode(X86ISD::VPERMV, dl, VecVT, Mask, Vec);
12488 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT, Vec, Idx);
12548 EltInVec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
12599 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, N2);
12624 Opc = X86ISD::PINSRW;
12627 Opc = X86ISD::PINSRB;
12643 // putting the '3' into bits [7:6] of X86ISD::INSERTPS.
12660 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, N2);
12665 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
12684 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
12803 // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
12815 unsigned WrapperKind = X86ISD::Wrapper;
12820 WrapperKind = X86ISD::WrapperRIP;
12831 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
12843 unsigned WrapperKind = X86ISD::Wrapper;
12848 WrapperKind = X86ISD::WrapperRIP;
12859 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
12872 unsigned WrapperKind = X86ISD::Wrapper;
12877 WrapperKind = X86ISD::WrapperRIP;
12889 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
12916 Result = DAG.getNode(X86ISD::WrapperRIP, dl, PtrVT, Result);
12918 Result = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, Result);
12923 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
12949 Result = DAG.getNode(X86ISD::WrapperRIP, dl, PtrVT, Result);
12951 Result = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, Result);
12956 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
12994 X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
12995 : X86ISD::TLSADDR;
13020 DAG.getNode(X86ISD::GlobalBaseReg,
13053 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
13064 unsigned WrapperKind = X86ISD::Wrapper;
13091 unsigned WrapperKind = X86ISD::Wrapper;
13097 WrapperKind = X86ISD::WrapperRIP;
13116 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
13164 X86ISD::WrapperRIP : X86ISD::Wrapper;
13182 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
13191 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
13271 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
13292 // X86ISD::SHLD and X86ISD::SHRD have defined overflow behavior but the
13303 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
13306 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
13315 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
13324 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
13325 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
13327 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
13328 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
13344 return DAG.getNode(X86ISD::CVTDQ2PD, dl, VT,
13414 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
13415 X86ISD::FILD, DL,
13438 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys,
13505 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
13508 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32,
13625 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
13633 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
13755 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
13851 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
13852 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
13853 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
13922 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
14006 return DAG.getNode(X86ISD::VZEXT, dl, VT, In);
14034 return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
14051 return DAG.getNode(X86ISD::VTRUNC, DL, VT, SelectedVal);
14103 return DAG.getNode(X86ISD::CVT2MASK, DL, VT, ShiftNode);
14117 return DAG.getNode(X86ISD::TESTM, DL, VT, ShiftNode, ShiftNode);
14144 return DAG.getNode(X86ISD::VTRUNC, DL, VT,
14145 DAG.getNode(X86ISD::VSEXT, DL, MVT::v16i32, In));
14146 return DAG.getNode(X86ISD::VTRUNC, DL, VT, In);
14188 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV);
14292 return DAG.getNode(X86ISD::VFPEXT, DL, VT,
14362 IsFABS ? X86ISD::FAND : IsFNABS ? X86ISD::FOR : X86ISD::FXOR;
14428 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Op1, Mask1);
14456 Val = DAG.getNode(X86ISD::FAND, dl, LogicVT, Op0, Val);
14459 Val = DAG.getNode(X86ISD::FOR, dl, LogicVT, Val, SignBit);
14474 // Lower ISD::FGETSIGN to (AND (X86ISD::MOVMSK ...) 1).
14477 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
14571 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
14607 return DAG.getNode(X86ISD::KTEST, SDLoc(Op), Op0VT, Op0, Op0);
14618 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp,
14661 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
14715 Opcode = X86ISD::INC;
14722 Opcode = X86ISD::DEC;
14729 Opcode = X86ISD::ADD;
14755 // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
14783 case ISD::SUB: Opcode = X86ISD::SUB; break;
14784 case ISD::XOR: Opcode = X86ISD::XOR; break;
14785 case ISD::AND: Opcode = X86ISD::AND; break;
14791 Opcode = X86ISD::OR;
14798 case X86ISD::ADD:
14799 case X86ISD::SUB:
14800 case X86ISD::INC:
14801 case X86ISD::DEC:
14802 case X86ISD::OR:
14803 case X86ISD::XOR:
14804 case X86ISD::AND:
14823 case ISD::ADD: ConvertedOp = X86ISD::ADD; break;
14824 case ISD::SUB: ConvertedOp = X86ISD::SUB; break;
14825 case ISD::AND: ConvertedOp = X86ISD::AND; break;
14826 case ISD::OR: ConvertedOp = X86ISD::OR; break;
14827 case ISD::XOR: ConvertedOp = X86ISD::XOR; break;
14846 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
14882 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs,
14886 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
14895 Cmp.getOpcode() != X86ISD::CMP ||
14906 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
14913 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
14946 return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
14977 return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
15050 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
15052 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
15196 case ISD::SETEQ: Opc = X86ISD::PCMPEQM; break;
15199 case ISD::SETGT: Opc = X86ISD::PCMPGTM; break;
15211 Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM;
15265 Opc = X86ISD::CMPM;
15267 Opc = X86ISD::CMPP;
15287 CombineOpc = Opc == X86ISD::CMPP ? static_cast<unsigned>(X86ISD::FOR) :
15293 CombineOpc = Opc == X86ISD::CMPP ? static_cast<unsigned>(X86ISD::FAND) :
15311 if (Opc == X86ISD::CMPP)
15393 ? X86ISD::VPCOMU : X86ISD::VPCOM;
15409 case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break;
15411 case ISD::SETGT: Opc = X86ISD::PCMPGT; break;
15413 case ISD::SETLE: Opc = X86ISD::PCMPGT;
15416 case ISD::SETUGT: Opc = X86ISD::PCMPGT;
15419 case ISD::SETULE: Opc = X86ISD::PCMPGT;
15467 Opc = X86ISD::SUBUS;
15478 if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) {
15500 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
15501 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
15519 if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) {
15560 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
15563 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
15606 if (Op0.getOpcode() == X86ISD::SETCC) {
15613 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
15642 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
15664 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry);
15665 SDValue SetCC = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
15678 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
15679 Opc == X86ISD::SAHF)
15682 (Opc == X86ISD::ADD ||
15683 Opc == X86ISD::SUB ||
15684 Opc == X86ISD::ADC ||
15685 Opc == X86ISD::SBB ||
15686 Opc == X86ISD::SMUL ||
15687 Opc == X86ISD::UMUL ||
15688 Opc == X86ISD::INC ||
15689 Opc == X86ISD::DEC ||
15690 Opc == X86ISD::OR ||
15691 Opc == X86ISD::XOR ||
15692 Opc == X86ISD::AND))
15695 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
15702 /// Like this: (i1 (trunc (i8 X86ISD::SETCC))).
15742 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1,
15744 return DAG.getNode(X86ISD::SELECT, DL, VT, Cmp, Op1, Op2);
15747 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
15782 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
15783 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
15784 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
15831 if (Cond.getOpcode() == X86ISD::SETCC &&
15832 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
15849 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, VTs,
15853 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
15859 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
15864 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
15878 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
15882 // If condition flag is set by a X86ISD::CMP, then use it as the condition
15883 // setting operand in place of the X86ISD::SETCC.
15885 if (CondOpcode == X86ISD::SETCC ||
15886 CondOpcode == X86ISD::SETCC_CARRY) {
15899 Opc == X86ISD::BT) { // FIXME
15913 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
15914 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
15915 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
15916 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
15917 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
15918 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
15962 if (Cond.getOpcode() == X86ISD::SUB) {
15969 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
15988 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VTs, T2, T1, CC, Cond);
15993 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
15997 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops);
16023 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16031 if (In.getOpcode() == X86ISD::VSEXT || In.getOpcode() == X86ISD::VZEXT)
16033 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16047 return DAG.getNode(X86ISD::VTRUNC, dl, VT, V);
16080 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16089 Curr = DAG.getNode(X86ISD::UNPCKL, dl, CurrVT, DAG.getUNDEF(CurrVT), Curr);
16099 SignExt = DAG.getNode(X86ISD::VSRAI, dl, CurrVT, Curr,
16107 SDValue Sign = DAG.getNode(X86ISD::VSRAI, dl, CurrVT, Curr,
16132 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
16161 OpLo = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpLo);
16162 OpHi = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpHi);
16320 // code. We generate X86ISD::VSEXT for SEXTLOADs if it's available, otherwise
16479 SDValue Sext = DAG.getNode(X86ISD::VSEXT, dl, RegVT, SlicedVec);
16508 /// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
16514 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16516 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
16520 /// Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the
16526 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16560 else if (Cond.getOpcode() == X86ISD::ADD ||
16561 Cond.getOpcode() == X86ISD::SUB ||
16562 Cond.getOpcode() == X86ISD::SMUL ||
16563 Cond.getOpcode() == X86ISD::UMUL)
16569 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
16573 // If condition flag is set by a X86ISD::CMP, then use it as the condition
16574 // setting operand in place of the X86ISD::SETCC.
16576 if (CondOpcode == X86ISD::SETCC ||
16577 CondOpcode == X86ISD::SETCC_CARRY) {
16583 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
16610 // instructions that can't be removed afterwards (i.e. X86ISD::ADD and
16611 // X86ISD::INC).
16613 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
16616 X86Opcode = X86ISD::INC; X86Cond = X86::COND_O;
16619 X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
16620 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
16623 X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O;
16626 X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
16627 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
16628 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
16659 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16690 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16731 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
16735 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16761 SDValue Cmp = DAG.getNode(X86ISD
16765 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16797 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16862 Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
16866 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Size);
16994 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl,
17047 if (Opc == X86ISD::VSRAI)
17053 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
17066 case X86ISD::VSHLI:
17078 case X86ISD::VSRLI:
17090 case X86ISD::VSRAI:
17127 case X86ISD::VSHLI: Opc = X86ISD::VSHL; break;
17128 case X86ISD::VSRLI: Opc = X86ISD::VSRL; break;
17129 case X86ISD::VSRAI: Opc = X86ISD::VSRA; break;
17233 case X86ISD::PCMPEQM:
17234 case X86ISD::PCMPGTM:
17235 case X86ISD::CMPM:
17236 case X86ISD::CMPMU:
17238 case X86ISD::VFPCLASS:
17239 case X86ISD::VFPCLASSS:
17241 case X86ISD::VTRUNC:
17242 case X86ISD::VTRUNCS:
17243 case X86ISD::VTRUNCUS:
17248 OpcodeSelect = X86ISD::SELECT;
17275 if (Op.getOpcode() == X86ISD::FSETCC)
17277 if (Op.getOpcode() == X86ISD::VFPCLASS ||
17278 Op.getOpcode() == X86ISD::VFPCLASSS)
17283 return DAG.getNode(X86ISD::SELECT, dl, VT, IMask, Op, PreservedSrc);
17776 SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17778 SDValue SetNP = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17785 SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17787 SDValue SetP = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17794 SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17798 SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17803 SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17807 SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17824 FCmp = DAG.getNode(X86ISD::FSETCC, dl, MVT::i1, LHS, RHS,
17827 FCmp = DAG.getNode(X86ISD::FSETCC, dl, MVT::i1, LHS, RHS,
17956 return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(),
18018 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
18021 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
18030 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
18031 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
18050 Opcode = X86ISD::PCMPISTRI;
18054 Opcode = X86ISD::PCMPESTRI;
18058 Opcode = X86ISD::PCMPISTRI;
18062 Opcode = X86ISD::PCMPESTRI;
18066 Opcode = X86ISD::PCMPISTRI;
18070 Opcode = X86ISD::PCMPESTRI;
18074 Opcode = X86ISD::PCMPISTRI;
18078 Opcode = X86ISD::PCMPESTRI;
18082 Opcode = X86ISD::PCMPISTRI;
18086 Opcode = X86ISD::PCMPESTRI;
18093 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18103 Opcode = X86ISD::PCMPISTRI;
18105 Opcode = X86ISD::PCMPESTRI;
18119 return DAG.getNode(X86ISD::Wrapper, dl, VT, DAG.getMCSymbol(S, PtrVT));
18133 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
18235 SDValue rd = DAG.getNode(X86ISD::RDPMC_DAG, DL, Tys, Chain);
18292 if (Opcode == X86ISD::RDTSCP_DAG) {
18326 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
18407 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl,
18467 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18478 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(2),
18485 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
18708 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
18725 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
18733 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
18740 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
18937 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
19062 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
19063 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
19144 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
19154 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
19197 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op.getOperand(0));
19206 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
19339 ALo = DAG.getNode(X86ISD::VSEXT, dl, ExVT, A);
19340 BLo = DAG.getNode(X86ISD::VSEXT, dl, ExVT, B);
19359 AHi = DAG.getNode(X86ISD::VSEXT, dl, ExVT, AHi);
19360 BHi = DAG.getNode(X86ISD::VSEXT, dl, ExVT, BHi);
19377 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
19391 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B);
19393 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds);
19418 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
19419 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
19431 SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
19434 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
19435 AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
19438 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
19439 AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
19468 unsigned ExSSE41 = (ISD::MULHU == Opcode ? X86ISD::VZEXT : X86ISD::VSEXT);
19496 return DAG.getNode(X86ISD::PACKUS, dl, VT,
19508 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
19557 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
19664 (!IsSigned || !Subtarget.hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
19761 unsigned X86Opc = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
19762 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
19772 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
19773 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
19782 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
19785 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
19824 SDValue CMP = DAG.getNode(X86ISD::PCMPGTM, dl, MVT::v64i1, Zeros, R);
19827 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
19836 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT,
19845 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT,
19931 unsigned X86OpcI = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
19932 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
19934 unsigned X86OpcV = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHL :
19935 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRL : X86ISD::VSRA;
20037 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
20039 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
20118 // lowered as X86ISD::VSRLI nodes. This would be cheaper than scalarizing
20122 unsigned TargetOpcode = X86ISD::MOVSS;
20139 TargetOpcode = X86ISD::MOVSD;
20150 TargetOpcode = X86ISD::MOVSD;
20170 if (TargetOpcode == X86ISD::MOVSD)
20199 Opc = X86ISD::VSHL;
20202 Opc = X86ISD::VSRL;
20205 Opc = X86ISD::VSRA;
20246 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
20283 SDValue ALo = DAG.getNode(X86ISD::UNPCKL, dl, VT, DAG.getUNDEF(VT), Amt);
20284 SDValue AHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, DAG.getUNDEF(VT), Amt);
20285 SDValue RLo = DAG.getNode(X86ISD::UNPCKL, dl, VT, DAG.getUNDEF(VT), R);
20286 SDValue RHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, DAG.getUNDEF(VT), R);
20331 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
20351 SDValue ALo = DAG.getNode(X86ISD::UNPCKL, dl, VT, Amt, Z);
20352 SDValue AHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, Amt, Z);
20353 SDValue RLo = DAG.getNode(X86ISD::UNPCKL, dl, VT, Z, R);
20354 SDValue RHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, Z, R);
20363 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
20464 return DAG.getNode(X86ISD::VPROTI, DL, VT, R,
20470 return DAG.getNode(X86ISD::VPROT, DL, VT, R, Amt);
20490 BaseOp = X86ISD::INC;
20494 BaseOp = X86ISD::ADD;
20498 BaseOp = X86ISD::ADD;
20505 BaseOp = X86ISD::DEC;
20509 BaseOp = X86ISD::SUB;
20513 BaseOp = X86ISD::SUB;
20517 BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL;
20522 BaseOp = X86ISD::UMUL8;
20528 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
20531 DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
20549 DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
20663 // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at
20700 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
20718 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
20746 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
20753 SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
20850 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
20861 SDValue Low = DAG.getNode(X86ISD::UNPCKL, DL, VT, V, Zeros);
20862 SDValue High = DAG.getNode(X86ISD::UNPCKH, DL, VT, V, Zeros);
20867 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
20869 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
20874 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
20941 DAG.getNode(X86ISD::PSHUFB, DL, ByteVecVT, InRegLUT, HighNibbles);
20943 DAG.getNode(X86ISD::PSHUFB, DL, ByteVecVT, InRegLUT, LowNibbles);
21113 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
21169 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
21170 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
21178 NewOpc = X86ISD::LADD;
21181 NewOpc = X86ISD::LSUB;
21184 NewOpc = X86ISD::LOR;
21187 NewOpc = X86ISD::LXOR;
21190 NewOpc = X86ISD::LAND;
21277 case ISD::ADDC: Opc = X86ISD::ADD; break;
21278 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
21279 case ISD::SUBC: Opc = X86ISD::SUB; break;
21280 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
21806 case X86ISD::AVG: {
21807 // Legalize types for X86ISD::AVG by expanding vectors.
21831 SDValue Res = DAG.getNode(X86ISD::AVG, dl, RegVT, InVec0, InVec1);
21837 case X86ISD::FMINC:
21838 case X86ISD::FMIN:
21839 case X86ISD::FMAXC:
21840 case X86ISD::FMAX: {
21901 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
21907 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
21924 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
21927 return getReadTimeStampCounter(N, dl, X86ISD::RDTSCP_DAG, DAG, Subtarget,
21939 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
21987 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_SAVE_RBX_DAG
21988 : X86ISD::LCMPXCHG8_SAVE_EBX_DAG;
21998 Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG;
22017 DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
22076 switch ((X86ISD::NodeType)Opcode) {
22077 case X86ISD::FIRST_NUMBER: break;
22078 case X86ISD::BSF: return "X86ISD::BSF";
22079 case X86ISD::BSR: return "X86ISD::BSR";
22080 case X86ISD::SHLD: return "X86ISD::SHLD";
22081 case X86ISD::SHRD: return "X86ISD::SHRD";
22082 case X86ISD::FAND: return "X86ISD::FAND";
22083 case X86ISD::FANDN: return "X86ISD::FANDN";
22084 case X86ISD::FOR: return "X86ISD::FOR";
22085 case X86ISD::FXOR: return "X86ISD::FXOR";
22086 case X86ISD::FILD: return "X86ISD::FILD";
22087 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
22088 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
22089 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
22090 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
22091 case X86ISD::FLD: return "X86ISD::FLD";
22092 case X86ISD::FST: return "X86ISD::FST";
22093 case X86ISD::CALL: return "X86ISD::CALL";
22094 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
22095 case X86ISD::RDTSCP_DAG: return "X86ISD::RDTSCP_DAG";
22096 case X86ISD::RDPMC_DAG: return "X86ISD::RDPMC_DAG";
22097 case X86ISD::BT: return "X86ISD::BT";
22098 case X86ISD::CMP: return "X86ISD::CMP";
22099 case X86ISD::COMI: return "X86ISD::COMI";
22100 case X86ISD::UCOMI: return "X86ISD::UCOMI";
22101 case X86ISD::CMPM: return "X86ISD::CMPM";
22102 case X86ISD::CMPMU: return "X86ISD::CMPMU";
22103 case X86ISD::CMPM_RND: return "X86ISD::CMPM_RND";
22104 case X86ISD::SETCC: return "X86ISD::SETCC";
22105 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
22106 case X86ISD::FSETCC: return "X86ISD::FSETCC";
22107 case X86ISD::CMOV: return "X86ISD::CMOV";
22108 case X86ISD::BRCOND: return "X86ISD::BRCOND";
22109 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
22110 case X86ISD::IRET: return "X86ISD::IRET";
22111 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
22112 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
22113 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
22114 case X86ISD::Wrapper: return "X86ISD::Wrapper";
22115 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
22116 case X86ISD::MOVDQ2Q: return "X86ISD::MOVDQ2Q";
22117 case X86ISD::MMX_MOVD2W: return "X86ISD::MMX_MOVD2W";
22118 case X86ISD::MMX_MOVW2D: return "X86ISD::MMX_MOVW2D";
22119 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
22120 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
22121 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
22122 case X86ISD::PINSRB: return "X86ISD::PINSRB";
22123 case X86ISD::PINSRW: return "X86ISD::PINSRW";
22124 case X86ISD::MMX_PINSRW: return "X86ISD::MMX_PINSRW";
22125 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
22126 case X86ISD::ANDNP: return "X86ISD::ANDNP";
22127 case X86ISD::BLENDI: return "X86ISD::BLENDI";
22128 case X86ISD::SHRUNKBLEND: return "X86ISD::SHRUNKBLEND";
22129 case X86ISD::ADDUS: return "X86ISD::ADDUS";
22130 case X86ISD::SUBUS: return "X86ISD::SUBUS";
22131 case X86ISD::HADD: return "X86ISD::HADD";
22132 case X86ISD::HSUB: return "X86ISD::HSUB";
22133 case X86ISD::FHADD: return "X86ISD::FHADD";
22134 case X86ISD::FHSUB: return "X86ISD::FHSUB";
22135 case X86ISD::ABS: return "X86ISD::ABS";
22136 case X86ISD::CONFLICT: return "X86ISD::CONFLICT";
22137 case X86ISD::FMAX: return "X86ISD::FMAX";
22138 case X86ISD::FMAX_RND: return "X86ISD::FMAX_RND";
22139 case X86ISD::FMIN: return "X86ISD::FMIN";
22140 case X86ISD::FMIN_RND: return "X86ISD::FMIN_RND";
22141 case X86ISD::FMAXC: return "X86ISD::FMAXC";
22142 case X86ISD::FMINC: return "X86ISD::FMINC";
22143 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
22144 case X86ISD::FRSQRTS: return "X86ISD::FRSQRTS";
22145 case X86ISD::FRCP: return "X86ISD::FRCP";
22146 case X86ISD::FRCPS: return "X86ISD::FRCPS";
22147 case X86ISD::EXTRQI: return "X86ISD::EXTRQI";
22148 case X86ISD::INSERTQI: return "X86ISD::INSERTQI";
22149 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
22150 case X86ISD::TLSBASEADDR: return "X86ISD::TLSBASEADDR";
22151 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
22152 case X86ISD::EH_SJLJ_SETJMP: return "X86ISD::EH_SJLJ_SETJMP";
22153 case X86ISD::EH_SJLJ_LONGJMP: return "X86ISD::EH_SJLJ_LONGJMP";
22154 case X86ISD::EH_SJLJ_SETUP_DISPATCH:
22155 return "X86ISD::EH_SJLJ_SETUP_DISPATCH";
22156 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
22157 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
22158 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
22159 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r";
22160 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
22161 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
22162 case X86ISD::LCMPXCHG16_DAG: return "X86ISD::LCMPXCHG16_DAG";
22163 case X86ISD::LCMPXCHG8_SAVE_EBX_DAG:
22164 return "X86ISD::LCMPXCHG8_SAVE_EBX_DAG";
22165 case X86ISD::LCMPXCHG16_SAVE_RBX_DAG:
22166 return "X86ISD::LCMPXCHG16_SAVE_RBX_DAG";
22167 case X86ISD::LADD: return "X86ISD::LADD";
22168 case X86ISD::LSUB: return "X86ISD::LSUB";
22169 case X86ISD::LOR: return "X86ISD::LOR";
22170 case X86ISD::LXOR: return "X86ISD::LXOR";
22171 case X86ISD::LAND: return "X86ISD::LAND";
22172 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
22173 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
22174 case X86ISD::VZEXT: return "X86ISD::VZEXT";
22175 case X86ISDX86ISD::VSEXT";
22176 case X86ISD::VTRUNC: return "X86ISD::VTRUNC";
22177 case X86ISD::VTRUNCS: return "X86ISD::VTRUNCS";
22178 case X86ISD::VTRUNCUS: return "X86ISD::VTRUNCUS";
22179 case X86ISD::VINSERT: return "X86ISD::VINSERT";
22180 case X86ISD::VFPEXT: return "X86ISD::VFPEXT";
22181 case X86ISD::VFPROUND: return "X86ISD::VFPROUND";
22182 case X86ISD::CVTDQ2PD: return "X86ISD::CVTDQ2PD";
22183 case X86ISD::CVTUDQ2PD: return "X86ISD::CVTUDQ2PD";
22184 case X86ISD::CVT2MASK: return "X86ISD::CVT2MASK";
22185 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ";
22186 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ";
22187 case X86ISD::VSHL: return "X86ISD::VSHL";
22188 case X86ISD::VSRL: return "X86ISD::VSRL";
22189 case X86ISD::VSRA: return "X86ISD::VSRA";
22190 case X86ISD::VSHLI: return "X86ISD::VSHLI";
22191 case X86ISD::VSRLI: return "X86ISD::VSRLI";
22192 case X86ISD::VSRAI: return "X86ISD::VSRAI";
22193 case X86ISD::VSRAV: return "X86ISD::VSRAV";
22194 case X86ISD::VROTLI: return "X86ISD::VROTLI";
22195 case X86ISD::VROTRI: return "X86ISD::VROTRI";
22196 case X86ISD::VPPERM: return "X86ISD::VPPERM";
22197 case X86ISD::CMPP: return "X86ISD::CMPP";
22198 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ";
22199 case X86ISD::PCMPGT: return "X86ISD::PCMPGT";
22200 case X86ISD::PCMPEQM: return "X86ISD::PCMPEQM";
22201 case X86ISD::PCMPGTM: return "X86ISD::PCMPGTM";
22202 case X86ISD::ADD: return "X86ISD::ADD";
22203 case X86ISD::SUB: return "X86ISD::SUB";
22204 case X86ISD::ADC: return "X86ISD::ADC";
22205 case X86ISD::SBB: return "X86ISD::SBB";
22206 case X86ISD::SMUL: return "X86ISD::SMUL";
22207 case X86ISD::UMUL: return "X86ISD::UMUL";
22208 case X86ISD::SMUL8: return "X86ISD::SMUL8";
22209 case X86ISD::UMUL8: return "X86ISD::UMUL8";
22210 case X86ISD::SDIVREM8_SEXT_HREG: return "X86ISD::SDIVREM8_SEXT_HREG";
22211 case X86ISD::UDIVREM8_ZEXT_HREG: return "X86ISD::UDIVREM8_ZEXT_HREG";
22212 case X86ISD::INC: return "X86ISD::INC";
22213 case X86ISD::DEC: return "X86ISD::DEC";
22214 case X86ISD::OR: return "X86ISD::OR";
22215 case X86ISD::XOR: return "X86ISD::XOR";
22216 case X86ISD::AND: return "X86ISD::AND";
22217 case X86ISD::BEXTR: return "X86ISD::BEXTR";
22218 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
22219 case X86ISD::MOVMSK: return "X86ISD::MOVMSK";
22220 case X86ISD::PTEST: return "X86ISD::PTEST";
22221 case X86ISD::TESTP: return "X86ISD::TESTP";
22222 case X86ISD::TESTM: return "X86ISD::TESTM";
22223 case X86ISD::TESTNM: return "X86ISD::TESTNM";
22224 case X86ISD::KORTEST: return "X86ISD::KORTEST";
22225 case X86ISD::KTEST: return "X86ISD::KTEST";
22226 case X86ISD::PACKSS: return "X86ISD::PACKSS";
22227 case X86ISD::PACKUS: return "X86ISD::PACKUS";
22228 case X86ISD::PALIGNR: return "X86ISD::PALIGNR";
22229 case X86ISD::VALIGN: return "X86ISD::VALIGN";
22230 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
22231 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
22232 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
22233 case X86ISD::SHUFP: return "X86ISD::SHUFP";
22234 case X86ISD::SHUF128: return "X86ISD::SHUF128";
22235 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
22236 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD";
22237 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
22238 case X86ISD::MOVLPS: return "X86ISD::MOVLPS";
22239 case X86ISD::MOVLPD: return "X86ISD::MOVLPD";
22240 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
22241 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
22242 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
22243 case X86ISD::MOVSD: return "X86ISD::MOVSD";
22244 case X86ISD::MOVSS: return "X86ISD::MOVSS";
22245 case X86ISD::UNPCKL: return "X86ISD::UNPCKL";
22246 case X86ISD::UNPCKH: return "X86ISD::UNPCKH";
22247 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
22248 case X86ISD::VBROADCASTM: return "X86ISD::VBROADCASTM";
22249 case X86ISD::SUBV_BROADCAST: return "X86ISD::SUBV_BROADCAST";
22250 case X86ISD::VEXTRACT: return "X86ISD::VEXTRACT";
22251 case X86ISD::VPERMILPV: return "X86ISD::VPERMILPV";
22252 case X86ISD::VPERMILPI: return "X86ISD::VPERMILPI";
22253 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128";
22254 case X86ISD::VPERMV: return "X86ISD::VPERMV";
22255 case X86ISD::VPERMV3: return "X86ISD::VPERMV3";
22256 case X86ISD::VPERMIV3: return "X86ISD::VPERMIV3";
22257 case X86ISD::VPERMI: return "X86ISD::VPERMI";
22258 case X86ISD::VPTERNLOG: return "X86ISD::VPTERNLOG";
22259 case X86ISD::VFIXUPIMM: return "X86ISD::VFIXUPIMM";
22260 case X86ISD::VFIXUPIMMS: return "X86ISD::VFIXUPIMMS";
22261 case X86ISD::VRANGE: return "X86ISD::VRANGE";
22262 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ";
22263 case X86ISD::PMULDQ: return "X86ISD::PMULDQ";
22264 case X86ISD::PSADBW: return "X86ISD::PSADBW";
22265 case X86ISD::DBPSADBW: return "X86ISD::DBPSADBW";
22266 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
22267 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
22268 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
22269 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
22270 case X86ISD::MFENCE: return "X86ISD::MFENCE";
22271 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
22272 case X86ISD::SAHF: return "X86ISD::SAHF";
22273 case X86ISD::RDRAND: return "X86ISD::RDRAND";
22274 case X86ISD::RDSEED: return "X86ISD::RDSEED";
22275 case X86ISD::VPMADDUBSW: return "X86ISD::VPMADDUBSW";
22276 case X86ISD::VPMADDWD: return "X86ISD::VPMADDWD";
22277 case X86ISD::VPROT: return "X86ISD::VPROT";
22278 case X86ISD::VPROTI: return "X86ISD::VPROTI";
22279 case X86ISD::VPSHA: return "X86ISD::VPSHA";
22280 case X86ISD::VPSHL: return "X86ISD::VPSHL";
22281 case X86ISD::VPCOM: return "X86ISD::VPCOM";
22282 case X86ISD::VPCOMU: return "X86ISD::VPCOMU";
22283 case X86ISD::VPERMIL2: return "X86ISD::VPERMIL2";
22284 case X86ISD::FMADD: return "X86ISD::FMADD";
22285 case X86ISD::FMSUB: return "X86ISD::FMSUB";
22286 case X86ISD::FNMADD: return "X86ISD::FNMADD";
22287 case X86ISD::FNMSUB: return "X86ISD::FNMSUB";
22288 case X86ISD::FMADDSUB: return "X86ISD::FMADDSUB";
22289 case X86ISD::FMSUBADD: return "X86ISD::FMSUBADD";
22290 case X86ISD::FMADD_RND: return "X86ISD::FMADD_RND";
22291 case X86ISD::FNMADD_RND: return "X86ISD::FNMADD_RND";
22292 case X86ISD::FMSUB_RND: return "X86ISD::FMSUB_RND";
22293 case X86ISD::FNMSUB_RND: return "X86ISD::FNMSUB_RND";
22294 case X86ISD::FMADDSUB_RND: return "X86ISD::FMADDSUB_RND";
22295 case X86ISD::FMSUBADD_RND: return "X86ISD::FMSUBADD_RND";
22296 case X86ISD::VPMADD52H: return "X86ISD::VPMADD52H";
22297 case X86ISD::VPMADD52L: return "X86ISD::VPMADD52L";
22298 case X86ISD::VRNDSCALE: return "X86ISD::VRNDSCALE";
22299 case X86ISD::VREDUCE: return "X86ISD::VREDUCE";
22300 case X86ISD::VGETMANT: return "X86ISD::VGETMANT";
22301 case X86ISD::PCMPESTRI: return "X86ISD::PCMPESTRI";
22302 case X86ISD::PCMPISTRI: return "X86ISD::PCMPISTRI";
22303 case X86ISD::XTEST: return "X86ISD::XTEST";
22304 case X86ISD::COMPRESS: return "X86ISD::COMPRESS";
22305 case X86ISD::EXPAND: return "X86ISD::EXPAND";
22306 case X86ISD::SELECT: return "X86ISD::SELECT";
22307 case X86ISD::ADDSUB: return "X86ISD::ADDSUB";
22308 case X86ISD::RCP28: return "X86ISD::RCP28";
22309 case X86ISD::EXP2: return "X86ISD::EXP2";
22310 case X86ISD::RSQRT28: return "X86ISD::RSQRT28";
22311 case X86ISD::FADD_RND: return "X86ISD::FADD_RND";
22312 case X86ISD::FSUB_RND: return "X86ISD::FSUB_RND";
22313 case X86ISD::FMUL_RND: return "X86ISD::FMUL_RND";
22314 case X86ISD::FDIV_RND: return "X86ISD::FDIV_RND";
22315 case X86ISD::FSQRT_RND: return "X86ISD::FSQRT_RND";
22316 case X86ISD::FGETEXP_RND: return "X86ISD::FGETEXP_RND";
22317 case X86ISD::SCALEF: return "X86ISD::SCALEF";
22318 case X86ISD::SCALEFS: return "X86ISD::SCALEFS";
22319 case X86ISD::ADDS: return "X86ISD::ADDS";
22320 case X86ISD::SUBS: return "X86ISD::SUBS";
22321 case X86ISD::AVG: return "X86ISD::AVG";
22322 case X86ISD::MULHRS: return "X86ISD::MULHRS";
22323 case X86ISD::SINT_TO_FP_RND: return "X86ISD::SINT_TO_FP_RND";
22324 case X86ISD::UINT_TO_FP_RND: return "X86ISD::UINT_TO_FP_RND";
22325 case X86ISD::FP_TO_SINT_RND: return "X86ISD::FP_TO_SINT_RND";
22326 case X86ISD::FP_TO_UINT_RND: return "X86ISD::FP_TO_UINT_RND";
22327 case X86ISD::VFPCLASS: return "X86ISD::VFPCLASS";
22328 case X86ISD::VFPCLASSS: return "X86ISD::VFPCLASSS";
22329 case X86ISD::MULTISHIFT: return "X86ISD::MULTISHIFT";
22330 case X86ISD::SCALAR_FP_TO_SINT_RND: return "X86ISD::SCALAR_FP_TO_SINT_RND";
22331 case X86ISD::SCALAR_FP_TO_UINT_RND: return "X86ISD::SCALAR_FP_TO_UINT_RND";
24617 case X86ISD::ADD:
24618 case X86ISD::SUB:
24619 case X86ISD::ADC:
24620 case X86ISD::SBB:
24621 case X86ISD::SMUL:
24622 case X86ISD::UMUL:
24623 case X86ISD::INC:
24624 case X86ISD::DEC:
24625 case X86ISD::OR:
24626 case X86ISD::XOR:
24627 case X86ISD::AND:
24632 case X86ISD::SETCC:
24635 case X86ISD::MOVMSK: {
24648 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
24660 if (N->getOpcode() == X86ISD::Wrapper) {
24715 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
24759 Shuffle = X86ISD::VZEXT_MOVL;
24769 Shuffle = X86ISD::MOVDDUP;
24774 Shuffle = X86ISD::MOVSLDUP;
24779 Shuffle = X86ISD::MOVSHDUP;
24788 Shuffle = X86ISD::MOVDDUP;
24793 Shuffle = X86ISD::MOVSLDUP;
24798 Shuffle = X86ISD::MOVSHDUP;
24808 Shuffle = X86ISD::MOVDDUP;
24814 Shuffle = X86ISD::MOVSLDUP;
24820 Shuffle = X86ISD::MOVSHDUP;
24835 Shuffle = X86ISD::VBROADCAST;
24871 Shuffle = X86ISD::PSHUFLW;
24885 Shuffle = X86ISD::PSHUFHW;
24914 Shuffle = X86ISD::VPERMI;
24922 Shuffle = X86ISD::VPERMI;
24933 Shuffle = X86ISD::VPERMILPI;
24956 Shuffle = (FloatDomain ? X86ISD::VPERMILPI : X86ISD::PSHUFD);
24972 Shuffle = X86ISD::MOVLHPS;
24977 Shuffle = X86ISD::MOVHLPS;
24982 Shuffle = X86ISD::UNPCKL;
24987 Shuffle = X86ISD::UNPCKH;
24994 Shuffle = X86ISD::UNPCKL;
25001 Shuffle = X86ISD::UNPCKH;
25064 if (Depth == 1 && Root.getOpcode() == X86ISD::VPERM2X128)
25074 Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res,
25151 // Convert VT to a type compatible with X86ISD::BLENDI.
25175 if (Depth == 1 && Root.getOpcode() == X86ISD::BLENDI)
25185 Res = DAG.getNode(X86ISD::BLENDI, DL, ShuffleVT, Res, Zero,
25206 if (Depth == 1 && Root.getOpcode() == X86ISD::INSERTPS)
25212 Res = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
25249 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
25287 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
25481 case X86ISD::PSHUFD:
25483 case X86ISD::PSHUFLW:
25486 case X86ISD::PSHUFHW:
25505 assert(N.getOpcode() == X86ISD::PSHUFD &&
25524 case X86ISD::PSHUFD:
25528 case X86ISD::PSHUFLW:
25538 case X86ISD::PSHUFHW:
25548 case X86ISD::UNPCKL:
25549 case X86ISD::UNPCKH:
25558 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
25569 case X86ISD::PSHUFLW:
25570 case X86ISD::PSHUFHW:
25611 case X86ISD::UNPCKL:
25612 case X86ISD::UNPCKH:
25616 case X86ISD::PSHUFD:
25617 case X86ISD::PSHUFLW:
25618 case X86ISD::PSHUFHW:
25640 (N.getOpcode() == X86ISD::PSHUFLW || N.getOpcode() == X86ISD::PSHUFHW) &&
25657 case X86ISD::PSHUFLW:
25658 case X86ISD::PSHUFHW:
25707 case X86ISD::PSHUFD:
25708 case X86ISD::PSHUFLW:
25709 case X86ISD::PSHUFHW:
25713 case X86ISD::UNPCKL: {
25714 // Combine X86ISD::UNPCKL and ISD::VECTOR_SHUFFLE into X86ISD::UNPCKH, in
25715 // which X86ISD::UNPCKL has a ISD::UNDEF operand, and ISD::VECTOR_SHUFFLE
25720 X86ISD::UNPCKL undef:v16i8, t2
25724 // t3: v16i8 = X86ISD::UNPCKH undef:v16i8, t1
25743 return DAG.getNode(X86ISD::UNPCKH, DL, VT, N.getOperand(0), ShufOp);
25747 case X86ISD::BLENDI: {
25767 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V0, NewMask);
25771 if (V0.getOpcode() == X86ISD::INSERTPS ||
25772 V1.getOpcode() == X86ISD::INSERTPS) {
25784 if (V.getOpcode() != X86ISD::INSERTPS)
25799 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, Op0, Op1,
25810 case X86ISD::INSERTPS: {
25822 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
25827 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
25838 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
25845 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
25890 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
25908 case X86ISD::PSHUFLW:
25909 case X86ISD::PSHUFHW:
25920 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
25926 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
25936 (V.getOpcode() == X86ISD::PSHUFLW ||
25937 V.getOpcode() == X86ISD::PSHUFHW) &&
25943 if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
25946 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
25947 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
25962 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
25963 : X86ISD::UNPCKH,
25971 case X86ISD::PSHUFD:
26036 return DAG.getNode(X86ISD::ADDSUB, DL, VT, LHS, RHS);
26265 return DAG.getNode(X86ISD::MMX_MOVW2D, SDLoc(N00), VT, N00);
26276 case ISD::AND: FPOpcode = X86ISD::FAND; break;
26277 case ISD::OR: FPOpcode = X86ISD::FOR; break;
26278 case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
26315 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, MMXSrc);
26473 Opcode = X86ISD::FMIN;
26481 Opcode = X86ISD::FMIN;
26490 Opcode = X86ISD::FMIN;
26499 Opcode = X86ISD::FMAX;
26511 Opcode = X86ISD::FMAX;
26520 Opcode = X86ISD::FMAX;
26538 Opcode = X86ISD::FMIN;
26545 Opcode = X86ISD::FMIN;
26554 Opcode = X86ISD::FMIN;
26561 Opcode = X86ISD::FMAX;
26573 Opcode = X86ISD::FMAX;
26582 Opcode = X86ISD::FMAX;
26765 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
26778 X86ISD::SUBUS, DL, VT, OpLHS,
26792 X86ISD::SUBUS, DL, VT, OpLHS,
26915 DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(*I), I->getValueType(0),
26925 DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(N), N->getValueType(0),
26943 if (!(Cmp.getOpcode() == X86ISD::CMP ||
26944 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
27002 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
27018 if (!(Cmp.getOpcode() == X86ISD::CMP ||
27019 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
27071 case X86ISD::SETCC_CARRY:
27081 case X86ISD::SETCC:
27087 case X86ISD::CMOV: {
27103 if ((Op.getOpcode() != X86ISD::RDRAND &&
27104 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
27138 if (Cond->getOpcode() == X86ISD::CMP) {
27151 case X86ISD::AND:
27155 case X86ISD::OR:
27162 if (SetCC0.getOpcode() != X86ISD::SETCC ||
27163 SetCC1.getOpcode() != X86ISD::SETCC ||
27183 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
27201 case X86ISD::BSR:
27202 case X86ISD::BSF:
27215 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
27236 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
27253 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
27291 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
27332 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
27346 return DAG.getNode(X86ISD
27381 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), LOps);
27383 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
27585 SDValue Res = DAG.getNode(X86ISD::UNPCKL, DL, OpsVT, MulLo, MulHi);
27648 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
27655 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
27712 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
27715 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
27719 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
27866 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
27908 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00,
27916 SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
27991 return DAG.getNode(X86ISD::ANDNP, DL, VT, N00, N1);
28172 FPOpcode = X86ISD::FAND;
28174 FPOpcode = X86ISD::FOR;
28176 FPOpcode = X86ISD::FXOR;
28211 if (Op0.getOpcode() != X86ISD::PCMPEQ && Op0.getOpcode() != X86ISD::PCMPGT)
28232 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
28280 return DAG.getNode(X86ISD::BEXTR, DL, VT, N0.getOperand(0),
28309 if (N0.getOpcode() == X86ISD::ANDNP)
28312 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP)
28343 } else if (Mask.getOpcode() == X86ISD::VSRAI) {
28467 unsigned Opc = X86ISD::SHLD;
28471 Opc = X86ISD::SHRD;
28525 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
28531 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops);
28633 return DAG.getNode(X86ISD::PCMPGT, SDLoc(N), VT, Shift.getOperand(0), Ones);
28660 /// X86ISD::AVG instruction.
28743 // demote it and emit X86ISD::AVG instruction.
28747 return DAG.getNode(X86ISD::AVG, DL, VT, Operands[0].getOperand(0),
28771 // The pattern is detected, emit X86ISD::AVG instruction.
28772 return DAG.getNode(X86ISD::AVG, DL, VT, Operands[0].getOperand(0),
29047 SDValue NewVec = DAG.getNode(X86ISD::VSEXT, dl, VT, WideLd);
29210 // replace the trunc store by a normal store with the result of X86ISD::AVG
29565 auto NewOpcode = IsFadd ? X86ISD::FHADD : X86ISD::FHSUB;
29571 /// Truncate a group of v4i32 into v16i8/v8i16 using X86ISD::PACKUS.
29608 Regs[i] = DAG.getNode(X86ISD::PACKUS, DL, PackedVT, Regs[i * 2],
29612 // If the type of the result is v8i8, we need do one more X86ISD::PACKUS, and
29615 Regs[0] = DAG.getNode(X86ISD::PACKUS, DL, PackedVT, Regs[0], Regs[0]);
29626 /// Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS.
29637 Reg = getTargetVShiftNode(X86ISD::VSHLI, DL, MVT::v4i32, Reg, ShAmt, DAG);
29638 Reg = getTargetVShiftNode(X86ISD::VSRAI, DL, MVT::v4i32, Reg, ShAmt, DAG);
29642 Regs[i] = DAG.getNode(X86ISD::PACKSS, DL, MVT::v8i16, Regs[i * 2],
29653 /// X86ISD::PACKUS/X86ISD::PACKSS operations. We do it here because after type
29670 // TODO: On AVX2, the behavior of X86ISD::PACKUS is different from that on
29727 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
29751 return DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
29759 case X86ISD::FMADD:
29760 return DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
29762 case X86ISD::FMSUB:
29763 return DAG.getNode(X86ISD::FNMADD, DL, VT, Arg.getOperand(0),
29765 case X86ISD::FNMADD:
29766 return DAG.getNode(X86ISD::FMSUB, DL, VT, Arg.getOperand(0),
29768 case X86ISD::FNMSUB:
29769 return DAG.getNode(X86ISD::FMADD, DL, VT, Arg.getOperand(0),
29791 case X86ISD::FOR: IntOpcode = ISD::OR; break;
29792 case X86ISD::FXOR: IntOpcode = ISD::XOR; break;
29793 case X86ISD::FAND: IntOpcode = ISD::AND; break;
29794 case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
29801 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
29804 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
29819 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
29821 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
29832 case X86ISDX86ISD::FMINC; break;
29833 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
29886 auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN;
29896 /// Do target-specific dag combines on X86ISD::FAND nodes.
29912 /// Do target-specific dag combines on X86ISD::FANDN nodes
29949 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
29978 // it may be replaced with X86ISD::VSEXT node.
30061 auto DivRemOpcode = OpcodeN0 == ISD::SDIVREM ? X86ISD::SDIVREM8_SEXT_HREG
30062 : X86ISD::UDIVREM8_ZEXT_HREG;
30129 // ISD::*_EXTEND_VECTOR_INREG which ensures lowering to X86ISD::V*EXT.
30229 Opcode = (!NegC) ? X86ISD::FMADD : X86ISD::FMSUB;
30231 Opcode = (!NegC) ? X86ISD::FNMADD : X86ISD::FNMSUB;
30239 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
30240 // (and (i32 x86isd::setcc_carry), 1)
30251 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
30255 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
30265 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
30267 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
30344 // For an SSE1-only target, lower to X86ISD::CMPP early to avoid scalarization
30373 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
30379 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
30384 // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
30399 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
30402 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
30419 return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
30438 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
30561 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
30576 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
30600 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse())
30608 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
30614 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
30619 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
30623 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
30738 SDValue Sad = DAG.getNode(X86ISD::PSADBW, DL, SadVT, Op0, Op1);
30778 return DAG.getNode(X86ISD::HADD, SDLoc(N), VT, Op0, Op1);
30811 return DAG.getNode(X86ISD::HSUB, SDLoc(N), VT, Op0, Op1);
30846 if (V != Op && V.getOpcode() == X86ISD::VZEXT) {
30854 return DAG.getNode(X86ISD::VZEXT, DL, VT, V.getOperand(0));
30865 return DAG.getNode(X86ISD::VZEXT, DL, VT, DAG.getBitcast(OpVT, V));
30887 return DAG.getNode(X86ISD::VZEXT, DL, VT, Op);
30909 return DAG.getMemIntrinsicNode(X86ISD::LADD, DL,
30925 return DAG.getNode(X86ISD::TESTM, DL, VT,
30935 if (N->getOpcode() == X86ISD::PCMPEQ)
30937 if (N->getOpcode() == X86ISD::PCMPGT)
30953 case X86ISD::SHRUNKBLEND: return combineSelect(N, DAG, DCI, Subtarget);
30955 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
30958 case X86ISD::ADC: return combineADC(N, DAG, DCI);
30976 case X86ISD::FXOR:
30977 case X86ISD::FOR: return combineFOr(N, DAG, Subtarget);
30978 case X86ISD::FMIN:
30979 case X86ISD::FMAX: return combineFMinFMax(N, DAG);
30982 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget);
30983 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget);
30984 case X86ISD::BT: return combineBT(N, DAG, DCI);
30985 case X86ISD::VZEXT_MOVL: return combineVZextMovl(N, DAG);
30991 case X86ISD::SETCC: return combineX86SetCC(N, DAG, DCI, Subtarget);
30992 case X86ISD::BRCOND: return combineBrCond(N, DAG, DCI, Subtarget);
30993 case X86ISD::VZEXT: return combineVZext(N, DAG, DCI, Subtarget);
30994 case X86ISD::SHUFP: // Handle all target specific shuffles
30995 case X86ISD::INSERTPS:
30996 case X86ISD::PALIGNR:
30997 case X86ISD::VSHLDQ:
30998 case X86ISD::VSRLDQ:
30999 case X86ISD::BLENDI:
31000 case X86ISD::UNPCKH:
31001 case X86ISD::UNPCKL:
31002 case X86ISD::MOVHLPS:
31003 case X86ISD::MOVLHPS:
31004 case X86ISD::PSHUFB:
31005 case X86ISD::PSHUFD:
31006 case X86ISD::PSHUFHW:
31007 case X86ISD::PSHUFLW:
31008 case X86ISD::MOVSHDUP:
31009 case X86ISD::MOVSLDUP:
31010 case X86ISD::MOVDDUP:
31011 case X86ISD::MOVSS:
31012 case X86ISD::MOVSD:
31013 case X86ISD::VPPERM:
31014 case X86ISD::VPERMV:
31015 case X86ISD::VPERMV3:
31016 case X86ISD::VPERMIL2:
31017 case X86ISD::VPERMILPI:
31018 case X86ISD::VPERMILPV:
31019 case X86ISD::VPERM2X128:
31024 case X86ISD::LSUB: return combineLockSub(N, DAG, Subtarget);
31025 case X86ISD::TESTM: return combineTestM(N, DAG);
31026 case X86ISD::PCMPEQ:
31027 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget);
