Lines Matching refs:X86ISD
2044 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
2473 X86ISD::NodeType opcode = X86ISD::RET_FLAG;
2475 opcode = X86ISD::IRET;
2497 if (UI->getOpcode() != X86ISD::RET_FLAG)
3124 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
3268 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
3674 unsigned(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
3830 Callee = DAG.getNode(X86ISD::WrapperRIP, dl,
3847 Callee = DAG.getNode(X86ISD::WrapperRIP, dl,
3943 return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
3947 Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops);
3949 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
4360 case X86ISD::BLENDI:
4361 case X86ISD::PSHUFB:
4362 case X86ISD::PSHUFD:
4363 case X86ISD::PSHUFHW:
4364 case X86ISD::PSHUFLW:
4365 case X86ISD::SHUFP:
4366 case X86ISD::INSERTPS:
4367 case X86ISD::EXTRQI:
4368 case X86ISD::INSERTQI:
4369 case X86ISD::PALIGNR:
4370 case X86ISD::VSHLDQ:
4371 case X86ISD::VSRLDQ:
4372 case X86ISD::MOVLHPS:
4373 case X86ISD::MOVHLPS:
4374 case X86ISD::MOVSHDUP:
4375 case X86ISD::MOVSLDUP:
4376 case X86ISD::MOVDDUP:
4377 case X86ISD::MOVSS:
4378 case X86ISD::MOVSD:
4379 case X86ISD::UNPCKL:
4380 case X86ISD::UNPCKH:
4381 case X86ISD::VBROADCAST:
4382 case X86ISD::VPERMILPI:
4383 case X86ISD::VPERMILPV:
4384 case X86ISD::VPERM2X128:
4385 case X86ISD::SHUF128:
4386 case X86ISD::VPERMIL2:
4387 case X86ISD::VPERMI:
4388 case X86ISD::VPPERM:
4389 case X86ISD::VPERMV:
4390 case X86ISD::VPERMV3:
4391 case X86ISD::VZEXT_MOVL:
4400 case X86ISD::PSHUFB:
4401 case X86ISD::VPERMILPV:
4402 case X86ISD::VPERMIL2:
4403 case X86ISD::VPPERM:
4404 case X86ISD::VPERMV:
4405 case X86ISD::VPERMV3:
4409 case X86ISD::ANDNP:
4671 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
5248 case X86ISD::CMPM:
5249 case X86ISD::CMPM_RND:
5309 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
5310 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
5324 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5334 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5337 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
5344 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5359 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
5360 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
5373 Op = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
5379 Op = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Op,
5383 Op = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Op,
5414 assert((X86ISD::VSEXT == Opc || X86ISD::VZEXT == Opc) && "Unexpected opcode");
5417 return X86ISD::VSEXT == Opc ? DAG.getSignExtendVectorInReg(In, DL, VT)
5494 if (Ptr->getOpcode() == X86ISD::Wrapper ||
5495 Ptr->getOpcode() == X86ISD::WrapperRIP)
5656 if (Op.getOpcode() == X86ISD::VBROADCAST &&
5674 if (Op.getOpcode() == X86ISD::VZEXT_MOVL &&
5748 case X86ISD::BLENDI:
5755 case X86ISD::SHUFP:
5763 case X86ISD::INSERTPS:
5770 case X86ISD::EXTRQI:
5781 case X86ISD::INSERTQI:
5793 case X86ISD::UNPCKH:
5799 case X86ISD::UNPCKL:
5805 case X86ISD::MOVHLPS:
5811 case X86ISD::MOVLHPS:
5817 case X86ISD::PALIGNR:
5828 case X86ISD::VSHLDQ:
5836 case X86ISD::VSRLDQ:
5844 case X86ISD::PSHUFD:
5845 case X86ISD::VPERMILPI:
5852 case X86ISD::PSHUFHW:
5859 case X86ISD::PSHUFLW:
5866 case X86ISD::VZEXT_MOVL:
5871 case X86ISD::VBROADCAST: {
5890 case X86ISD::VPERMILPV: {
5906 case X86ISD::PSHUFB: {
5923 X86ISD::VPERMI:
5929 case X86ISD::MOVSS:
5930 case X86ISD::MOVSD:
5935 case X86ISD::VPERM2X128:
5943 case X86ISD::SHUF128:
5952 case X86ISD::MOVSLDUP:
5957 case X86ISD::MOVSHDUP:
5962 case X86ISD::MOVDDUP:
5967 case X86ISD::VPERMIL2: {
5989 case X86ISD::VPPERM: {
6005 case X86ISD::VPERMV: {
6023 case X86ISD::VPERMV3: {
6180 case X86ISD::ANDNP: {
6186 bool IsAndN = (X86ISD::ANDNP == Opcode);
6211 (N0.getOpcode() == X86ISD::PEXTRW &&
6213 (N0.getOpcode() == X86ISD::PEXTRB &&
6236 case X86ISD::PINSRB:
6237 case X86ISD::PINSRW: {
6257 (X86ISD::PINSRB == Opcode ? X86ISD::PEXTRB : X86ISD::PEXTRW);
6274 case X86ISD::PACKSS:
6275 case X86ISD::PACKUS: {
6284 if (Opcode == X86ISD::PACKSS) {
6304 case X86ISD::VSHLI:
6305 case X86ISD::VSRLI: {
6325 if (X86ISD::VSHLI == Opcode) {
6337 case X86ISD::VZEXT: {
6497 V = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, V);
6560 V = DAG.getNode(X86ISD::VZEXT_MOVL, dl, MVT::v4i32, V);
6685 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
6696 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
6925 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT,
7055 DAG.getNode(X86ISD::VBROADCASTM, dl,
7096 SDValue Brdcst = DAG.getNode(X86ISD::VBROADCAST, dl,
7120 SDValue Brdcst = DAG.getNode(X86ISD::VBROADCAST, dl,
7135 SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);
7194 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7203 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7211 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7217 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
7714 /// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
7725 /// X86ISD::ADDSUB operation:
7726 /// %Addsub = X86ISD::ADDSUB %AB, %C
7760 /// 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or
7761 /// X86ISD::FMSUBADD node.
7775 // Try to generate X86ISD::FMADDSUB node here.
7778 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
7786 // Do not generate X86ISD::ADDSUB node for 512-bit types even though
7794 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
7826 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
7829 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
7833 return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
7836 return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
7850 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
7856 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
7867 X86Opcode = X86ISD::HADD;
7872 X86Opcode = X86ISD::HSUB;
7900 X86Opcode = X86ISD::HADD;
7902 X86Opcode = X86ISD::HSUB;
7904 X86Opcode = X86ISD::FHADD;
7906 X86Opcode = X86ISD::FHSUB;
8079 Opcode = X86ISD::PSHUFB;
8083 Opcode = X86ISD::VPERMV;
8085 Opcode = X86ISD::PSHUFB;
8092 Opcode = X86ISD::VPERMILPV;
8095 Opcode = X86ISD::PSHUFB;
8104 Opcode = X86ISD::VPERMILPV;
8118 Opcode = X86ISD::VPERMV;
8126 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
8127 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
8141 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
8142 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
8152 Opcode = X86ISD::VPERMV;
8165 Opcode = X86ISD::VPERMV;
8173 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32,
8180 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
8181 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
8199 Opcode = X86ISD::VPERMV;
8209 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64,
8216 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
8217 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
8224 Opcode = X86ISD::VPERMV;
8228 Opcode = X86ISD::VPERMV;
8235 Opcode = X86ISD::VPERMV;
8253 SDValue Res = Opcode == X86ISD::VPERMV
8541 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
9306 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
9336 DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector),
9363 UnpackOpcode = X86ISD::UNPCKL;
9371 UnpackOpcode = X86ISD::UNPCKH;
9398 UnpackOpcode = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
9409 UnpackOpcode = X86ISD::UNPCKL;
9416 UnpackOpcode = X86ISD::UNPCKH;
9433 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
9438 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
9443 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
9447 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
9542 return DAG.getNode(X86ISD::VTRUNC, DL, VT, Src);
9567 PackOpcode = X86ISD::PACKUS;
9576 PackOpcode = X86ISD::PACKSS;
9669 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::ANDNP, DL, MaskVT,
9736 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
9763 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
9782 VT, DAG.getNode(X86ISD::BLENDI, DL, BlendVT, V1, V2,
9794 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2,
9808 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
10085 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
10100 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
10102 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
10135 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
10190 Opcode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI)
10191 : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI);
10390 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
10395 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
10455 InputV = getExtendInVec(X86ISD::VZEXT, DL, ExtVT, InputV, DAG);
10467 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
10474 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
10478 unsigned OddEvenOp = (Offset & 1 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW);
10493 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
10503 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
10507 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
10523 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
10540 unsigned UnpackLoHi = X86ISD::UNPCKL;
10542 UnpackLoHi = X86ISD::UNPCKH;
10678 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
10788 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL,
10796 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
10812 X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
10869 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
10892 ? X86ISD::MOVDDUP
10893 : X86ISD::VBROADCAST;
10894 bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2();
11004 ? X86ISD::MOVDDUP
11048 if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector())
11194 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
11268 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
11302 VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL,
11339 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
11344 X86ISD::SHUFP, DL, MVT::v2f64,
11375 X86ISD::MOVSD, DL, MVT::v2f64, V2,
11389 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
11423 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
11544 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
11583 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
11595 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
11624 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
11626 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
11632 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
11640 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
11642 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
11647 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
11681 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
11683 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
11735 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
11836 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
11843 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
11874 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
11916 return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFLW);
11918 return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFHW);
12024 FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
12051 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
12286 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
12289 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
12294 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
12306 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
12314 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
12353 V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
12357 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8,
12579 return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1);
12581 return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2);
12702 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
12821 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
12824 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
12873 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
12875 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
12881 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
13212 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
13244 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
13683 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
13712 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
13719 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
13725 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
13820 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
13827 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
13909 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
13911 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
13914 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
13938 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
13941 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
14025 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
14065 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
14429 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
14446 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
14455 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
14461 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
14508 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
14510 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
14513 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
14534 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
14565 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
14572 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
14637 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
14921 // i1 was sign extended we can use X86ISD::CVT2MASK.
15240 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
15315 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, VecVT, Vec,
15409 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
15552 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, N2);
15579 Opc = X86ISD::PINSRW;
15583 Opc = X86ISD::PINSRB;
15598 // putting the '3' into bits [7:6] of X86ISD::INSERTPS.
15615 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, N2);
15620 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
15704 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
15716 return X86ISD::Wrapper;
15721 return X86ISD::WrapperRIP;
15725 return X86ISD::WrapperRIP;
15727 return X86ISD::Wrapper;
15731 // their target counterpart wrapped in the X86ISD::Wrapper node. Suppose N is
15753 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
15775 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
15799 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
15826 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
15855 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
15892 X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
15893 : X86ISD::TLSADDR;
15918 DAG.getNode(X86ISD::GlobalBaseReg,
15951 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
15962 unsigned WrapperKind = X86ISD::Wrapper;
15989 unsigned WrapperKind = X86ISD::Wrapper;
15995 WrapperKind = X86ISD::WrapperRIP;
16014 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
16061 X86ISD::WrapperRIP : X86ISD::Wrapper;
16079 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
16088 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
16164 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
16185 // X86ISD::SHLD and X86ISD::SHRD have defined overflow behavior but the
16196 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
16199 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
16260 return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
16326 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
16327 X86ISD::FILD, DL,
16350 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys,
16416 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
16480 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
16483 // but using v2i32 to v2f64 with X86ISD::CVTSI2P.
16494 SDValue fHI = DAG.getNode(X86ISD::CVTSI2P, DL, MVT::v2f64, HI);
16496 SDValue fLO = DAG.getNode(X86ISD::CVTSI2P, DL, MVT::v2f64, LO);
16562 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
16570 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
16676 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
16771 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
16772 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
16773 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
16841 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
16914 return DAG.getNode(X86ISD::VZEXT, dl, VT, In);
17036 assert((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
17070 (Opcode == X86ISD::PACKSS || Subtarget.hasSSE41())) {
17232 DAG.getNode(X86ISD::VSEXT, DL, MVT::v16i32, In));
17248 truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget))
17255 truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget))
17352 unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
17370 return DAG.getNode(IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI, dl, VT,
17403 return DAG.getNode(X86ISD::VFPEXT, DL, VT,
17463 IsFABS ? X86ISD::FAND : IsFNABS ? X86ISD::FOR : X86ISD::FXOR;
17524 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
17538 MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask);
17542 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
17556 // Lower ISD::FGETSIGN to (AND (X86ISD::MOVMSK ...) 1).
17559 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
17565 /// Helper for creating a X86ISD::SETCC node.
17568 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
17661 SDValue Res = DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
17725 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
17781 Opcode = X86ISD::INC;
17790 Opcode = X86ISD::DEC;
17797 Opcode = X86ISD::ADD;
17823 // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
17894 case ISD::SUB: Opcode = X86ISD::SUB; break;
17895 case ISD::XOR: Opcode = X86ISD::XOR; break;
17896 case ISD::AND: Opcode = X86ISD::AND; break;
17897 case ISD::OR: Opcode = X86ISD::OR; break;
17902 case X86ISD::ADD:
17903 case X86ISD::SUB:
17904 case X86ISD::INC:
17905 case X86ISD::DEC:
17906 case X86ISD::OR:
17907 case X86ISD::XOR:
17908 case X86ISD::AND:
17927 case ISD::ADD: ConvertedOp = X86ISD::ADD; break;
17928 case ISD::SUB: ConvertedOp = X86ISD::SUB; break;
17929 case ISD::AND: ConvertedOp = X86ISD::AND; break;
17930 case ISD::OR: ConvertedOp = X86ISD::OR; break;
17931 case ISD::XOR: ConvertedOp = X86ISD::XOR; break;
17947 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
17983 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
17986 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
17995 Cmp.getOpcode() != X86ISD::CMP ||
18006 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
18013 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
18021 if (DAG.getNodeIfExists(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
18056 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
18090 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
18131 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo);
18365 SDValue Result = DAG.getNode(X86ISD::SUBUS, dl, VT, Op0, Op1);
18366 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
18389 Opc = X86ISD::CMPM;
18391 Opc = X86ISD::CMPP;
18409 CombineOpc = X86ISD::FOR;
18414 CombineOpc = X86ISD::FAND;
18431 if (Opc == X86ISD::CMPP)
18486 ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM;
18548 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
18564 unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
18565 : X86ISD::PCMPGT;
18577 if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) {
18599 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
18600 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
18618 if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) {
18697 SDValue KORTEST = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
18742 if (Op0.getOpcode() == X86ISD::SETCC) {
18776 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
18780 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
18787 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
18788 Opc == X86ISD::SAHF)
18791 (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
18792 Opc == X86ISD::SBB || Opc == X86ISD::SMUL ||
18793 Opc == X86ISD::INC || Opc == X86ISD::DEC || Opc == X86ISD::OR ||
18794 Opc == X86ISD::XOR || Opc == X86ISD::AND))
18797 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
18834 SDValue Cmp = DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0,
18837 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
18841 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
18876 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
18877 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
18878 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
18885 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
18949 if (Cond.getOpcode() == X86ISD::SETCC &&
18950 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
18968 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, VTs, Zero, CmpOp0);
18969 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
18975 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
18980 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
19031 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
19035 // If condition flag is set by a X86ISD::CMP, then use it as the condition
19036 // setting operand in place of the X86ISD::SETCC.
19038 if (CondOpcode == X86ISD::SETCC ||
19039 CondOpcode == X86ISD::SETCC_CARRY) {
19052 Opc == X86ISD::BT) { // FIXME
19066 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
19067 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
19068 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
19069 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
19070 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
19071 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
19116 if (Cond.getOpcode() == X86ISD::SUB) {
19123 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
19141 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
19152 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
19156 // X86ISD
19159 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops);
19276 X86ISD::VSEXT : X86ISD::VZEXT;
19291 Curr = DAG.getNode(X86ISD::UNPCKL, dl, CurrVT, DAG.getUNDEF(CurrVT), Curr);
19301 SignExt = DAG.getNode(X86ISD::VSRAI, dl, CurrVT, Curr,
19309 SDValue Sign = DAG.getNode(X86ISD::VSRAI, dl, CurrVT, Curr,
19341 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
19403 // code. We generate X86ISD::VSEXT for SEXTLOADs if it's available, otherwise
19585 SDValue Sext = getExtendInVec(X86ISD::VSEXT, dl, RegVT, SlicedVec, DAG);
19602 SDValue Sext = getExtendInVec(X86ISD::VZEXT, dl, RegVT, SlicedVec, DAG);
19621 /// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
19627 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
19629 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
19633 /// Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the
19639 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
19673 else if (Cond.getOpcode() == X86ISD::ADD ||
19674 Cond.getOpcode() == X86ISD::SUB ||
19675 Cond.getOpcode() == X86ISD::SMUL ||
19676 Cond.getOpcode() == X86ISD::UMUL)
19682 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
19686 // If condition flag is set by a X86ISD::CMP, then use it as the condition
19687 // setting operand in place of the X86ISD::SETCC.
19689 if (CondOpcode == X86ISD::SETCC ||
19690 CondOpcode == X86ISD::SETCC_CARRY) {
19696 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
19723 // instructions that can't be removed afterwards (i.e. X86ISD::ADD and
19724 // X86ISD::INC).
19726 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
19729 X86Opcode = X86ISD::INC; X86Cond = X86::COND_O;
19732 X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
19733 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
19736 X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O;
19739 X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
19740 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
19741 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
19772 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
19803 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
19844 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
19848 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
19874 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
19878 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
19911 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
19977 Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
19981 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Size);
20110 X86ISD::VAARG_64, dl,
20162 if (Opc == X86ISD::VSRAI)
20168 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
20180 case X86ISD::VSHLI:
20192 case X86ISD::VSRLI:
20204 case X86ISD::VSRAI:
20242 case X86ISD::VSHLI: Opc = X86ISD::VSHL; break;
20243 case X86ISD::VSRLI: Opc = X86ISD::VSRL; break;
20244 case X86ISD::VSRAI: Opc = X86ISD::VSRA; break;
20345 case X86ISD::CMPM:
20346 case X86ISD::CMPM_RND:
20347 case X86ISD::VPSHUFBITQMB:
20348 case X86ISD::VFPCLASS:
20351 case X86ISD::VTRUNC:
20352 case X86ISD::VTRUNCS:
20353 case X86ISD::VTRUNCUS:
20354 case X86ISD::CVTPS2PH:
20358 OpcodeSelect = X86ISD::SELECT;
20387 if (Op.getOpcode() == X86ISD::FSETCCM ||
20388 Op.getOpcode() == X86ISD::FSETCCM_RND ||
20389 Op.getOpcode() == X86ISD::VFPCLASSS)
20394 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);
20933 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
20936 FCmp = DAG.getNode(X86ISD::FSETCCM_RND, dl, MVT::v1i1, LHS, RHS,
20991 assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode");
21001 assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode");
21080 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
21101 Opcode = X86ISD::PCMPISTR;
21105 Opcode = X86ISD::PCMPESTR;
21109 Opcode = X86ISD::PCMPISTR;
21113 Opcode = X86ISD::PCMPESTR;
21117 Opcode = X86ISD::PCMPISTR;
21121 Opcode = X86ISD::PCMPESTR;
21125 Opcode = X86ISD::PCMPISTR;
21129 Opcode = X86ISD::PCMPESTR;
21133 Opcode = X86ISD::PCMPISTR;
21137 Opcode = X86ISD::PCMPESTR;
21152 Opcode = X86ISD::PCMPISTR;
21154 Opcode = X86ISD::PCMPESTR;
21165 Opcode = X86ISD::PCMPISTR;
21167 Opcode = X86ISD::PCMPESTR;
21196 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
21379 SDValue rd = DAG.getNode(X86ISD::RDPMC_DAG, DL, Tys, Chain);
21436 if (Opcode == X86ISD::RDTSCP_DAG) {
21470 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
21573 Opcode = X86ISD::UMWAIT;
21576 Opcode = X86ISD::TPAUSE;
21580 Opcode = X86ISD::LWPINS;
21611 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
21696 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(2),
21721 case X86ISD::VTRUNC: {
21732 case X86ISD::VTRUNCUS:
21733 case X86ISD::VTRUNCS: {
21734 bool IsSigned = (TruncationOp == X86ISD::VTRUNCS);
21905 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
21922 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
21930 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
21937 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
22129 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
22273 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
22274 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
22358 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
22368 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
22411 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op.getOperand(0));
22420 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
22499 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
22503 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
22620 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
22634 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
22638 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
22681 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
22685 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
22686 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
22691 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
22692 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
22696 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);
22766 return DAG.getNode(X86ISD::PACKUS, dl, VT,
22783 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
22834 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
22948 (!IsSigned || !Subtarget.hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
23046 unsigned X86Opc = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
23047 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
23058 return DAG.getNode(X86ISD::PCMPGT, dl, VT,
23065 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
23066 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
23075 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
23078 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
23122 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
23131 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT,
23140 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT,
23293 unsigned X86OpcI = (Opcode == ISD::SHL) ? X86ISD::VSHLI :
23294 (Opcode == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
23296 unsigned X86OpcV = (Opcode == ISD::SHL) ? X86ISD::VSHL :
23297 (Opcode == ISD::SRL) ? X86ISD::VSRL : X86ISD::VSRA;
23388 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
23427 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
23429 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
23465 // lowered as X86ISD::VSRLI nodes in parallel before blending.
23550 Opc = X86ISD::VSHL;
23553 Opc = X86ISD::VSRL;
23556 Opc = X86ISD::VSRA;
23652 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
23689 SDValue ALo = DAG.getNode(X86ISD::UNPCKL, dl, VT, DAG.getUNDEF(VT), Amt);
23690 SDValue AHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, DAG.getUNDEF(VT), Amt);
23691 SDValue RLo = DAG.getNode(X86ISD::UNPCKL, dl, VT, DAG.getUNDEF(VT), R);
23692 SDValue RHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, DAG.getUNDEF(VT), R);
23737 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
23744 SDValue ALo = DAG.getNode(X86ISD::UNPCKL, dl, VT, Amt, Z);
23745 SDValue AHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, Amt, Z);
23746 SDValue RLo = DAG.getNode(X86ISD::UNPCKL, dl, VT, Z, R);
23747 SDValue RHi = DAG.getNode(X86ISD::UNPCKH, dl, VT, Z, R);
23756 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
23850 unsigned Op = (Opcode == ISD::ROTL ? X86ISD::VROTLI : X86ISD::VROTRI);
23876 return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
23947 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
24021 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
24024 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
24052 BaseOp = X86ISD::INC;
24056 BaseOp = X86ISD::ADD;
24060 BaseOp = X86ISD::ADD;
24067 BaseOp = X86ISD::DEC;
24071 BaseOp = X86ISD::SUB;
24075 BaseOp = X86ISD::SUB;
24079 BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL;
24084 BaseOp = X86ISD::UMUL8;
24090 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
24209 // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at
24246 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
24264 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
24292 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
24315 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
24316 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
24322 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
24450 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
24462 SDValue Low = DAG.getNode(X86ISD::UNPCKL, DL, VT, V32, Zeros);
24463 SDValue High = DAG.getNode(X86ISD::UNPCKH, DL, VT, V32, Zeros);
24468 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
24470 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
24475 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
24542 DAG.getNode(X86ISD::PSHUFB, DL, ByteVecVT, InRegLUT, HighNibbles);
24544 DAG.getNode(X86ISD::PSHUFB, DL, ByteVecVT, InRegLUT, LowNibbles);
24705 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
24756 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
24757 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
24767 NewOpc = X86ISD::LADD;
24770 NewOpc = X86ISD::LSUB;
24773 NewOpc = X86ISD::LOR;
24776 NewOpc = X86ISD::LXOR;
24779 NewOpc = X86ISD::LAND;
24791 if ((NewOpc == X86ISD::LADD && C->isOne()) ||
24792 (NewOpc == X86ISD::LSUB && C->isAllOnesValue()))
24793 return DAG.getMemIntrinsicNode(X86ISD::LINC, SDLoc(N),
24797 if ((NewOpc == X86ISD::LSUB && C->isOne()) ||
24798 (NewOpc == X86ISD::LADD && C->isAllOnesValue()))
24799 return DAG.getMemIntrinsicNode(X86ISD::LDEC, SDLoc(N),
24885 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
24888 unsigned Opc = Op.getOpcode() == ISD::ADDCARRY ? X86ISD::ADC : X86ISD::SBB;
25428 case X86ISD::AVG: {
25429 // Legalize types for X86ISD::AVG by expanding vectors.
25447 SDValue Res = DAG.getNode(X86ISD::AVG, dl, RegVT, InVec0, InVec1);
25475 case X86ISD::FMINC:
25476 case X86ISD::FMIN:
25477 case X86ISD::FMAXC:
25478 case X86ISD::FMAX: {
25512 unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
25585 Results.push_back(DAG.getNode(X86ISD::CVTSI2P, dl, MVT::v4f32, Src));
25596 Results.push_back(DAG.getNode(X86ISD::CVTUI2P, dl, MVT::v4f32, Src));
25609 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
25615 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
25632 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
25635 return getReadTimeStampCounter(N, dl, X86ISD::RDTSCP_DAG, DAG, Subtarget,
25650 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
25698 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_SAVE_RBX_DAG
25699 : X86ISD::LCMPXCHG8_SAVE_EBX_DAG;
25709 Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG;
25894 switch ((X86ISD::NodeType)Opcode) {
25895 case X86ISD::FIRST_NUMBER: break;
25896 case X86ISD::BSF: return "X86ISD::BSF";
25897 case X86ISD::BSR: return "X86ISD::BSR";
25898 case X86ISD::SHLD: return "X86ISD::SHLD";
25899 case X86ISD::SHRD: return "X86ISD::SHRD";
25900 case X86ISD::FAND: return "X86ISD::FAND";
25901 case X86ISD::FANDN: return "X86ISD::FANDN";
25902 case X86ISD::FOR: return "X86ISD::FOR";
25903 case X86ISD::FXOR: return "X86ISD::FXOR";
25904 case X86ISD::FILD: return "X86ISD::FILD";
25905 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
25906 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
25907 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
25908 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
25909 case X86ISD::FLD: return "X86ISD::FLD";
25910 case X86ISD::FST: return "X86ISD::FST";
25911 case X86ISD::CALL: return "X86ISD::CALL";
25912 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
25913 case X86ISD::RDTSCP_DAG: return "X86ISD::RDTSCP_DAG";
25914 case X86ISD::RDPMC_DAG: return "X86ISD::RDPMC_DAG";
25915 case X86ISD::BT: return "X86ISD::BT";
25916 case X86ISD::CMP: return "X86ISD::CMP";
25917 case X86ISD::COMI: return "X86ISD::COMI";
25918 case X86ISD::UCOMI: return "X86ISD::UCOMI";
25919 case X86ISD::CMPM: return "X86ISD::CMPM";
25920 case X86ISD::CMPM_RND: return "X86ISD::CMPM_RND";
25921 case X86ISD::SETCC: return "X86ISD::SETCC";
25922 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
25923 case X86ISD::FSETCC: return "X86ISD::FSETCC";
25924 case X86ISD::FSETCCM: return "X86ISD::FSETCCM";
25925 case X86ISD::FSETCCM_RND: return "X86ISD::FSETCCM_RND";
25926 case X86ISD::CMOV: return "X86ISD::CMOV";
25927 case X86ISD::BRCOND: return "X86ISD::BRCOND";
25928 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
25929 case X86ISD::IRET: return "X86ISD::IRET";
25930 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
25931 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
25932 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
25933 case X86ISD::Wrapper: return "X86ISD::Wrapper";
25934 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
25935 case X86ISD::MOVDQ2Q: return "X86ISD::MOVDQ2Q";
25936 case X86ISD::MMX_MOVD2W: return "X86ISD::MMX_MOVD2W";
25937 case X86ISD::MMX_MOVW2D: return "X86ISD::MMX_MOVW2D";
25938 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
25939 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
25940 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
25941 case X86ISD::PINSRB: return "X86ISD::PINSRB";
25942 case X86ISD::PINSRW: return "X86ISD::PINSRW";
25943 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
25944 case X86ISD::ANDNP: return "X86ISD::ANDNP";
25945 case X86ISD::BLENDI: return "X86ISD::BLENDI";
25946 case X86ISD::SHRUNKBLEND: return "X86ISD::SHRUNKBLEND";
25947 case X86ISD::ADDUS: return "X86ISD::ADDUS";
25948 case X86ISD::SUBUS: return "X86ISD::SUBUS";
25949 case X86ISD::HADD: return "X86ISD::HADD";
25950 case X86ISD::HSUB: return "X86ISD::HSUB";
25951 case X86ISD::FHADD: return "X86ISD::FHADD";
25952 case X86ISD::FHSUB: return "X86ISD::FHSUB";
25953 case X86ISD::CONFLICT: return "X86ISD::CONFLICT";
25954 case X86ISD::FMAX: return "X86ISD::FMAX";
25955 case X86ISD::FMAXS: return "X86ISD::FMAXS";
25956 case X86ISD::FMAX_RND: return "X86ISD::FMAX_RND";
25957 case X86ISD::FMAXS_RND: return "X86ISD::FMAX_RND";
25958 case X86ISD::FMIN: return "X86ISD::FMIN";
25959 case X86ISD::FMINS: return "X86ISD::FMINS";
25960 case X86ISD::FMIN_RND: return "X86ISD::FMIN_RND";
25961 case X86ISD::FMINS_RND: return "X86ISD::FMINS_RND";
25962 case X86ISD::FMAXC: return "X86ISD::FMAXC";
25963 case X86ISD::FMINC: return "X86ISD::FMINC";
25964 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
25965 case X86ISD::FRCP: return "X86ISD::FRCP";
25966 case X86ISD::EXTRQI: return "X86ISD::EXTRQI";
25967 case X86ISD::INSERTQI: return "X86ISD::INSERTQI";
25968 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
25969 case X86ISD::TLSBASEADDR: return "X86ISD::TLSBASEADDR";
25970 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
25971 case X86ISD::EH_SJLJ_SETJMP: return "X86ISD::EH_SJLJ_SETJMP";
25972 case X86ISD::EH_SJLJ_LONGJMP: return "X86ISD::EH_SJLJ_LONGJMP";
25973 case X86ISD::EH_SJLJ_SETUP_DISPATCH:
25974 return "X86ISD::EH_SJLJ_SETUP_DISPATCH";
25975 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
25976 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
25977 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
25978 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r";
25979 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
25980 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
25981 case X86ISD::LCMPXCHG16_DAG: return "X86ISD::LCMPXCHG16_DAG";
25982 case X86ISD::LCMPXCHG8_SAVE_EBX_DAG:
25983 return "X86ISD::LCMPXCHG8_SAVE_EBX_DAG";
25984 case X86ISD::LCMPXCHG16_SAVE_RBX_DAG:
25985 return "X86ISD::LCMPXCHG16_SAVE_RBX_DAG";
25986 case X86ISD::LADD: return "X86ISD::LADD";
25987 case X86ISD::LSUB: return "X86ISD::LSUB";
25988 case X86ISD::LOR: return "X86ISD::LOR";
25989 case X86ISD::LXOR: return "X86ISD::LXOR";
25990 case X86ISD::LAND: return "X86ISD::LAND";
25991 case X86ISD::LINC: return "X86ISD::LINC";
25992 case X86ISD::LDEC: return "X86ISD::LDEC";
25993 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
25994 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
25995 case X86ISD::VZEXT: return "X86ISD::VZEXT";
25996 case X86ISD::VSEXT: return "X86ISD::VSEXT";
25997 case X86ISD::VTRUNC: return "X86ISD::VTRUNC";
25998 case X86ISD::VTRUNCS: return "X86ISD::VTRUNCS";
25999 case X86ISD::VTRUNCUS: return "X86ISD::VTRUNCUS";
26000 case X86ISD::VTRUNCSTORES: return "X86ISD::VTRUNCSTORES";
26001 case X86ISD::VTRUNCSTOREUS: return "X86ISD::VTRUNCSTOREUS";
26002 case X86ISD::VMTRUNCSTORES: return "X86ISD::VMTRUNCSTORES";
26003 case X86ISD::VMTRUNCSTOREUS: return "X86ISD::VMTRUNCSTOREUS";
26004 case X86ISD::VFPEXT: return "X86ISD::VFPEXT";
26005 case X86ISD::VFPEXT_RND: return "X86ISD::VFPEXT_RND";
26006 case X86ISD::VFPEXTS_RND: return "X86ISD::VFPEXTS_RND";
26007 case X86ISD::VFPROUND: return "X86ISD::VFPROUND";
26008 case X86ISD::VFPROUND_RND: return "X86ISD::VFPROUND_RND";
26009 case X86ISD::VFPROUNDS_RND: return "X86ISD::VFPROUNDS_RND";
26010 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ";
26011 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ";
26012 case X86ISD::VSHL: return "X86ISD::VSHL";
26013 case X86ISD::VSRL: return "X86ISD::VSRL";
26014 case X86ISD::VSRA: return "X86ISD::VSRA";
26015 case X86ISD::VSHLI: return "X86ISD::VSHLI";
26016 case X86ISD::VSRLI: return "X86ISD::VSRLI";
26017 case X86ISD::VSRAI: return "X86ISD::VSRAI";
26018 case X86ISD::VSRAV: return "X86ISD::VSRAV";
26019 case X86ISD::VROTLI: return "X86ISD::VROTLI";
26020 case X86ISD::VROTRI: return "X86ISD::VROTRI";
26021 case X86ISD::VPPERM: return "X86ISD::VPPERM";
26022 case X86ISD::CMPP: return "X86ISD::CMPP";
26023 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ";
26024 case X86ISD::PCMPGT: return "X86ISD::PCMPGT";
26025 case X86ISD::PHMINPOS: return "X86ISD::PHMINPOS";
26026 case X86ISD::ADD: return "X86ISD::ADD";
26027 case X86ISD::SUB: return "X86ISD::SUB";
26028 case X86ISD::ADC: return "X86ISD::ADC";
26029 case X86ISD::SBB: return "X86ISD::SBB";
26030 case X86ISD::SMUL: return "X86ISD::SMUL";
26031 case X86ISD::UMUL: return "X86ISD::UMUL";
26032 case X86ISD::SMUL8: return "X86ISD::SMUL8";
26033 case X86ISD::UMUL8: return "X86ISD::UMUL8";
26034 case X86ISD::SDIVREM8_SEXT_HREG: return "X86ISD::SDIVREM8_SEXT_HREG";
26035 case X86ISD::UDIVREM8_ZEXT_HREG: return "X86ISD::UDIVREM8_ZEXT_HREG";
26036 case X86ISD::INC: return "X86ISD::INC";
26037 case X86ISD::DEC: return "X86ISD::DEC";
26038 case X86ISD::OR: return "X86ISD::OR";
26039 case X86ISD::XOR: return "X86ISD::XOR";
26040 case X86ISD::AND: return "X86ISD::AND";
26041 case X86ISD::BEXTR: return "X86ISD::BEXTR";
26042 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
26043 case X86ISD::MOVMSK: return "X86ISD::MOVMSK";
26044 case X86ISD::PTEST: return "X86ISD::PTEST";
26045 case X86ISD::TESTP: return "X86ISD::TESTP";
26046 case X86ISD::KORTEST: return "X86ISD::KORTEST";
26047 case X86ISD::KTEST: return "X86ISD::KTEST";
26048 case X86ISD::KADD: return "X86ISD::KADD";
26049 case X86ISD::KSHIFTL: return "X86ISD::KSHIFTL";
26050 case X86ISD::KSHIFTR: return "X86ISD::KSHIFTR";
26051 case X86ISD::PACKSS: return "X86ISD::PACKSS";
26052 case X86ISD::PACKUS: return "X86ISD::PACKUS";
26053 case X86ISD::PALIGNR: return "X86ISD::PALIGNR";
26054 case X86ISD::VALIGN: return "X86ISD::VALIGN";
26055 case X86ISD::VSHLD: return "X86ISD::VSHLD";
26056 case X86ISD::VSHRD: return "X86ISD::VSHRD";
26057 case X86ISD::VSHLDV: return "X86ISD::VSHLDV";
26058 case X86ISD::VSHRDV: return "X86ISD::VSHRDV";
26059 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
26060 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
26061 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
26062 case X86ISD::SHUFP: return "X86ISD::SHUFP";
26063 case X86ISD::SHUF128: return "X86ISD::SHUF128";
26064 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
26065 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
26066 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
26067 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
26068 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
26069 case X86ISD::MOVSD: return "X86ISD::MOVSD";
26070 case X86ISD::MOVSS: return "X86ISD::MOVSS";
26071 case X86ISD::UNPCKL: return "X86ISD::UNPCKL";
26072 case X86ISD::UNPCKH: return "X86ISD::UNPCKH";
26073 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
26074 case X86ISD::VBROADCASTM: return "X86ISD::VBROADCASTM";
26075 case X86ISD::SUBV_BROADCAST: return "X86ISD::SUBV_BROADCAST";
26076 case X86ISD::VPERMILPV: return "X86ISD::VPERMILPV";
26077 case X86ISD::VPERMILPI: return "X86ISD::VPERMILPI";
26078 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128";
26079 case X86ISD::VPERMV: return "X86ISD::VPERMV";
26080 case X86ISD::VPERMV3: return "X86ISD::VPERMV3";
26081 case X86ISD::VPERMI: return "X86ISD::VPERMI";
26082 case X86ISD::VPTERNLOG: return "X86ISD::VPTERNLOG";
26083 X86ISD::VFIXUPIMM: return "X86ISD::VFIXUPIMM";
26084 case X86ISD::VFIXUPIMMS: return "X86ISD::VFIXUPIMMS";
26085 case X86ISD::VRANGE: return "X86ISD::VRANGE";
26086 case X86ISD::VRANGE_RND: return "X86ISD::VRANGE_RND";
26087 case X86ISD::VRANGES: return "X86ISD::VRANGES";
26088 case X86ISD::VRANGES_RND: return "X86ISD::VRANGES_RND";
26089 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ";
26090 case X86ISD::PMULDQ: return "X86ISD::PMULDQ";
26091 case X86ISD::PSADBW: return "X86ISD::PSADBW";
26092 case X86ISD::DBPSADBW: return "X86ISD::DBPSADBW";
26093 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
26094 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
26095 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
26096 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
26097 case X86ISD::MFENCE: return "X86ISD::MFENCE";
26098 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
26099 case X86ISD::SAHF: return "X86ISD::SAHF";
26100 case X86ISD::RDRAND: return "X86ISD::RDRAND";
26101 case X86ISD::RDSEED: return "X86ISD::RDSEED";
26102 case X86ISD::VPMADDUBSW: return "X86ISD::VPMADDUBSW";
26103 case X86ISD::VPMADDWD: return "X86ISD::VPMADDWD";
26104 case X86ISD::VPSHA: return "X86ISD::VPSHA";
26105 case X86ISD::VPSHL: return "X86ISD::VPSHL";
26106 case X86ISD::VPCOM: return "X86ISD::VPCOM";
26107 case X86ISD::VPCOMU: return "X86ISD::VPCOMU";
26108 case X86ISD::VPERMIL2: return "X86ISD::VPERMIL2";
26109 case X86ISD::FMSUB: return "X86ISD::FMSUB";
26110 case X86ISD::FNMADD: return "X86ISD::FNMADD";
26111 case X86ISD::FNMSUB: return "X86ISD::FNMSUB";
26112 case X86ISD::FMADDSUB: return "X86ISD::FMADDSUB";
26113 case X86ISD::FMSUBADD: return "X86ISD::FMSUBADD";
26114 case X86ISD::FMADD_RND: return "X86ISD::FMADD_RND";
26115 case X86ISD::FNMADD_RND: return "X86ISD::FNMADD_RND";
26116 case X86ISD::FMSUB_RND: return "X86ISD::FMSUB_RND";
26117 case X86ISD::FNMSUB_RND: return "X86ISD::FNMSUB_RND";
26118 case X86ISD::FMADDSUB_RND: return "X86ISD::FMADDSUB_RND";
26119 case X86ISD::FMSUBADD_RND: return "X86ISD::FMSUBADD_RND";
26120 case X86ISD::VPMADD52H: return "X86ISD::VPMADD52H";
26121 case X86ISD::VPMADD52L: return "X86ISD::VPMADD52L";
26122 case X86ISD::VRNDSCALE: return "X86ISD::VRNDSCALE";
26123 case X86ISD::VRNDSCALE_RND: return "X86ISD::VRNDSCALE_RND";
26124 case X86ISD::VRNDSCALES: return "X86ISD::VRNDSCALES";
26125 case X86ISD::VRNDSCALES_RND: return "X86ISD::VRNDSCALES_RND";
26126 case X86ISD::VREDUCE: return "X86ISD::VREDUCE";
26127 case X86ISD::VREDUCE_RND: return "X86ISD::VREDUCE_RND";
26128 case X86ISD::VREDUCES: return "X86ISD::VREDUCES";
26129 case X86ISD::VREDUCES_RND: return "X86ISD::VREDUCES_RND";
26130 case X86ISD::VGETMANT: return "X86ISD::VGETMANT";
26131 case X86ISD::VGETMANT_RND: return "X86ISD::VGETMANT_RND";
26132 case X86ISD::VGETMANTS: return "X86ISD::VGETMANTS";
26133 case X86ISD::VGETMANTS_RND: return "X86ISD::VGETMANTS_RND";
26134 case X86ISD::PCMPESTR: return "X86ISD::PCMPESTR";
26135 case X86ISD::PCMPISTR: return "X86ISD::PCMPISTR";
26136 case X86ISD::XTEST: return "X86ISD::XTEST";
26137 case X86ISD::COMPRESS: return "X86ISD::COMPRESS";
26138 case X86ISD::EXPAND: return "X86ISD::EXPAND";
26139 case X86ISD::SELECT: return "X86ISD::SELECT";
26140 case X86ISD::SELECTS: return "X86ISD::SELECTS";
26141 case X86ISD::ADDSUB: return "X86ISD::ADDSUB";
26142 case X86ISD::RCP14: return "X86ISD::RCP14";
26143 case X86ISD::RCP14S: return "X86ISD::RCP14S";
26144 case X86ISD::RCP28: return "X86ISD::RCP28";
26145 case X86ISD::RCP28S: return "X86ISD::RCP28S";
26146 case X86ISD::EXP2: return "X86ISD::EXP2";
26147 case X86ISD::RSQRT14: return "X86ISD::RSQRT14";
26148 case X86ISD::RSQRT14S: return "X86ISD::RSQRT14S";
26149 case X86ISD::RSQRT28: return "X86ISD::RSQRT28";
26150 case X86ISD::RSQRT28S: return "X86ISD::RSQRT28S";
26151 case X86ISD::FADD_RND: return "X86ISD::FADD_RND";
26152 case X86ISD::FADDS_RND: return "X86ISD::FADDS_RND";
26153 case X86ISD::FSUB_RND: return "X86ISD::FSUB_RND";
26154 case X86ISD::FSUBS_RND: return "X86ISD::FSUBS_RND";
26155 case X86ISD::FMUL_RND: return "X86ISD::FMUL_RND";
26156 case X86ISD::FMULS_RND: return "X86ISD::FMULS_RND";
26157 case X86ISD::FDIV_RND: return "X86ISD::FDIV_RND";
26158 case X86ISD::FDIVS_RND: return "X86ISD::FDIVS_RND";
26159 case X86ISD::FSQRT_RND: return "X86ISD::FSQRT_RND";
26160 case X86ISD::FSQRTS_RND: return "X86ISD::FSQRTS_RND";
26161 case X86ISD::FGETEXP_RND: return "X86ISD::FGETEXP_RND";
26162 case X86ISD::FGETEXPS_RND: return "X86ISD::FGETEXPS_RND";
26163 case X86ISD::SCALEF: return "X86ISD::SCALEF";
26164 case X86ISD::SCALEFS: return "X86ISD::SCALEFS";
26165 case X86ISD::ADDS: return "X86ISD::ADDS";
26166 case X86ISD::SUBS: return "X86ISD::SUBS";
26167 case X86ISD::AVG: return "X86ISD::AVG";
26168 case X86ISD::MULHRS: return "X86ISD::MULHRS";
26169 case X86ISD::SINT_TO_FP_RND: return "X86ISD::SINT_TO_FP_RND";
26170 case X86ISD::UINT_TO_FP_RND: return "X86ISD::UINT_TO_FP_RND";
26171 case X86ISD::CVTTP2SI: return "X86ISD::CVTTP2SI";
26172 case X86ISD::CVTTP2UI: return "X86ISD::CVTTP2UI";
26173 case X86ISD::CVTTP2SI_RND: return "X86ISD::CVTTP2SI_RND";
26174 case X86ISD::CVTTP2UI_RND: return "X86ISD::CVTTP2UI_RND";
26175 case X86ISD::CVTTS2SI_RND: return "X86ISD::CVTTS2SI_RND";
26176 case X86ISD::CVTTS2UI_RND: return "X86ISD::CVTTS2UI_RND";
26177 case X86ISD::CVTSI2P: return "X86ISD::CVTSI2P";
26178 case X86ISD::CVTUI2P: return "X86ISD::CVTUI2P";
26179 case X86ISD::VFPCLASS: return "X86ISD::VFPCLASS";
26180 case X86ISD::VFPCLASSS: return "X86ISD::VFPCLASSS";
26181 case X86ISD::MULTISHIFT: return "X86ISD::MULTISHIFT";
26182 case X86ISD::SCALAR_SINT_TO_FP_RND: return "X86ISD::SCALAR_SINT_TO_FP_RND";
26183 case X86ISD::SCALAR_UINT_TO_FP_RND: return "X86ISD::SCALAR_UINT_TO_FP_RND";
26184 case X86ISD::CVTPS2PH: return "X86ISD::CVTPS2PH";
26185 case X86ISD::CVTPH2PS: return "X86ISD::CVTPH2PS";
26186 case X86ISD::CVTPH2PS_RND: return "X86ISD::CVTPH2PS_RND";
26187 case X86ISD::CVTP2SI: return "X86ISD::CVTP2SI";
26188 case X86ISD::CVTP2UI: return "X86ISD::CVTP2UI";
26189 case X86ISD::CVTP2SI_RND: return "X86ISD::CVTP2SI_RND";
26190 case X86ISD::CVTP2UI_RND: return "X86ISD::CVTP2UI_RND";
26191 case X86ISD::CVTS2SI_RND: return "X86ISD::CVTS2SI_RND";
26192 case X86ISD::CVTS2UI_RND: return "X86ISD::CVTS2UI_RND";
26193 case X86ISD::LWPINS: return "X86ISD::LWPINS";
26194 case X86ISD::MGATHER: return "X86ISD::MGATHER";
26195 case X86ISD::MSCATTER: return "X86ISD::MSCATTER";
26196 case X86ISD::VPDPBUSD: return "X86ISD::VPDPBUSD";
26197 case X86ISD::VPDPBUSDS: return "X86ISD::VPDPBUSDS";
26198 case X86ISD::VPDPWSSD: return "X86ISD::VPDPWSSD";
26199 case X86ISD::VPDPWSSDS: return "X86ISD::VPDPWSSDS";
26200 case X86ISD::VPSHUFBITQMB: return "X86ISD::VPSHUFBITQMB";
26201 case X86ISD::GF2P8MULB: return "X86ISD::GF2P8MULB";
26202 case X86ISD::GF2P8AFFINEQB: return "X86ISD::GF2P8AFFINEQB";
26203 case X86ISD::GF2P8AFFINEINVQB: return "X86ISD::GF2P8AFFINEINVQB";
26204 case X86ISD::NT_CALL: return "X86ISD::NT_CALL";
26205 case X86ISD::NT_BRIND: return "X86ISD::NT_BRIND";
26206 case X86ISD::UMWAIT: return "X86ISD::UMWAIT";
26207 case X86ISD::TPAUSE: return "X86ISD::TPAUSE";
28879 case X86ISD::SETCC:
28882 case X86ISD::MOVMSK: {
28887 case X86ISD::PEXTRB:
28888 case X86ISD::PEXTRW: {
28898 case X86ISD::VSHLI:
28899 case X86ISD::VSRLI: {
28908 if (Opc == X86ISD::VSHLI) {
28922 case X86ISD::PACKUS: {
28935 case X86ISD::VZEXT: {
28952 case X86ISD::CMOV: {
28965 case X86ISD::UDIVREM8_ZEXT_HREG:
29032 case X86ISD::SETCC_CARRY:
29036 case X86ISD::VSEXT: {
29044 case X86ISD::VTRUNC: {
29055 case X86ISD::PACKSS: {
29067 case X86ISD::VSHLI: {
29078 case X86ISD::VSRAI: {
29088 case X86ISD::PCMPGT:
29089 case X86ISD::PCMPEQ:
29090 case X86ISD::CMPP:
29091 case X86ISD::VPCOM:
29092 case X86ISD::VPCOMU:
29096 case X86ISD::CMOV: {
29102 case X86ISD::SDIVREM8_SEXT_HREG:
29115 if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
29125 if (N->getOpcode() == X86ISD::Wrapper) {
29150 Shuffle = X86ISD::VZEXT_MOVL;
29175 Shuffle = unsigned(X86ISD::VZEXT);
29190 Shuffle = X86ISD::VZEXT_MOVL;
29200 Shuffle = X86ISD::MOVDDUP;
29205 Shuffle = X86ISD::MOVSLDUP;
29210 Shuffle = X86ISD::MOVSHDUP;
29219 Shuffle = X86ISD::MOVDDUP;
29224 Shuffle = X86ISD::MOVSLDUP;
29229 Shuffle = X86ISD::MOVSHDUP;
29239 Shuffle = X86ISD::MOVDDUP;
29245 Shuffle = X86ISD::MOVSLDUP;
29251 Shuffle = X86ISD::MOVSHDUP;
29262 Shuffle = X86ISD::VBROADCAST;
29294 Shuffle = X86ISD::VPERMI;
29302 Shuffle = X86ISD::VPERMI;
29310 Shuffle = X86ISD::VPERMILPI;
29336 Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI);
29354 Shuffle = X86ISD::PSHUFLW;
29368 Shuffle = X86ISD::PSHUFHW;
29408 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS;
29414 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS;
29421 Shuffle = X86ISD::MOVSD;
29427 Shuffle = X86ISD::MOVSS;
29474 Shuffle = X86ISD::PALIGNR;
29481 // Attempt to combine to X86ISD::BLENDI.
29503 Shuffle = X86ISD::BLENDI;
29508 // Determine a type compatible with X86ISD::BLENDI.
29535 Shuffle = X86ISD::BLENDI;
29546 Shuffle = X86ISD::INSERTPS;
29558 Shuffle = X86ISD::SHUFP;
29603 Shuffle = X86ISD::SHUFP;
29679 if (Depth == 1 && Root.getOpcode() == X86ISD::VPERM2X128)
29687 Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res,
29734 // If we are shuffling a X86ISD::VZEXT_LOAD then we can use the load
29737 if (V1.getOpcode() == X86ISD::VZEXT_LOAD &&
29810 if (Depth == 1 && Root.getOpcode() == X86ISD::EXTRQI)
29813 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
29820 if (Depth == 1 && Root.getOpcode() == X86ISD::INSERTQI)
29824 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
29857 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
29882 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, Res, VPermMask, Zero);
29901 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2);
29929 FloatDomain ? unsigned(X86ISD::FAND) : unsigned(ISD::AND);
29948 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
29982 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
30016 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
30047 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
30400 case X86ISD::PSHUFD:
30402 case X86ISD::PSHUFLW:
30405 case X86ISD::PSHUFHW:
30423 assert(N.getOpcode() == X86ISD::PSHUFD &&
30442 case X86ISD::PSHUFD:
30446 case X86ISD::PSHUFLW:
30456 case X86ISD::PSHUFHW:
30466 case X86ISD::UNPCKL:
30467 case X86ISD::UNPCKH:
30476 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
30487 case X86ISD::PSHUFLW:
30488 case X86ISD::PSHUFHW:
30529 case X86ISD::UNPCKL:
30530 case X86ISD::UNPCKH:
30534 case X86ISD::PSHUFD:
30535 case X86ISD::PSHUFLW:
30536 case X86ISD::PSHUFHW:
30558 (N.getOpcode() == X86ISD::PSHUFLW || N.getOpcode() == X86ISD::PSHUFHW) &&
30575 case X86ISD::PSHUFLW:
30576 case X86ISD::PSHUFHW:
30628 (Opcode == X86ISD::MOVSD || Opcode == X86ISD::UNPCKH ||
30629 Opcode == X86ISD::UNPCKL)) {
30637 (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD ||
30638 Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB ||
30639 Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS)) {
30641 if (Opcode == X86ISD::MOVSD) {
30645 Lo = BC0.getOperand(Opcode == X86ISD::UNPCKH ? 1 : 0);
30646 Hi = BC1.getOperand(Opcode == X86ISD::UNPCKH ? 1 : 0);
30654 case X86ISD::VBROADCAST: {
30671 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
30676 case X86ISD::PSHUFD:
30677 case X86ISD::PSHUFLW:
30678 case X86ISD::PSHUFHW:
30682 case X86ISD::UNPCKL: {
30683 // Combine X86ISD::UNPCKL and ISD::VECTOR_SHUFFLE into X86ISD::UNPCKH, in
30684 // which X86ISD::UNPCKL has a ISD::UNDEF operand, and ISD::VECTOR_SHUFFLE
30689 // t3: v16i8 = X86ISD::UNPCKL undef:v16i8, t2
30693 // t3: v16i8 = X86ISD::UNPCKH undef:v16i8, t1
30712 return DAG.getNode(X86ISD::UNPCKH, DL, VT, N.getOperand(0), ShufOp);
30716 case X86ISD::MOVSD:
30717 case X86ISD::MOVSS: {
30745 case X86ISD::INSERTPS: {
30757 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
30762 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
30773 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
30780 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
30825 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
30843 case X86ISD::PSHUFLW:
30844 case X86ISD::PSHUFHW:
30855 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
30860 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
30869 (V.getOpcode() == X86ISD::PSHUFLW ||
30870 V.getOpcode() == X86ISD::PSHUFHW) &&
30874 if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
30877 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
30878 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
30892 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
30893 : X86ISD::UNPCKH,
30901 case X86ISD::PSHUFD:
31029 if (FMSub.getOpcode() != X86ISD::FMSUB)
31032 if (FMAdd.getOpcode() != ISD::FMA || FMSub.getOpcode() != X86ISD::FMSUB ||
31047 unsigned Opcode = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
31068 // Try to generate X86ISD::FMADDSUB node here.
31071 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
31078 // Do not generate X86ISD::ADDSUB node for 512-bit types even though
31084 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
31138 if (HOp.getOpcode() != X86ISD::HADD && HOp.getOpcode() != X86ISD::FHADD &&
31139 HOp.getOpcode() != X86ISD::HSUB && HOp.getOpcode() != X86ISD::FHSUB)
31481 V = DAG.getNode(X86ISD::PACKSS, DL, MVT::v16i8, V,
31488 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
31582 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
31588 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
31692 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
31715 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);
31733 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,
31742 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
31777 case ISD::AND: FPOpcode = X86ISD::FAND; break;
31778 case ISD::OR: FPOpcode = X86ISD::FOR; break;
31779 case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
31962 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
32037 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
32115 Res = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Res);
32223 if (X86ISD::VBROADCAST == Src.getOpcode() &&
32284 unsigned OpCode = (SrcVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);
32304 // X86ISD::PEXTRW/X86ISD::PEXTRB in:
32337 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, MMXSrc);
32462 SDValue AndN = DAG.getNode(X86ISD::ANDNP, DL, AndNVT, CastCond, CastRHS);
32597 SDValue SB = DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(U), U->getValueType(0),
32664 Opcode = X86ISD::FMIN;
32672 Opcode = X86ISD::FMIN;
32682 Opcode = X86ISD::FMIN;
32691 Opcode = X86ISD::FMAX;
32704 Opcode = X86ISD::FMAX;
32714 Opcode = X86ISD::FMAX;
32733 Opcode = X86ISD::FMIN;
32740 Opcode = X86ISD::FMIN;
32750 Opcode = X86ISD::FMIN;
32757 Opcode = X86ISD::FMAX;
32770 Opcode = X86ISD::FMAX;
32780 Opcode = X86ISD::FMAX;
32886 return DAG.getNode(X86ISD::SUBUS, DL, Ops[0].getValueType(), Ops);
32959 if (!(Cmp.getOpcode() == X86ISD::CMP ||
32960 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
33057 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
33073 if (!(Cmp.getOpcode() == X86ISD::CMP ||
33074 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
33125 case X86ISD::SETCC_CARRY:
33135 case X86ISD::SETCC:
33141 case X86ISD::CMOV: {
33157 if ((Op.getOpcode() != X86ISD::RDRAND &&
33158 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
33192 if (Cond->getOpcode() == X86ISD::CMP) {
33205 case X86ISD::AND:
33209 case X86ISD::OR:
33216 if (SetCC0.getOpcode() != X86ISD::SETCC ||
33217 SetCC1.getOpcode() != X86ISD::SETCC ||
33231 if (EFLAGS.getOpcode() == X86ISD::ADD) {
33241 if (Carry.getOpcode() == X86ISD::SETCC ||
33242 Carry.getOpcode() == X86ISD::SETCC_CARRY) {
33267 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
33284 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
33391 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
33405 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
33440 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
33442 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
33452 Cond.getOpcode() == X86ISD::CMP && isNullConstant(Cond.getOperand(1))) {
33470 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
33701 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
33711 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
33713 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
33815 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, Ops);
33843 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
33854 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
33903 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
33942 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
33949 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
34031 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
34034 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
34038 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
34188 assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
34212 bool IsSigned = (X86ISD::PACKSS == Opcode);
34270 assert((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode ||
34271 X86ISD::VSRLI == Opcode) &&
34273 bool LogicalShift = X86ISD::VSHLI == Opcode || X86ISD::VSRLI == Opcode;
34302 if (Opcode == X86ISD::VSRLI && (ShiftVal + 1) == NumBitsPerElt &&
34303 N0.getOpcode() == X86ISD::VSRAI)
34304 return DAG.getNode(X86ISD::VSRLI, SDLoc(N), VT, N0.getOperand(0), N1);
34307 if (Opcode == X86ISD::VSRAI && N0.getOpcode() == X86ISD::VSHLI &&
34333 if (X86ISD::VSHLI == Opcode)
34335 else if (X86ISD::VSRAI == Opcode)
34350 ((N->getOpcode() == X86ISD::PINSRB && N->getValueType(0) == MVT::v16i8) ||
34351 (N->getOpcode() == X86ISD::PINSRW &&
34383 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
34426 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
34436 SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
34504 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
34585 FPOpcode = X86ISD::FAND;
34587 FPOpcode = X86ISD::FOR;
34589 FPOpcode = X86ISD::FXOR;
34641 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
34772 MVT::v4i32, DAG.getNode(X86ISD::FAND, SDLoc(N), MVT::v4f32,
34873 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP)
35037 return N->getOpcode() == X86ISD::SETCC && N->hasOneUse() &&
35039 N->getOperand(1).getOpcode() == X86ISD::CMP &&
35108 DAG.getNode(X86ISD::FOR, SDLoc(N), MVT::v4f32,
35158 unsigned Opc = X86ISD::SHLD;
35163 Opc = X86ISD::SHRD;
35195 unsigned InnerShift = (X86ISD::SHLD == Opc ? ISD::SRL : ISD::SHL);
35316 return DAG.getNode(X86ISD::PCMPGT, SDLoc(N), VT, Shift.getOperand(0), Ones);
35476 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
35478 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
35488 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
35491 return truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
35494 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
35498 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
35506 /// X86ISD::AVG instruction.
35574 return DAG.getNode(X86ISD::AVG, DL, Ops[0].getValueType(), Ops);
35583 // demote it and emit X86ISD::AVG instruction.
35612 // The pattern is detected, emit X86ISD::AVG instruction(s).
35891 SDValue NewVec = getExtendInVec(X86ISD::VSEXT, dl, VT, WideLd, DAG);
35941 if (Mask.getOpcode() == X86ISD::PCMPGT &&
36137 // replace the trunc store by a normal store with the result of X86ISD::AVG
36472 auto NewOpcode = IsFadd ? X86ISD::FHADD : X86ISD::FHSUB;
36573 /// Truncate using ISD::AND mask and X86ISD::PACKUS.
36604 return truncateVectorWithPACK(X86ISD::PACKUS, OutVT, In, DL, DAG, Subtarget);
36607 /// Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS.
36616 return truncateVectorWithPACK(X86ISD::PACKSS, OutVT, In, DL, DAG, Subtarget);
36620 /// X86ISD::PACKUS/X86ISD::PACKSS operations. We do it here because after type
36637 // TODO: On AVX2, the behavior of X86ISD::PACKUS is different from that on
36670 /// vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
36708 return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget);
36714 return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget);
36905 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
36942 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
36963 if (Op.getOpcode() != X86ISD::FXOR && Op.getOpcode() != ISD::XOR)
37002 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
37012 case ISD::FMA: NewOpcode = X86ISD::FNMSUB; break;
37013 case X86ISD::FMSUB: NewOpcode = X86ISD::FNMADD; break;
37014 case X86ISD::FNMADD: NewOpcode = X86ISD::FMSUB; break;
37015 case X86ISD::FNMSUB: NewOpcode = ISD::FMA; break;
37016 case X86ISD::FMADD_RND: NewOpcode = X86ISD::FNMSUB_RND; break;
37017 case X86ISD::FMSUB_RND: NewOpcode = X86ISD::FNMADD_RND; break;
37018 case X86ISD::FNMADD_RND: NewOpcode = X86ISD::FMSUB_RND; break;
37019 case X86ISD::FNMSUB_RND: NewOpcode = X86ISD::FMADD_RND; break;
37046 case X86ISD::FOR: IntOpcode = ISD::OR; break;
37047 case X86ISD::FXOR: IntOpcode = ISD::XOR; break;
37048 case X86ISD::FAND: IntOpcode = ISD::AND; break;
37049 case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
37065 if (!RHSC || RHSC->getZExtValue() != 1 || LHS->getOpcode() != X86ISD::SETCC)
37081 MVT::v4i32, DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32,
37125 return DAG.getNode(X86ISD::BEXTR, SDLoc(N), VT, Op0,
37182 if (N0.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N0.getOperand(1)))
37183 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
37186 if (N1.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N1.getOperand(1)))
37187 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
37192 /// Do target-specific dag combines on X86ISD::FAND nodes.
37209 /// Do target-specific dag combines on X86ISD::FANDN nodes.
37223 /// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
37226 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
37243 /// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
37245 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
37256 case X86ISD::FMIN: NewOp = X86ISD::FMINC; break;
37257 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
37281 auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN;
37323 /// Do target-specific dag combines on X86ISD::ANDNP nodes.
37358 return DAG.getNode(X86ISD::BT, SDLoc(N), MVT::i32, N0, DemandedN1);
37379 if (N0.getOpcode() != X86ISD::CMOV || !N0.hasOneUse())
37401 return DAG.getNode(X86ISD::CMOV, DL, VT, CMovOp0, CMovOp1,
37426 // it may be replaced with X86ISD::VSEXT node.
37524 auto DivRemOpcode = OpcodeN0 == ISD::SDIVREM ? X86ISD::SDIVREM8_SEXT_HREG
37525 : X86ISD::UDIVREM8_ZEXT_HREG;
37549 if (CMovN.getOpcode() != X86ISD::CMOV || !CMovN.hasOneUse())
37582 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
37741 // ISD::*_EXTEND_VECTOR_INREG which ensures lowering to X86ISD::V*EXT.
37877 case ISD::FMA: Opcode = X86ISD::FNMADD; break;
37878 case X86ISD::FMADD_RND: Opcode = X86ISD::FNMADD_RND; break;
37879 case X86ISD::FMSUB: Opcode = X86ISD::FNMSUB; break;
37880 case X86ISD::FMSUB_RND: Opcode = X86ISD::FNMSUB_RND; break;
37881 case X86ISD::FNMADD: Opcode = ISD::FMA; break;
37882 case X86ISD::FNMADD_RND: Opcode = X86ISD::FMADD_RND; break;
37883 case X86ISD::FNMSUB: Opcode = X86ISD::FMSUB; break;
37884 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMSUB_RND; break;
37891 case ISD::FMA: Opcode = X86ISD::FMSUB; break;
37892 case X86ISD::FMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
37893 case X86ISD::FMSUB: Opcode = ISD::FMA; break;
37894 case X86ISD::FMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
37895 case X86ISD::FNMADD: Opcode = X86ISD::FNMSUB; break;
37896 case X86ISD::FNMADD_RND: Opcode = X86ISD::FNMSUB_RND; break;
37897 case X86ISD::FNMSUB: Opcode = X86ISD::FNMADD; break;
37898 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FNMADD_RND; break;
37971 case X86ISD::FMADDSUB: NewOpcode = X86ISD::FMSUBADD; break;
37972 case X86ISD::FMADDSUB_RND: NewOpcode = X86ISD::FMSUBADD_RND; break;
37973 case X86ISD::FMSUBADD: NewOpcode = X86ISD::FMADDSUB; break;
37974 case X86ISD::FMSUBADD_RND: NewOpcode = X86ISD::FMADDSUB_RND; break;
37987 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
37988 // (and (i32 x86isd::setcc_carry), 1)
37999 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
38003 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
38013 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
38015 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
38111 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
38192 // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early
38314 // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
38340 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
38492 return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs,
38500 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
38515 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
38526 return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs,
38540 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, CF, EFLAGS);
38573 if (!IsSub && !PeekedThroughZext && X.getOpcode() == X86ISD::SETCC &&
38574 Y.getOpcode() != X86ISD::SETCC)
38577 if (Y.getOpcode() != X86ISD::SETCC || !Y.hasOneUse())
38593 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
38601 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
38608 X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
38611 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
38636 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
38639 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
38654 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
38673 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
38674 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
38686 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
38687 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
38694 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
38702 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
38707 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
38749 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, Ops);
38957 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
39081 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
39111 return DAG.getNode(X86ISD::HADD, DL, Ops[0].getValueType(), Ops);
39167 return DAG.getNode(X86ISD::SUBUS, DL, Ops[0].getValueType(), Ops);
39242 return DAG.getNode(X86ISD::HSUB, DL, Ops[0].getValueType(), Ops);
39285 (Opcode == X86ISD::VZEXT) || (Opcode == ISD::ZERO_EXTEND_VECTOR_INREG);
39300 if (Opcode == X86ISD::VZEXT && V != Op && V.getOpcode() == X86ISD::VZEXT) {
39308 return DAG.getNode(X86ISD::VZEXT, DL, VT, V.getOperand(0));
39319 return DAG.getNode(X86ISD::VZEXT, DL, VT, DAG.getBitcast(OpVT, V));
39325 // TODO: Add X86ISD::VSEXT support
39326 if (Opcode == X86ISD::VZEXT &&
39343 return DAG.getNode(X86ISD::VZEXT, DL, VT, Op);
39356 if (N->getOpcode() == X86ISD::PCMPEQ)
39358 if (N->getOpcode() == X86ISD::PCMPGT)
39495 return DAG.getNode(X86ISD::SUBV_BROADCAST, dl, OpVT, SubVec);
39499 if (SubVec.getOpcode() == X86ISD::SUBV_BROADCAST && SubVec == SubVec2)
39500 return DAG.getNode(X86ISD::SUBV_BROADCAST, dl, OpVT,
39560 return DAG.getNode(X86ISD::CVTSI2P, SDLoc(N), OpVT, InVec.getOperand(0));
39565 return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), OpVT, InVec.getOperand(0));
39568 if ((InOpcode == X86ISD::VZEXT || InOpcode == X86ISD::VSEXT) &&
39571 unsigned ExtOp = InOpcode == X86ISD::VZEXT ? ISD::ZERO_EXTEND_VECTOR_INREG
39632 case X86ISD::PEXTRW:
39633 case X86ISD::PEXTRB:
39641 case X86ISD::SHRUNKBLEND: return combineSelect(N, DAG, DCI, Subtarget);
39643 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
39646 case X86ISD::SBB: return combineSBB(N, DAG);
39647 case X86ISD::ADC: return combineADC(N, DAG, DCI);
39655 case X86ISD::BEXTR: return combineBEXTR(N, DAG, DCI, Subtarget);
39666 case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget);
39667 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget);
39668 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget);
39669 case X86ISD::FXOR:
39670 case X86ISD::FOR: return combineFOr(N, DAG, Subtarget);
39671 case X86ISD::FMIN:
39672 case X86ISD::FMAX: return combineFMinFMax(N, DAG);
39675 case X86ISD::BT: return combineBT(N, DAG, DCI);
39681 case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget);
39682 case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget);
39683 case X86ISD::PACKSS:
39684 case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget);
39685 case X86ISD::VSHLI:
39686 case X86ISD::VSRAI:
39687 case X86ISD::VSRLI:
39691 case X86ISD::VSEXT:
39692 case X86ISD::VZEXT: return combineVSZext(N, DAG, DCI, Subtarget);
39693 case X86ISD::PINSRB:
39694 case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget);
39695 case X86ISD::SHUFP: // Handle all target specific shuffles
39696 case X86ISD::INSERTPS:
39697 case X86ISD::EXTRQI:
39698 case X86ISD::INSERTQI:
39699 case X86ISD::PALIGNR:
39700 case X86ISD::VSHLDQ:
39701 case X86ISD::VSRLDQ:
39702 case X86ISD::BLENDI:
39703 case X86ISD::UNPCKH:
39704 case X86ISD::UNPCKL:
39705 case X86ISD::MOVHLPS:
39706 case X86ISD::MOVLHPS:
39707 case X86ISD::PSHUFB:
39708 case X86ISD::PSHUFD:
39709 case X86ISD::PSHUFHW:
39710 case X86ISD::PSHUFLW:
39711 case X86ISD::MOVSHDUP:
39712 case X86ISD::MOVSLDUP:
39713 case X86ISD::MOVDDUP:
39714 case X86ISD::MOVSS:
39715 case X86ISD::MOVSD:
39716 case X86ISD::VBROADCAST:
39717 case X86ISD::VPPERM:
39718 case X86ISD::VPERMI:
39719 case X86ISD::VPERMV:
39720 case X86ISD::VPERMV3:
39721 case X86ISD::VPERMIL2:
39722 case X86ISD::VPERMILPI:
39723 case X86ISD::VPERMILPV:
39724 case X86ISD::VPERM2X128:
39725 case X86ISD::SHUF128:
39726 case X86ISD::VZEXT_MOVL:
39728 case X86ISD::FMADD_RND:
39729 case X86ISD::FMSUB:
39730 case X86ISD::FMSUB_RND:
39731 case X86ISD::FNMADD:
39732 case X86ISD::FNMADD_RND:
39733 case X86ISD::FNMSUB:
39734 case X86ISD::FNMSUB_RND:
39736 case X86ISD::FMADDSUB_RND:
39737 case X86ISD::FMSUBADD_RND:
39738 case X86ISD::FMADDSUB:
39739 case X86ISD::FMSUBADD: return combineFMADDSUB(N, DAG, Subtarget);
39740 case X86ISD::MOVMSK: return combineMOVMSK(N, DAG, DCI);
39741 case X86ISD::MGATHER:
39742 case X86ISD::MSCATTER:
39745 case X86ISD::PCMPEQ:
39746 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget);
39747 case X86ISD::PMULDQ:
39748 case X86ISD::PMULUDQ: return combinePMULDQ(N, DAG, DCI);
39798 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr);
