Lines Matching refs:X86ISD
1770 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy());
1983 return DAG.getNode(X86ISD::RET_FLAG, dl, MVT::Other, RetOps);
2006 if (UI->getOpcode() != X86ISD::RET_FLAG)
2320 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
2478 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
2758 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), getPointerTy())));
2914 WrapperKind = X86ISD::WrapperRIP;
2923 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee);
2990 return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
2993 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
3365 case X86ISD::PSHUFD:
3366 case X86ISD::PSHUFHW:
3367 case X86ISD::PSHUFLW:
3368 case X86ISD::SHUFP:
3369 case X86ISD::PALIGNR:
3370 case X86ISD::MOVLHPS:
3371 case X86ISD::MOVLHPD:
3372 case X86ISD::MOVHLPS:
3373 case X86ISD::MOVLPS:
3374 case X86ISD::MOVLPD:
3375 case X86ISD::MOVSHDUP:
3376 case X86ISD::MOVSLDUP:
3377 case X86ISD::MOVDDUP:
3378 case X86ISD::MOVSS:
3379 case X86ISD::MOVSD:
3380 case X86ISD::UNPCKL:
3381 case X86ISD::UNPCKH:
3382 case X86ISD::VPERMILP:
3383 case X86ISD::VPERM2X128:
3384 case X86ISD::VPERMI:
3393 case X86ISD::MOVSHDUP:
3394 case X86ISD::MOVSLDUP:
3395 case X86ISD::MOVDDUP:
3405 case X86ISD::PSHUFD:
3406 X86ISD::PSHUFHW:
3407 case X86ISD::PSHUFLW:
3408 case X86ISD::VPERMILP:
3409 case X86ISD::VPERMI:
3419 case X86ISD::PALIGNR:
3420 case X86ISD::SHUFP:
3421 case X86ISD::VPERM2X128:
3431 case X86ISD::MOVLHPS:
3432 case X86ISD::MOVLHPD:
3433 case X86ISD::MOVHLPS:
3434 case X86ISD::MOVLPS:
3435 case X86ISD::MOVLPD:
3436 case X86ISD::MOVSS:
3437 case X86ISD::MOVSD:
3438 case X86ISD::UNPCKL:
3439 case X86ISD::UNPCKH:
5121 case X86ISD::SHUFP:
5125 case X86ISD::UNPCKH:
5128 case X86ISD::UNPCKL:
5131 case X86ISD::MOVHLPS:
5134 case X86ISD::MOVLHPS:
5137 case X86ISD::PALIGNR:
5141 case X86ISD::PSHUFD:
5142 case X86ISD::VPERMILP:
5147 case X86ISD::PSHUFHW:
5152 case X86ISD::PSHUFLW:
5157 case X86ISD::VPERMI:
5162 case X86ISD::MOVSS:
5163 case X86ISD::MOVSD: {
5173 case X86ISD::VPERM2X128:
5178 case X86ISD::MOVDDUP:
5179 case X86ISD::MOVLHPD:
5180 case X86ISD::MOVLPD:
5181 case X86ISD::MOVLPS:
5182 case X86ISD::MOVSHDUP:
5183 case X86ISD::MOVSLDUP:
5545 return DAG.getNode(X86ISD::INSERTPS, dl, VT, V, V, InsertpsMask);
5558 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
5720 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64,
5807 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Sc);
5854 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5864 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5871 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
5877 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
6390 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6393 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6397 return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
6400 return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
6416 return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
6424 return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
6437 X86Opcode = X86ISD::HADD;
6444 X86Opcode = X86ISD::HSUB;
6472 X86Opcode = X86ISD::HADD;
6474 X86Opcode = X86ISD::HSUB;
6476 X86Opcode = X86ISD::FHADD;
6478 X86Opcode = X86ISD::FHSUB;
6972 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V1,
6979 return DAG.getNode(X86ISD::SHUFP, SDLoc(Op), MVT::v2f64, V1, V2,
7010 DAG.getNode(X86ISD::PSHUFD, SDLoc(Op), MVT::v4i32, V1,
7049 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
7072 V2 = DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V2, V1,
7103 V1 = DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V2,
7115 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, LowV, HighV,
7137 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
7216 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
7398 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
7401 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
7405 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
7420 V = DAG.getNode(X86ISD::PSHUFLW, DL, MVT::v8i16, V,
7428 V = DAG.getNode(X86ISD::PSHUFHW, DL, MVT::v8i16, V,
7580 MVT::v8i16, DL, DAG.getNode(MergeFromLo ? X86ISD::UNPCKL : X86ISD::UNPCKH,
7647 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v8i16, Evens, Odds);
7664 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, LoV, HiV));
7764 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
7803 return DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, Evens, Odds);
7857 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
7859 DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
7873 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
8085 SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, V1, V2,
8138 return DAG.getNode(X86ISD::PSHUFB, dl, ShufVT, V1,
8265 unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW;
8324 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16,
8348 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16,
8439 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
8457 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
8628 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
8638 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
8919 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64,
8933 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
8937 getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32,
8955 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
8986 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
8991 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG);
9004 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
9005 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
9011 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V2, V1,
9089 return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, LoadScalarToVector,
9102 return DAG.getNode(X86ISD::INSERTPS, dl, VT, To, From, InsertpsMask);
9190 DAG.getNode(X86ISD::VZEXT, DL, NVT, V1));
9309 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
9311 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
9323 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
9330 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG);
9335 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG);
9338 return getTargetShuffleNode(X86ISD::VPERMILP, dl, VT, V1, TargetMask,
9341 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V1,
9346 return getTargetShuffleNode(X86ISD::PALIGNR, dl, VT, V1, V2,
9368 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
9371 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
9383 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG);
9386 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG);
9432 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
9435 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
9444 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
9446 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
9457 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V2, DAG);
9460 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V2, DAG);
9474 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
9478 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1,
9483 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1,
9493 return getTargetShuffleNode(X86ISD::SHUFP, dl, VT, V1, V2,
9497 return getTargetShuffleNode(X86ISD::UNPCKL, dl, VT, V1, V1, DAG);
9499 return getTargetShuffleNode(X86ISD::UNPCKH, dl, VT, V1, V1, DAG);
9508 return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG);
9513 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1,
9515 return getTargetShuffleNode(X86ISD::VPERMILP, dl, VT, V1,
9526 return getTargetShuffleNode(X86ISD::VPERM2X128, dl, VT, V1,
9534 return getTargetShuffleNode(X86ISD::VPERMI, dl, VT, V1, Imm8, DAG);
9548 return DAG.getNode(X86ISD::VPERMV, dl, VT,
9550 return DAG.getNode(X86ISD::VPERMV3, dl, VT, V1,
9672 SDValue Ret = DAG.getNode(X86ISD::BLENDI, dl, BlendVT, LHS, RHS,
9708 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
9725 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
9787 Vec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, Vec,
9789 Vec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, Vec,
9791 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec,
9817 SDValue Mask = DAG.getNode(X86ISD::VINSERT, dl, MaskVT,
9820 SDValue Perm = DAG.getNode(X86ISD::VPERMV, dl, VecVT, Mask, Vec);
9866 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT,
9925 Opc = X86ISD::PINSRW;
9927 Opc = X86ISD::PINSRB;
9929 Opc = X86ISD::PINSRB;
9944 // the '3' into bits [7:6] of X86ISD::INSERTPS.
9952 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
9986 return DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
9990 EltInVec = DAG.getNode(X86ISD::VSHLI, dl, VecVT, EltInVec,
9992 EltInVec = DAG.getNode(X86ISD::VSRLI, dl, VecVT, EltInVec,
10043 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
10132 // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
10144 unsigned WrapperKind = X86ISD::Wrapper;
10149 WrapperKind = X86ISD::WrapperRIP;
10163 DAG.getNode(X86ISD::GlobalBaseReg,
10177 unsigned WrapperKind = X86ISD::Wrapper;
10182 WrapperKind = X86ISD::WrapperRIP;
10196 DAG.getNode(X86ISD::GlobalBaseReg,
10210 unsigned WrapperKind = X86ISD::Wrapper;
10217 WrapperKind = X86ISD::WrapperRIP;
10235 DAG.getNode(X86ISD::GlobalBaseReg,
10263 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
10265 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
10270 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
10297 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
10299 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
10304 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
10342 X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
10343 : X86ISD::TLSADDR;
10367 DAG.getNode(X86ISD::GlobalBaseReg,
10400 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
10411 unsigned WrapperKind = X86ISD::Wrapper;
10438 unsigned WrapperKind = X86ISD::Wrapper;
10444 WrapperKind = X86ISD::WrapperRIP;
10463 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
10506 X86ISD::WrapperRIP : X86ISD::Wrapper;
10525 DAG.getNode(X86ISD::GlobalBaseReg,
10534 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
10605 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), TGA);
10626 // X86ISD::SHLD and X86ISD::SHRD have defined overflow behavior but the
10637 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
10640 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
10649 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
10658 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
10659 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
10661 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0);
10662 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1);
10727 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
10728 X86ISD::FILD, DL,
10751 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys,
10814 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
10817 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32,
10946 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
11013 Opc = X86ISD::WIN_FTOL;
11017 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
11018 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
11019 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
11040 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
11050 if (Opc != X86ISD::WIN_FTOL) {
11057 SDValue ftol = DAG.getNode(X86ISD::WIN_FTOL, DL,
11098 return DAG.getNode(X86ISD::VZEXT, dl, VT, In);
11126 return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
11140 SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, DL, ExtVT, In, Ld);
11143 return DAG.getNode(X86ISD::VTRUNC, DL, VT, Brcst);
11200 return DAG.getNode(X86ISD::VTRUNC, DL, VT, In);
11218 SDValue OneV = DAG.getNode(X86ISD::VBROADCAST, DL, InVT, Ld);
11220 return DAG.getNode(X86ISD::TESTM, DL, VT, And, And);
11263 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV);
11365 return DAG.getNode(X86ISD::VFPEXT, DL, VT,
11402 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
11438 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
11482 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
11488 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit,
11515 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2);
11518 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
11526 // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1).
11527 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
11621 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
11650 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
11692 // return DAG.getNode(X86ISD::CMP, dl, MVT::i1, Op,
11694 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
11748 Opcode = X86ISD::INC;
11755 Opcode = X86ISD::DEC;
11762 Opcode = X86ISD::ADD;
11790 // If the primary and result isn't used, don't bother using X86ISD::AND,
11808 case ISD::SUB: Opcode = X86ISD::SUB; break;
11809 case ISD::XOR: Opcode = X86ISD::XOR; break;
11810 case ISD::AND: Opcode = X86ISD::AND; break;
11817 Opcode = X86ISD::OR;
11824 case X86ISD::ADD:
11825 case X86ISD::SUB:
11826 case X86ISD::INC:
11827 case X86ISD::DEC:
11828 case X86ISD::OR:
11829 case X86ISD::XOR:
11830 case X86ISD::AND:
11849 case ISD::ADD: ConvertedOp = X86ISD::ADD; break;
11850 case ISD::SUB: ConvertedOp = X86ISD::SUB; break;
11851 case ISD::AND: ConvertedOp = X86ISD::AND; break;
11852 case ISD::OR: ConvertedOp = X86ISD::OR; break;
11853 case ISD::XOR: ConvertedOp = X86ISD::XOR; break;
11868 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
11910 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs,
11914 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
11923 Cmp.getOpcode() != X86ISD::CMP ||
11934 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
11938 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
12008 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
12010 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
12112 case ISD::SETEQ: Opc = X86ISD::PCMPEQM; break;
12115 case ISD::SETGT: Opc = X86ISD::PCMPGTM; break;
12127 Opc = Unsigned ? X86ISD::CMPMU: X86ISD::CMPM;
12179 unsigned Opc = X86ISD::CMPP;
12182 Opc = X86ISD::CMPM;
12238 case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break;
12240 case ISD::SETGT: Opc = X86ISD::PCMPGT; break;
12242 case ISD::SETLE: Opc = X86ISD::PCMPGT;
12245 case ISD::SETUGT: Opc = X86ISD::PCMPGT;
12248 case ISD::SETULE: Opc = X86ISD::PCMPGT;
12261 case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break;
12262 case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break;
12297 Opc = X86ISD::SUBUS;
12308 if (Opc == X86ISD::PCMPGT && !Subtarget->hasSSE42()) {
12331 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
12332 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
12350 if (Opc == X86ISD::PCMPEQ && !Subtarget->hasSSE41()) {
12390 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
12393 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
12434 if (Op0.getOpcode() == X86ISD::SETCC) {
12442 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
12465 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
12475 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
12476 Opc == X86ISD::SAHF)
12479 (Opc == X86ISD::ADD ||
12480 Opc == X86ISD::SUB ||
12481 Opc == X86ISD::ADC ||
12482 Opc == X86ISD::SBB ||
12483 Opc == X86ISD::SMUL ||
12484 Opc == X86ISD::UMUL ||
12485 Opc == X86ISD::INC ||
12486 Opc == X86ISD::DEC ||
12487 Opc == X86ISD::OR ||
12488 Opc == X86ISD::XOR ||
12489 Opc == X86ISD::AND))
12492 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
12530 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1,
12532 return DAG.getNode(X86ISD::SELECT, DL, VT, Cmp, Op1, Op2);
12534 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
12536 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
12537 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
12538 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
12552 if (Cond.getOpcode() == X86ISD::SETCC &&
12553 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
12571 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, VTs,
12574 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
12580 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
12585 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
12600 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
12606 // If condition flag is set by a X86ISD::CMP, then use it as the condition
12607 // setting operand in place of the X86ISD::SETCC.
12609 if (CondOpcode == X86ISD::SETCC ||
12610 CondOpcode == X86ISD::SETCC_CARRY) {
12623 Opc == X86ISD::BT) { // FIXME
12637 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
12638 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
12639 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
12640 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
12641 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
12642 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
12688 if (Cond.getOpcode() == X86ISD::SUB) {
12694 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
12712 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VTs, T2, T1, CC, Cond);
12717 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
12721 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops);
12735 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
12749 SDValue Brcst = DAG.getNode(X86ISD::VBROADCASTM, dl, ExtVT, In, Ld);
12752 return DAG.getNode(X86ISD::VTRUNC, dl, VT, Brcst);
12771 return DAG.getNode(X86ISD::VSEXT, dl, VT, In);
12800 OpLo = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpLo);
12801 OpHi = DAG.getNode(X86ISD::VSEXT, dl, HalfVT, OpHi);
12807 // ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart
12813 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
12815 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
12819 // isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and
12826 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
12863 else if (Cond.getOpcode() == X86ISD::ADD ||
12864 Cond.getOpcode() == X86ISD::SUB ||
12865 Cond.getOpcode() == X86ISD::SMUL ||
12866 Cond.getOpcode() == X86ISD::UMUL)
12872 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
12878 // If condition flag is set by a X86ISD::CMP, then use it as the condition
12879 // setting operand in place of the X86ISD::SETCC.
12881 if (CondOpcode == X86ISD::SETCC ||
12882 CondOpcode == X86ISD::SETCC_CARRY) {
12888 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
12915 // instructions that can't be removed afterwards (i.e. X86ISD::ADD and
12916 // X86ISD::INC).
12918 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
12922 X86Opcode = X86ISD::INC; X86Cond = X86::COND_O;
12925 X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
12926 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
12930 X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O;
12933 X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
12934 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
12935 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
12966 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
12997 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
13038 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
13042 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
13068 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
13072 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
13106 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
13190 SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
13202 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
13331 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl,
13378 if (Opc == X86ISD::VSRAI)
13384 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)
13397 case X86ISD::VSHLI:
13409 case X86ISD::VSRLI:
13421 case X86ISD::VSRAI:
13456 case X86ISD::VSHLI: Opc = X86ISD::VSHL; break;
13457 case X86ISD::VSRLI: Opc = X86ISD::VSRL; break;
13458 case X86ISD::VSRAI: Opc = X86ISD::VSRA; break;
13514 Opc = X86ISD::COMI;
13519 Opc = X86ISD::COMI;
13524 Opc = X86ISD::COMI;
13529 Opc = X86ISD::COMI;
13534 Opc = X86ISD::COMI;
13539 Opc = X86ISD::COMI;
13544 Opc = X86ISD::UCOMI;
13549 Opc = X86ISD::UCOMI;
13554 Opc = X86ISD::UCOMI;
13559 Opc = X86ISD::UCOMI;
13564 Opc = X86ISD::UCOMI;
13569 Opc = X86ISD::UCOMI;
13579 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
13587 return DAG.getNode(X86ISD::PMULUDQ, dl, Op.getValueType(),
13592 return DAG.getNode(X86ISD::PMULDQ, dl, Op.getValueType(),
13610 return DAG.getNode(X86ISD::SUBUS, dl, Op.getValueType(),
13637 Opcode = X86ISD::FHADD;
13643 Opcode = X86ISD::FHSUB;
13649 Opcode = X86ISD::HADD;
13655 Opcode = X86ISD::HSUB;
13696 Opcode = X86ISD::UMAX;
13704 Opcode = X86ISD::UMIN;
13712 Opcode = X86ISD::SMAX;
13720 Opcode = X86ISD::SMIN;
13743 Opcode = X86ISD::FMAX;
13749 Opcode = X86ISD::FMIN;
13795 return DAG.getNode(X86ISD::PACKSS, dl, Op.getValueType(),
13802 return DAG.getNode(X86ISD::PACKUS, dl, Op.getValueType(),
13807 return DAG.getNode(X86ISD::PSHUFB, dl, Op.getValueType(),
13811 return DAG.getNode(X86ISD::PSHUFD, dl, Op.getValueType(),
13815 return DAG.getNode(X86ISD::PSHUFLW, dl, Op.getValueType(),
13819 return DAG.getNode(X86ISD::PSHUFHW, dl, Op.getValueType(),
13828 return DAG.getNode(X86ISD::PSIGN, dl, Op.getValueType(),
13832 return DAG.getNode(X86ISD::INSERTPS, dl, Op.getValueType(),
13839 return DAG.getNode(X86ISD::VPERM2X128, dl, Op.getValueType(),
13846 return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(),
13914 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
13917 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
13926 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
13927 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test);
13957 Opcode = X86ISD::VSHL;
13965 Opcode = X86ISD::VSRL;
13971 Opcode = X86ISD::VSRA;
14004 Opcode = X86ISD::VSHLI;
14012 Opcode = X86ISD::VSRLI;
14018 Opcode = X86ISD::VSRAI;
14040 Opcode = X86ISD::PCMPISTRI;
14044 Opcode = X86ISD::PCMPESTRI;
14048 Opcode = X86ISD::PCMPISTRI;
14052 Opcode = X86ISD::PCMPESTRI;
14056 Opcode = X86ISD::PCMPISTRI;
14060 Opcode = X86ISD::PCMPESTRI;
14064 Opcode = X86ISD::PCMPISTRI;
14068 Opcode = X86ISD::PCMPESTRI;
14072 Opcode = X86ISD::PCMPISTRI;
14076 Opcode = X86ISD::PCMPESTRI;
14083 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
14093 Opcode = X86ISD::PCMPISTRI;
14095 Opcode = X86ISD::PCMPESTRI;
14146 Opc = X86ISD::FMADD;
14154 Opc = X86ISD::FMSUB;
14162 Opc = X86ISD::FNMADD;
14170 Opc = X86ISD::FNMSUB;
14178 Opc = X86ISD::FMADDSUB;
14186 Opc = X86ISD::FMSUBADD;
14282 SDValue rd = DAG.getNode(X86ISD::RDPMC_DAG, DL, Tys, Chain);
14338 if (Opcode == X86ISD::RDTSCP_DAG) {
14372 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
14455 IntrinsicData(RDRAND, X86ISD::RDRAND, 0)));
14457 IntrinsicData(RDRAND, X86ISD::RDRAND, 0)));
14459 IntrinsicData(RDRAND, X86ISD::RDRAND, 0)));
14461 IntrinsicData(RDSEED, X86ISD::RDSEED, 0)));
14463 IntrinsicData(RDSEED, X86ISD::RDSEED, 0)));
14465 IntrinsicData(RDSEED, X86ISD::RDSEED, 0)));
14467 IntrinsicData(XTEST, X86ISD::XTEST, 0)));
14469 IntrinsicData(RDTSC, X86ISD::RDTSC_DAG, 0)));
14471 IntrinsicData(RDTSC, X86ISD::RDTSCP_DAG, 0)));
14473 IntrinsicData(RDPMC, X86ISD::RDPMC_DAG, 0)));
14500 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl,
14558 SDValue InTrans = DAG.getNode(X86ISD::XTEST, dl, VTs, Op.getOperand(0));
14559 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
14663 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
14670 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
14678 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
14873 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
14919 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
14928 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
14953 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
14971 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
14980 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
15049 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B);
15051 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds);
15076 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
15077 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
15087 SDValue AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
15088 SDValue AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
15089 SDValue AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
15091 AloBhi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AloBhi, 32, DAG);
15092 AhiBlo = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, AhiBlo, 32, DAG);
15169 (!IsSigned || !Subtarget->hasSSE41()) ? X86ISD::PMULUDQ : X86ISD::PMULDQ;
15224 return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
15227 return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
15230 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
15237 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
15250 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
15265 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
15283 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl,
15296 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl,
15311 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
15363 return getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, R, ShiftAmt,
15366 return getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt,
15369 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, ShiftAmt,
15459 return getTargetVShiftNode(X86ISD::VSHLI, dl, VT, R, BaseShAmt, DAG);
15470 return getTargetVShiftNode(X86ISD::VSRAI, dl, VT, R, BaseShAmt, DAG);
15483 return getTargetVShiftNode(X86ISD::VSRLI, dl, VT, R, BaseShAmt, DAG);
15510 return DAG.getNode(X86ISD::VSHL, dl, VT, R, Op.getOperand(1));
15512 return DAG.getNode(X86ISD::VSRL, dl, VT, R, Op.getOperand(1));
15514 return DAG.getNode(X86ISD::VSRA, dl, VT, R, Op.getOperand(1));
15608 // lowered as X86ISD::VSRLI nodes. This would be cheaper than scalarizing
15613 unsigned TargetOpcode = X86ISD::MOVSS;
15631 TargetOpcode = X86ISD::MOVSD;
15642 TargetOpcode = X86ISD::MOVSD;
15662 if (TargetOpcode == X86ISD::MOVSD)
15682 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
15689 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 4, DAG);
15696 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
15700 M = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, MVT::v8i16, M, 2, DAG);
15707 OpVSel = DAG.getNode(X86ISD::PCMPEQ, dl, VT, OpVSel, VSelM);
15786 BaseOp = X86ISD::INC;
15790 BaseOp = X86ISD::ADD;
15794 BaseOp = X86ISD::ADD;
15802 BaseOp = X86ISD::DEC;
15806 BaseOp = X86ISD::SUB;
15810 BaseOp = X86ISD::SUB;
15814 BaseOp = X86ISD::SMUL;
15820 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
15823 DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
15836 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
15902 assert(Tmp1Op0.getOpcode() == X86ISD::VZEXT &&
15904 return DAG.getNode(X86ISD::VSEXT, dl, VT, Tmp1Op0.getOperand(0));
15911 Tmp1 = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Op0, BitsDiff,
15913 return getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Tmp1, BitsDiff,
15934 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
15952 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
15980 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
15987 SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
16101 case ISD::ADDC: Opc = X86ISD::ADD; break;
16102 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
16103 case ISD::SUBC: Opc = X86ISD::SUB; break;
16104 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
16341 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
16347 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
16357 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
16360 return getReadTimeStampCounter(N, dl, X86ISD::RDTSCP_DAG, DAG, Subtarget,
16367 return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
16402 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG :
16403 X86ISD::LCMPXCHG8_DAG;
16416 DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
16479 case X86ISD::BSF: return "X86ISD::BSF";
16480 case X86ISD::BSR: return "X86ISD::BSR";
16481 case X86ISD::SHLD: return "X86ISD::SHLD";
16482 case X86ISD::SHRD: return "X86ISD::SHRD";
16483 case X86ISD::FAND: return "X86ISD::FAND";
16484 case X86ISD::FANDN: return "X86ISD::FANDN";
16485 case X86ISD::FOR: return "X86ISD::FOR";
16486 case X86ISD::FXOR: return "X86ISD::FXOR";
16487 case X86ISD::FSRL: return "X86ISD::FSRL";
16488 case X86ISD::FILD: return "X86ISD::FILD";
16489 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
16490 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
16491 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
16492 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
16493 case X86ISD::FLD: return "X86ISD::FLD";
16494 case X86ISD::FST: return "X86ISD::FST";
16495 case X86ISD::CALL: return "X86ISD::CALL";
16496 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
16497 case X86ISD::RDTSCP_DAG: return "X86ISD::RDTSCP_DAG";
16498 case X86ISD::RDPMC_DAG: return "X86ISD::RDPMC_DAG";
16499 case X86ISD::BT: return "X86ISD::BT";
16500 case X86ISD::CMP: return "X86ISD::CMP";
16501 case X86ISD::COMI: return "X86ISD::COMI";
16502 case X86ISD::UCOMI: return "X86ISD::UCOMI";
16503 case X86ISD::CMPM: return "X86ISD::CMPM";
16504 case X86ISD::CMPMU: return "X86ISD::CMPMU";
16505 case X86ISD::SETCC: return "X86ISD::SETCC";
16506 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
16507 case X86ISD::FSETCC: return "X86ISD::FSETCC";
16508 case X86ISD::CMOV: return "X86ISD::CMOV";
16509 case X86ISD::BRCOND: return "X86ISD::BRCOND";
16510 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
16511 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
16512 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
16513 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
16514 case X86ISD::Wrapper: return "X86ISD::Wrapper";
16515 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
16516 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
16517 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
16518 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
16519 case X86ISD::PINSRB: return "X86ISD::PINSRB";
16520 case X86ISD::PINSRW: return "X86ISD::PINSRW";
16521 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
16522 case X86ISD::ANDNP: return "X86ISD::ANDNP";
16523 case X86ISD::PSIGN: return "X86ISD::PSIGN";
16524 case X86ISD::BLENDV: return "X86ISD::BLENDV";
16525 case X86ISD::BLENDI: return "X86ISD::BLENDI";
16526 case X86ISD::SUBUS: return "X86ISD::SUBUS";
16527 case X86ISD::HADD: return "X86ISD::HADD";
16528 case X86ISD::HSUB: return "X86ISD::HSUB";
16529 case X86ISD::FHADD: return "X86ISD::FHADD";
16530 case X86ISD::FHSUB: return "X86ISD::FHSUB";
16531 case X86ISD::UMAX: return "X86ISD::UMAX";
16532 case X86ISD::UMIN: return "X86ISD::UMIN";
16533 case X86ISD::SMAX: return "X86ISD::SMAX";
16534 case X86ISD::SMIN: return "X86ISD::SMIN";
16535 case X86ISD::FMAX: return "X86ISD::FMAX";
16536 case X86ISD::FMIN: return "X86ISD::FMIN";
16537 case X86ISD::FMAXC: return "X86ISD::FMAXC";
16538 case X86ISD::FMINC: return "X86ISD::FMINC";
16539 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
16540 case X86ISD::FRCP: return "X86ISD::FRCP";
16541 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
16542 case X86ISD::TLSBASEADDR: return "X86ISD::TLSBASEADDR";
16543 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
16544 case X86ISD::EH_SJLJ_SETJMP: return "X86ISD::EH_SJLJ_SETJMP";
16545 case X86ISD::EH_SJLJ_LONGJMP: return "X86ISD::EH_SJLJ_LONGJMP";
16546 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
16547 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
16548 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
16549 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r";
16550 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
16551 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
16552 case X86ISD::LCMPXCHG16_DAG: return "X86ISD::LCMPXCHG16_DAG";
16553 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
16554 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
16555 case X86ISD::VZEXT: return "X86ISD::VZEXT";
16556 case X86ISD::VSEXT: return "X86ISD::VSEXT";
16557 case X86ISD::VTRUNC: return "X86ISD::VTRUNC";
16558 case X86ISD::VTRUNCM: return "X86ISD::VTRUNCM";
16559 case X86ISD::VINSERT: return "X86ISD::VINSERT";
16560 case X86ISD::VFPEXT: return "X86ISD::VFPEXT";
16561 case X86ISD::VFPROUND: return "X86ISD::VFPROUND";
16562 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ";
16563 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ";
16564 case X86ISD::VSHL: return "X86ISD::VSHL";
16565 case X86ISD::VSRL: return "X86ISD::VSRL";
16566 case X86ISD::VSRA: return "X86ISD::VSRA";
16567 case X86ISD::VSHLI: return "X86ISD::VSHLI";
16568 case X86ISD::VSRLI: return "X86ISD::VSRLI";
16569 case X86ISD::VSRAI: return "X86ISD::VSRAI";
16570 case X86ISD::CMPP: return "X86ISD::CMPP";
16571 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ";
16572 case X86ISD::PCMPGT: return "X86ISD::PCMPGT";
16573 case X86ISD::PCMPEQM: return "X86ISD::PCMPEQM";
16574 case X86ISD::PCMPGTM: return "X86ISD::PCMPGTM";
16575 case X86ISD::ADD: return "X86ISD::ADD";
16576 case X86ISD::SUB: return "X86ISD::SUB";
16577 case X86ISD::ADC: return "X86ISD::ADC";
16578 case X86ISD::SBB: return "X86ISD::SBB";
16579 case X86ISD::SMUL: return "X86ISD::SMUL";
16580 case X86ISD::UMUL: return "X86ISD::UMUL";
16581 case X86ISD::INC: return "X86ISD::INC";
16582 case X86ISD::DEC: return "X86ISD::DEC";
16583 case X86ISD::OR: return "X86ISD::OR";
16584 case X86ISD::XOR: return "X86ISD::XOR";
16585 case X86ISD::AND: return "X86ISD::AND";
16586 case X86ISD::BEXTR: return "X86ISD::BEXTR";
16587 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
16588 case X86ISD::PTEST: return "X86ISD::PTEST";
16589 case X86ISD::TESTP: return "X86ISD::TESTP";
16590 case X86ISD::TESTM: return "X86ISD::TESTM";
16591 case X86ISD::TESTNM: return "X86ISD::TESTNM";
16592 case X86ISD::KORTEST: return "X86ISD::KORTEST";
16593 case X86ISD::PACKSS: return "X86ISD::PACKSS";
16594 case X86ISD::PACKUS: return "X86ISD::PACKUS";
16595 case X86ISD::PALIGNR: return "X86ISD::PALIGNR";
16596 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
16597 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
16598 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
16599 case X86ISD::SHUFP: return "X86ISD::SHUFP";
16600 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
16601 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD";
16602 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
16603 case X86ISD::MOVLPS: return "X86ISD::MOVLPS";
16604 case X86ISD::MOVLPD: return "X86ISD::MOVLPD";
16605 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
16606 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
16607 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
16608 case X86ISD::MOVSD: return "X86ISD::MOVSD";
16609 case X86ISD::MOVSS: return "X86ISD::MOVSS";
16610 case X86ISD::UNPCKL: return "X86ISD::UNPCKL";
16611 case X86ISD::UNPCKH: return "X86ISD::UNPCKH";
16612 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
16613 case X86ISD::VBROADCASTM: return "X86ISD::VBROADCASTM";
16614 case X86ISD::VEXTRACT: return "X86ISD::VEXTRACT";
16615 case X86ISD::VPERMILP: return "X86ISD::VPERMILP";
16616 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128";
16617 case X86ISD::VPERMV: return "X86ISD::VPERMV";
16618 case X86ISD::VPERMV3: return "X86ISD::VPERMV3";
16619 case X86ISD::VPERMIV3: return "X86ISD::VPERMIV3";
16620 case X86ISD::VPERMI: return "X86ISD::VPERMI";
16621 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ";
16622 case X86ISD::PMULDQ: return "X86ISD::PMULDQ";
16623 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
16624 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
16625 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
16626 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
16627 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
16628 case X86ISD::WIN_FTOL: return "X86ISD::WIN_FTOL";
16629 case X86ISD::SAHF: return "X86ISD::SAHF";
16630 case X86ISD::RDRAND: return "X86ISD::RDRAND";
16631 case X86ISD::RDSEED: return "X86ISD::RDSEED";
16632 case X86ISD::FMADD: return "X86ISD::FMADD";
16633 case X86ISD::FMSUB: return "X86ISD::FMSUB";
16634 case X86ISD::FNMADD: return "X86ISD::FNMADD";
16635 case X86ISD::FNMSUB: return "X86ISD::FNMSUB";
16636 case X86ISD::FMADDSUB: return "X86ISD::FMADDSUB";
16637 case X86ISD::FMSUBADD: return "X86ISD::FMSUBADD";
16638 case X86ISD::PCMPESTRI: return "X86ISD::PCMPESTRI";
16639 case X86ISD::PCMPISTRI: return "X86ISD::PCMPISTRI";
16640 case X86ISD::XTEST: return "X86ISD::XTEST";
18219 case X86ISD::ADD:
18220 case X86ISD::SUB:
18221 case X86ISD::ADC:
18222 case X86ISD::SBB:
18223 case X86ISD::SMUL:
18224 case X86ISD::UMUL:
18225 case X86ISD::INC:
18226 case X86ISD::DEC:
18227 case X86ISD::OR:
18228 case X86ISD::XOR:
18229 case X86ISD::AND:
18234 case X86ISD::SETCC:
18274 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
18286 if (N->getOpcode() == X86ISD::Wrapper) {
18373 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
18435 case X86ISD::PSHUFD:
18437 case X86ISD::PSHUFLW:
18440 case X86ISD::PSHUFHW:
18458 assert(N.getOpcode() == X86ISD::PSHUFD &&
18474 case X86ISD::PSHUFD:
18478 case X86ISD::PSHUFLW:
18487 case X86ISD::PSHUFHW:
18496 case X86ISD::UNPCKL:
18497 case X86ISD::UNPCKH:
18505 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
18515 case X86ISD::PSHUFLW:
18516 case X86ISD::PSHUFHW:
18572 (N.getOpcode() == X86ISD::PSHUFLW || N.getOpcode() == X86ISD::PSHUFHW) &&
18589 case X86ISD::PSHUFLW:
18590 case X86ISD::PSHUFHW:
18597 case X86ISD::PSHUFD: {
18602 int DOffset = CombineOpcode == X86ISD::PSHUFLW ? 0 : 2;
18613 VMask[DOffset + Mask[0] / 2] < 2 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
18654 case X86ISD::PSHUFD:
18655 case X86ISD::PSHUFLW:
18656 case X86ISD::PSHUFHW:
18673 case X86ISD::PSHUFLW:
18674 case X86ISD::PSHUFHW:
18686 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
18691 V = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V,
18701 (V.getOpcode() == X86ISD::PSHUFLW ||
18702 V.getOpcode() == X86ISD::PSHUFHW) &&
18708 if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
18711 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
18712 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
18731 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
18732 : X86ISD::UNPCKH,
18740 case X86ISD::PSHUFD:
19008 return DAG.getNode(X86ISD::MMX_MOVD2W, SDLoc(InputVector),
19130 Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
19133 Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
19136 Opc = hasSigned ? X86ISD::SMIN : 0; break;
19139 Opc = hasSigned ? X86ISD::SMAX : 0; break;
19148 Opc = hasUnsigned ? X86ISD::UMAX : 0; break;
19151 Opc = hasUnsigned ? X86ISD::UMIN : 0; break;
19154 Opc = hasSigned ? X86ISD::SMAX : 0; break;
19157 Opc = hasSigned ? X86ISD::SMIN : 0; break;
19248 Opcode = X86ISD::FMIN;
19256 Opcode = X86ISD::FMIN;
19265 Opcode = X86ISD::FMIN;
19274 Opcode = X86ISD::FMAX;
19286 Opcode = X86ISD::FMAX;
19295 Opcode = X86ISD::FMAX;
19313 Opcode = X86ISD::FMIN;
19320 Opcode = X86ISD::FMIN;
19329 Opcode = X86ISD::FMIN;
19336 Opcode = X86ISD::FMAX;
19348 Opcode = X86ISD::FMAX;
19357 Opcode = X86ISD::FMAX;
19537 return DAG.getNode(X86ISD::SUBUS, DL, VT, OpLHS, OpRHS);
19550 X86ISD::SUBUS, DL, VT, OpLHS,
19564 X86ISD::SUBUS, DL, VT, OpLHS,
19672 return getTargetShuffleNode(X86ISD::MOVSS, DL, VT, A, B, DAG);
19673 return getTargetShuffleNode(X86ISD::MOVSD, DL, VT, A, B, DAG);
19710 SDValue Select = getTargetShuffleNode(X86ISD::MOVSD, DL, NVT, NewA,
19756 // We should generate an X86ISD::BLENDI from a vselect if its argument
19777 // X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
19793 if (Cmp.getOpcode() != X86ISD::CMP &&
19794 (Cmp.getOpcode() != X86ISD::SUB || Cmp.getNode()->hasAnyUseOfValue(0)))
19848 case X86ISD::SETCC_CARRY:
19858 case X86ISD::SETCC:
19864 case X86ISD::CMOV: {
19880 if ((Op.getOpcode() != X86ISD::RDRAND &&
19881 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
19908 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
19926 case X86ISD::BSR:
19927 case X86ISD::BSF:
19942 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(), Ops);
19962 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
19979 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
20017 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
20058 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
20072 return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops);
20229 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
20236 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
20257 if (N00.getOpcode() == X86ISD::SETCC_CARRY ||
20260 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) {
20353 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
20395 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00,
20402 SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
20578 return DAG.getNode(X86ISD::BEXTR, DL, VT, N0.getOperand(0),
20602 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
20608 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
20634 if (N0.getOpcode() == X86ISD::ANDNP)
20637 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) {
20670 } else if (Mask.getOpcode() == X86ISD::VSRAI) {
20687 Mask = DAG.getNode(X86ISD::PSIGN, DL, MaskVT, X, Mask.getOperand(0));
20739 unsigned Opc = X86ISD::SHLD;
20743 Opc = X86ISD::SHRD;
20797 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
20803 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops);
20872 // We generate X86ISD::VSEXT for SEXTLOADs if it's available, otherwise we'll
20970 SDValue Sext = DAG.getNode(X86ISD::VSEXT, dl, RegVT, SlicedVec);
21386 return DAG.getNode(X86ISD::FHADD, SDLoc(N), VT, LHS, RHS);
21401 return DAG.getNode(X86ISD::FHSUB, SDLoc(N), VT, LHS, RHS);
21405 /// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and
21406 /// X86ISD::FXOR nodes.
21408 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
21420 /// PerformFMinFMaxCombine - Do target-specific dag combines on X86ISD::FMIN and
21421 /// X86ISD::FMAX nodes.
21423 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
21434 case X86ISD::FMIN: NewOp = X86ISD::FMINC; break;
21435 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
21442 /// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes.
21455 /// PerformFANDNCombine - Do target-specific dag combines on X86ISD::FANDN nodes
21492 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
21521 // it may be replaced with X86ISD::VSEXT node.
21587 Opcode = (!NegC) ? X86ISD::FMADD : X86ISD::FMSUB;
21589 Opcode = (!NegC) ? X86ISD::FNMADD : X86ISD::FNMSUB;
21597 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
21598 // (and (i32 x86isd::setcc_carry), 1)
21609 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
21614 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
21624 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
21626 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
21720 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N->getOperand(0),
21731 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
21736 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
21740 // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
21755 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
21758 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
21777 return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
21799 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), Chain, Dest, Cond,
21837 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
21852 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
21875 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse())
21883 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
21889 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
21894 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
21897 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
21913 return DAG.getNode(X86ISD::HADD, SDLoc(N), VT, Op0, Op1);
21946 return DAG.getNode(X86ISD::HSUB, SDLoc(N), VT, Op0, Op1);
21960 if (In.getOpcode() != X86ISD::VZEXT)
21963 return DAG.getNode(X86ISD::VZEXT, SDLoc(N), N->getValueType(0),
21976 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI, Subtarget);
21979 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI);
21992 case X86ISD::FXOR:
21993 case X86ISD::FOR: return PerformFORCombine(N, DAG);
21994 case X86ISD::FMIN:
21995 case X86ISD::FMAX: return PerformFMinFMaxCombine(N, DAG);
21996 case X86ISD::FAND: return PerformFANDCombine(N, DAG);
21997 case X86ISD::FANDN: return PerformFANDNCombine(N, DAG);
21998 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI);
21999 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG);
22007 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG, DCI, Subtarget);
22008 case X86ISD::BRCOND: return PerformBrCondCombine(N, DAG, DCI, Subtarget);
22009 case X86ISD::VZEXT: return performVZEXTCombine(N, DAG, DCI, Subtarget);
22010 case X86ISD::SHUFP: // Handle all target specific shuffles
22011 case X86ISD::PALIGNR:
22012 case X86ISD::UNPCKH:
22013 case X86ISD::UNPCKL:
22014 case X86ISD::MOVHLPS:
22015 case X86ISD::MOVLHPS:
22016 case X86ISD::PSHUFD:
22017 case X86ISD::PSHUFHW:
22018 case X86ISD::PSHUFLW:
22019 case X86ISD::MOVSS:
22020 case X86ISD::MOVSD:
22021 case X86ISD::VPERMILP:
22022 case X86ISD::VPERM2X128:
22027 case X86ISD::INSERTPS:
