Lines Matching refs:X86ISD
1311 return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc(), getPointerTy());
1501 return DAG.getNode(X86ISD::RET_FLAG, dl,
1519 if (UI->getOpcode() != X86ISD::RET_FLAG)
1798 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(),
1945 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
2202 DAG.getNode(X86ISD::GlobalBaseReg,
2358 WrapperKind = X86ISD::WrapperRIP;
2367 Callee = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Callee);
2435 return DAG.getNode(X86ISD::TC_RETURN, dl,
2439 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size());
2786 case X86ISD::PSHUFD:
2787 case X86ISD::PSHUFHW:
2788 case X86ISD::PSHUFLW:
2789 case X86ISD::SHUFPD:
2790 case X86ISD::PALIGN:
2791 case X86ISD::SHUFPS:
2792 case X86ISD::MOVLHPS:
2793 case X86ISD::MOVLHPD:
2794 case X86ISD::MOVHLPS:
2795 case X86ISD::MOVLPS:
2796 case X86ISD::MOVLPD:
2797 case X86ISD::MOVSHDUP:
2798 case X86ISD::MOVSLDUP:
2799 case X86ISD::MOVDDUP:
2800 case X86ISD::MOVSS:
2801 case X86ISD::MOVSD:
2802 case X86ISD::UNPCKLPS:
2803 case X86ISD::UNPCKLPD:
2804 case X86ISD::VUNPCKLPSY:
2805 case X86ISD::VUNPCKLPDY:
2806 case X86ISD::PUNPCKLWD:
2807 case X86ISD::PUNPCKLBW:
2808 case X86ISD::PUNPCKLDQ:
2809 case X86ISD::PUNPCKLQDQ:
2810 case X86ISD::UNPCKHPS:
2811 case X86ISD::UNPCKHPD:
2812 case X86ISD::VUNPCKHPSY:
2813 case X86ISD::VUNPCKHPDY:
2814 case X86ISD::PUNPCKHWD:
2815 case X86ISD::PUNPCKHBW:
2816 case X86ISD::PUNPCKHDQ:
2817 case X86ISD::PUNPCKHQDQ:
2818 case X86ISD::VPERMILPS:
2819 case X86ISD::VPERMILPSY:
2820 case X86ISD::VPERMILPD:
2821 case X86ISD::VPERMILPDY:
2822 case X86ISD::VPERM2F128:
2832 case X86ISD::MOVSHDUP:
2833 case X86ISD::MOVSLDUP:
2834 case X86ISD::MOVDDUP:
2845 case X86ISD::PSHUFD:
2846 case X86ISD::PSHUFHW:
2847 case X86ISD::PSHUFLW:
2848 case X86ISD::VPERMILPS:
2849 case X86ISD::VPERMILPSY:
2850 case X86ISD::VPERMILPD:
2851 case X86ISD::VPERMILPDY:
2862 case X86ISD::PALIGN:
2863 case X86ISD::SHUFPD:
2864 case X86ISD::SHUFPS:
2865 case X86ISD::VPERM2F128:
2876 case X86ISD::MOVLHPS:
2877 case X86ISD::MOVLHPD:
2878 case X86ISD::MOVHLPS:
2879 case X86ISD::MOVLPS:
2880 case X86ISD::MOVLPD:
2881 case X86ISD::MOVSS:
2882 case X86ISD::MOVSD:
2883 case X86ISD::UNPCKLPS:
2884 case X86ISD::UNPCKLPD:
2885 case X86ISD::VUNPCKLPSY:
2886 case X86ISD::VUNPCKLPDY:
2887 case X86ISD::PUNPCKLWD:
2888 case X86ISD::PUNPCKLBW:
2889 case X86ISD::PUNPCKLDQ:
2890 case X86ISD::PUNPCKLQDQ:
2891 case X86ISD::UNPCKHPS:
2892 case X86ISD::UNPCKHPD:
2893 case X86ISD::VUNPCKHPSY:
2894 case X86ISD::VUNPCKHPDY:
2895 case X86ISD::PUNPCKHWD:
2896 case X86ISD::PUNPCKHBW:
2897 case X86ISD::PUNPCKHDQ:
2898 case X86ISD::PUNPCKHQDQ:
4543 case X86ISD::SHUFPS:
4544 case X86ISD::SHUFPD:
4550 case X86ISD::PUNPCKHBW:
4551 case X86ISD::PUNPCKHWD:
4552 case X86ISD::PUNPCKHDQ:
4553 case X86ISD::PUNPCKHQDQ:
4556 case X86ISD::UNPCKHPS:
4557 case X86ISD::UNPCKHPD:
4558 case X86ISD::VUNPCKHPSY:
4559 case X86ISD::VUNPCKHPDY:
4562 case X86ISD::PUNPCKLBW:
4563 case X86ISD::PUNPCKLWD:
4564 case X86ISD::PUNPCKLDQ:
4565 case X86ISD::PUNPCKLQDQ:
4568 case X86ISD::UNPCKLPS:
4569 case X86ISD::UNPCKLPD:
4570 case X86ISD::VUNPCKLPSY:
4571 case X86ISD::VUNPCKLPDY:
4574 case X86ISD::MOVHLPS:
4577 case X86ISD::MOVLHPS:
4580 case X86ISD::PSHUFD:
4586 case X86ISD::PSHUFHW:
4591 case X86ISD::PSHUFLW:
4596 case X86ISD::MOVSS:
4597 case X86ISD::MOVSD: {
4605 case X86ISD::VPERMILPS:
4610 case X86ISD::VPERMILPSY:
4615 case X86ISD::VPERMILPD:
4620 case X86ISD::VPERMILPDY:
4625 case X86ISD::VPERM2F128:
4630 case X86ISD::MOVDDUP:
4631 case X86ISD::MOVLHPD:
4632 case X86ISD::MOVLPD:
4633 case X86ISD::MOVLPS:
4634 case X86ISD::MOVSHDUP:
4635 case X86ISD::MOVSLDUP:
4636 case X86ISD::PALIGN:
4888 unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL;
5035 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, 2, MVT::i64,
5374 SDValue VecOp = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec);
5383 SDValue VecOp2 = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec);
5544 unsigned Opc = pshufhw ? X86ISD::PSHUFHW : X86ISD::PSHUFLW;
5577 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
5597 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
5628 NewV = getTargetShuffleNode(X86ISD::PSHUFLW, dl, MVT::v8i16,
5657 NewV = getTargetShuffleNode(X86ISD::PSHUFHW, dl, MVT::v8i16,
5744 V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
5761 V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
5904 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
5914 DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
6273 getTargetShuffleNode(X86ISD::MOVDDUP, dl, MVT::v2f64,
6287 return getTargetShuffleNode(X86ISD::MOVLHPD, dl, VT, V1, V2, DAG);
6291 getTargetShuffleNode(X86ISD::MOVLHPS, dl, MVT::v4f32,
6309 return getTargetShuffleNode(X86ISD::MOVHLPS, dl, VT, V1, V2, DAG);
6317 case MVT::v4f32: return X86ISD::SHUFPS;
6321 case MVT::v2f64: return X86ISD::SHUFPD;
6359 return getTargetShuffleNode(X86ISD::MOVLPD, dl, VT, V1, V2, DAG);
6362 return getTargetShuffleNode(X86ISD::MOVLPS, dl, VT, V1, V2, DAG);
6376 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
6377 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
6389 case MVT::v4i32: return X86ISD::PUNPCKLDQ;
6390 case MVT::v2i64: return X86ISD::PUNPCKLQDQ;
6391 case MVT::v4f32: return X86ISD::UNPCKLPS;
6392 case MVT::v2f64: return X86ISD::UNPCKLPD;
6394 case MVT::v8f32: return X86ISD::VUNPCKLPSY;
6396 case MVT::v4f64: return X86ISD::VUNPCKLPDY;
6397 case MVT::v16i8: return X86ISD::PUNPCKLBW;
6398 case MVT::v8i16: return X86ISD::PUNPCKLWD;
6407 case MVT::v4i32: return X86ISD::PUNPCKHDQ;
6408 case MVT::v2i64: return X86ISD::PUNPCKHQDQ;
6409 case MVT::v4f32: return X86ISD::UNPCKHPS;
6410 case MVT::v2f64: return X86ISD::UNPCKHPD;
6412 case MVT::v8f32: return X86ISD::VUNPCKHPSY;
6414 case MVT::v4f64: return X86ISD::VUNPCKHPDY;
6415 case MVT::v16i8: return X86ISD::PUNPCKHBW;
6416 case MVT::v8i16: return X86ISD::PUNPCKHWD;
6426 case MVT::v4f32: return X86ISD::VPERMILPS;
6428 case MVT::v2f64: return X86ISD::VPERMILPD;
6430 case MVT::v8f32: return X86ISD::VPERMILPSY;
6432 case MVT::v4f64: return X86ISD::VPERMILPDY;
6511 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, V1);
6616 return getTargetShuffleNode(X86ISD::MOVLHPS, dl, VT, V1, V1, DAG);
6621 return getTargetShuffleNode(X86ISD::PSHUFD, dl, VT, V1, TargetMask, DAG);
6648 return getTargetShuffleNode(X86ISD::MOVSD, dl, VT, V1, V2, DAG);
6651 return getTargetShuffleNode(X86ISD::MOVSS, dl, VT, V1, V2, DAG);
6663 return getTargetShuffleNode(X86ISD::MOVSHDUP, dl, VT, V1, DAG);
6666 return getTargetShuffleNode(X86ISD::MOVSLDUP, dl, VT, V1, DAG);
6754 return getTargetShuffleNode(X86ISD::PALIGN, dl, VT, V1, V2,
6761 return getTargetShuffleNode(X86ISD::UNPCKLPD, dl, VT, V1, V1, DAG);
6763 return getTargetShuffleNode(X86ISD::PUNPCKLQDQ, dl, VT, V1, V1, DAG);
6767 return getTargetShuffleNode(X86ISD::PSHUFHW, dl, VT, V1,
6772 return getTargetShuffleNode(X86ISD::PSHUFLW, dl, VT, V1,
6792 return getTargetShuffleNode(X86ISD::MOVDDUP, dl, VT, V1, DAG);
6806 return getTargetShuffleNode(X86ISD::VPERM2F128, dl, VT, V1, V2,
6860 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
6875 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
6957 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT,
7014 Opc = X86ISD::PINSRW;
7016 Opc = X86ISD::PINSRB;
7018 Opc = X86ISD::PINSRB;
7031 // the '3' into bits [7:6] of X86ISD::INSERTPS.
7039 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
7091 return DAG.getNode(X86ISD::PINSRW, dl, VT, N0, N1, N2);
7167 // their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
7179 unsigned WrapperKind = X86ISD::Wrapper;
7184 WrapperKind = X86ISD::WrapperRIP;
7198 DAG.getNode(X86ISD::GlobalBaseReg,
7212 unsigned WrapperKind = X86ISD::Wrapper;
7217 WrapperKind = X86ISD::WrapperRIP;
7231 DAG.getNode(X86ISD::GlobalBaseReg,
7245 unsigned WrapperKind = X86ISD::Wrapper;
7252 WrapperKind = X86ISD::WrapperRIP;
7271 DAG.getNode(X86ISD::GlobalBaseReg,
7298 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
7300 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
7305 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
7333 Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
7335 Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
7340 DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
7379 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3);
7382 Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2);
7399 DAG.getNode(X86ISD::GlobalBaseReg,
7432 unsigned WrapperKind = X86ISD::Wrapper;
7438 WrapperKind = X86ISD::WrapperRIP;
7494 X86ISD::WrapperRIP : X86ISD::Wrapper;
7513 DAG.getNode(X86ISD::GlobalBaseReg,
7522 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args, 2);
7560 Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
7563 Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
7569 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
7578 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
7579 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
7581 Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
7582 Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
7647 SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
7648 X86ISD::FILD, DL,
7672 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys,
7868 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops, 3,
7936 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
7937 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
7938 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
7957 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, 3,
8027 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
8061 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
8103 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
8109 SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit,
8132 SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2);
8135 return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
8143 // Lower ISD::FGETSIGN to (AND (X86ISD::FGETSIGNx86 ...) 1).
8144 SDValue xFGETSIGN = DAG.getNode(X86ISD::FGETSIGNx86, dl, VT, N0,
8177 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
8203 Opcode = X86ISD::INC;
8210 Opcode = X86ISD::DEC;
8217 Opcode = X86ISD::ADD;
8221 // If the primary and result isn't used, don't bother using X86ISD::AND,
8259 case ISD::SUB: Opcode = X86ISD::SUB; break;
8260 case ISD::OR: Opcode = X86ISD::OR; break;
8261 case ISD::XOR: Opcode = X86ISD::XOR; break;
8262 case ISD::AND: Opcode = X86ISD::AND; break;
8267 case X86ISD::ADD:
8268 case X86ISD::SUB:
8269 case X86ISD::INC:
8270 case X86ISD::DEC:
8271 case X86ISD::OR:
8272 case X86ISD::XOR:
8273 case X86ISD::AND:
8282 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
8304 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
8361 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
8363 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
8402 if (Op0.getOpcode() == X86ISD::SETCC) {
8409 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
8420 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
8472 unsigned Opc = EltVT == MVT::f32 ? X86ISD::CMPPS : X86ISD::CMPPD;
8540 case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break;
8541 case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break;
8542 case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break;
8543 case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break;
8564 if (Opc == X86ISD::PCMPGTQ && !Subtarget->hasSSE42() && !Subtarget->hasAVX())
8566 if (Opc == X86ISD::PCMPEQQ && !Subtarget->hasSSE41() && !Subtarget->hasAVX())
8594 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI)
8597 (Opc == X86ISD::ADD ||
8598 Opc == X86ISD::SUB ||
8599 Opc == X86ISD::ADC ||
8600 Opc == X86ISD::SBB ||
8601 Opc == X86ISD::SMUL ||
8602 Opc == X86ISD::UMUL ||
8603 Opc == X86ISD::INC ||
8604 Opc == X86ISD::DEC ||
8605 Opc == X86ISD::OR ||
8606 Opc == X86ISD::XOR ||
8607 Opc == X86ISD::AND))
8610 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
8644 if (Cond.getOpcode() == X86ISD::SETCC &&
8645 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
8656 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
8660 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
8675 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
8681 // If condition flag is set by a X86ISD::CMP, then use it as the condition
8682 // setting operand in place of the X86ISD::SETCC.
8683 if (Cond.getOpcode() == X86ISD::SETCC ||
8684 Cond.getOpcode() == X86ISD::SETCC_CARRY) {
8697 Opc == X86ISD::BT) { // FIXME
8729 if (Cond.getOpcode() == X86ISD::CMP) {
8734 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
8742 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
8746 return DAG.getNode(X86ISD::CMOV, DL, VTs, Ops, array_lengthof(Ops));
8750 // ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart
8756 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
8758 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
8762 // isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and
8769 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
8790 else if (Cond.getOpcode() == X86ISD::ADD ||
8791 Cond.getOpcode() == X86ISD::SUB ||
8792 Cond.getOpcode() == X86ISD::SMUL ||
8793 Cond.getOpcode() == X86ISD::UMUL)
8799 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
8805 // If condition flag is set by a X86ISD::CMP, then use it as the condition
8806 // setting operand in place of the X86ISD::SETCC.
8807 if (Cond.getOpcode() == X86ISD::SETCC ||
8808 Cond.getOpcode() == X86ISD::SETCC_CARRY) {
8814 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
8840 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
8871 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
8916 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
8964 SDValue Value = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
8976 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
9095 SDValue VAARG = DAG.getMemIntrinsicNode(X86ISD::VAARG_64, dl,
9166 Opc = X86ISD::COMI;
9171 Opc = X86ISD::COMI;
9176 Opc = X86ISD::COMI;
9181 Opc = X86ISD::COMI;
9186 Opc = X86ISD::COMI;
9191 Opc = X86ISD::COMI;
9196 Opc = X86ISD::UCOMI;
9201 Opc = X86ISD::UCOMI;
9206 Opc = X86ISD::UCOMI;
9211 Opc = X86ISD::UCOMI;
9216 Opc = X86ISD::UCOMI;
9221 Opc = X86ISD::UCOMI;
9231 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
9240 return DAG.getNode(X86ISD::FHADD, dl, Op.getValueType(),
9246 return DAG.getNode(X86ISD::FHSUB, dl, Op.getValueType(),
9307 unsigned TestOpc = IsTestPacked ? X86ISD::TESTP : X86ISD::PTEST;
9310 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
9486 return DAG.getNode(X86ISD::EH_RETURN, dl,
9683 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
9730 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
9739 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));
9763 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
9772 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));
10060 BaseOp = X86ISD::INC;
10064 BaseOp = X86ISD::ADD;
10068 BaseOp = X86ISD::ADD;
10076 BaseOp = X86ISD::DEC;
10080 BaseOp = X86ISD::SUB;
10084 BaseOp = X86ISD::SUB;
10088 BaseOp = X86ISD::SMUL;
10094 SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
10097 DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
10110 DAG.getNode(X86ISD::SETCC, DL, N->getValueType(1),
10185 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
10194 return DAG.getNode(X86ISD::SFENCE, dl, MVT::Other, Op.getOperand(0));
10198 return DAG.getNode(X86ISD::LFENCE, dl, MVT::Other, Op.getOperand(0));
10202 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
10220 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
10240 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
10269 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
10282 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
10370 case ISD::ADDC: Opc = X86ISD::ADD; break;
10371 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break;
10372 case ISD::SUBC: Opc = X86ISD::SUB; break;
10373 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break;
10534 SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
10577 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_DAG :
10578 X86ISD::LCMPXCHG8_DAG;
10593 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG);
10596 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG);
10599 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG);
10602 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG);
10605 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG);
10608 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG);
10611 ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG);
10621 case X86ISD::BSF: return "X86ISD::BSF";
10622 case X86ISD::BSR: return "X86ISD::BSR";
10623 case X86ISD::SHLD: return "X86ISD::SHLD";
10624 case X86ISD::SHRD: return "X86ISD::SHRD";
10625 case X86ISD::FAND: return "X86ISD::FAND";
10626 case X86ISD::FOR: return "X86ISD::FOR";
10627 case X86ISD::FXOR: return "X86ISD::FXOR";
10628 case X86ISD::FSRL: return "X86ISD::FSRL";
10629 case X86ISD::FILD: return "X86ISD::FILD";
10630 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
10631 case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
10632 case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
10633 case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
10634 case X86ISD::FLD: return "X86ISD::FLD";
10635 case X86ISD::FST: return "X86ISD::FST";
10636 case X86ISD::CALL: return "X86ISD::CALL";
10637 case X86ISD::RDTSC_DAG: return "X86ISD::RDTSC_DAG";
10638 case X86ISD::BT: return "X86ISD::BT";
10639 case X86ISD::CMP: return "X86ISD::CMP";
10640 case X86ISD::COMI: return "X86ISD::COMI";
10641 case X86ISD::UCOMI: return "X86ISD::UCOMI";
10642 case X86ISD::SETCC: return "X86ISD::SETCC";
10643 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
10644 case X86ISD::FSETCCsd: return "X86ISD::FSETCCsd";
10645 case X86ISD::FSETCCss: return "X86ISD::FSETCCss";
10646 case X86ISD::CMOV: return "X86ISD::CMOV";
10647 case X86ISD::BRCOND: return "X86ISD::BRCOND";
10648 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
10649 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
10650 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
10651 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
10652 case X86ISD::Wrapper: return "X86ISD::Wrapper";
10653 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
10654 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
10655 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
10656 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
10657 case X86ISD::PINSRB: return "X86ISD::PINSRB";
10658 case X86ISD::PINSRW: return "X86ISD::PINSRW";
10659 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
10660 case X86ISD::ANDNP: return "X86ISD::ANDNP";
10661 case X86ISD::PSIGNB: return "X86ISD::PSIGNB";
10662 case X86ISD::PSIGNW: return "X86ISD::PSIGNW";
10663 case X86ISD::PSIGND: return "X86ISD::PSIGND";
10664 case X86ISD::FMAX: return "X86ISD::FMAX";
10665 case X86ISD::FMIN: return "X86ISD::FMIN";
10666 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
10667 case X86ISD::FRCP: return "X86ISD::FRCP";
10668 case X86ISD::FHADD: return "X86ISD::FHADD";
10669 case X86ISD::FHSUB: return "X86ISD::FHSUB";
10670 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
10671 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
10672 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
10673 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
10674 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
10675 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
10676 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
10677 case X86ISD::ATOMADD64_DAG: return "X86ISD::ATOMADD64_DAG";
10678 case X86ISD::ATOMSUB64_DAG: return "X86ISD::ATOMSUB64_DAG";
10679 case X86ISD::ATOMOR64_DAG: return "X86ISD::ATOMOR64_DAG";
10680 case X86ISD::ATOMXOR64_DAG: return "X86ISD::ATOMXOR64_DAG";
10681 case X86ISD::ATOMAND64_DAG: return "X86ISD::ATOMAND64_DAG";
10682 case X86ISD::ATOMNAND64_DAG: return "X86ISD::ATOMNAND64_DAG";
10683 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
10684 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
10685 case X86ISD::VSHL: return "X86ISD::VSHL";
10686 case X86ISD::VSRL: return "X86ISD::VSRL";
10687 case X86ISD::CMPPD: return "X86ISD::CMPPD";
10688 case X86ISD::CMPPS: return "X86ISD::CMPPS";
10689 case X86ISD::PCMPEQB: return "X86ISD::PCMPEQB";
10690 case X86ISD::PCMPEQW: return "X86ISD::PCMPEQW";
10691 case X86ISD::PCMPEQD: return "X86ISD::PCMPEQD";
10692 case X86ISD::PCMPEQQ: return "X86ISD::PCMPEQQ";
10693 case X86ISD::PCMPGTB: return "X86ISD::PCMPGTB";
10694 case X86ISD::PCMPGTW: return "X86ISD::PCMPGTW";
10695 case X86ISD::PCMPGTD: return "X86ISD::PCMPGTD";
10696 case X86ISD::PCMPGTQ: return "X86ISD::PCMPGTQ";
10697 case X86ISD::ADD: return "X86ISD::ADD";
10698 case X86ISD::SUB: return "X86ISD::SUB";
10699 case X86ISD::ADC: return "X86ISD::ADC";
10700 case X86ISD::SBB: return "X86ISD::SBB";
10701 case X86ISD::SMUL: return "X86ISD::SMUL";
10702 case X86ISD::UMUL: return "X86ISD::UMUL";
10703 case X86ISD::INC: return "X86ISD::INC";
10704 case X86ISD::DEC: return "X86ISD::DEC";
10705 case X86ISD::OR: return "X86ISD::OR";
10706 case X86ISD::XOR: return "X86ISD::XOR";
10707 case X86ISD::AND: return "X86ISD::AND";
10708 case X86ISD::ANDN: return "X86ISD::ANDN";
10709 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
10710 case X86ISD::PTEST: return "X86ISD::PTEST";
10711 case X86ISD::TESTP: return "X86ISD::TESTP";
10712 case X86ISD::PALIGN: return "X86ISD::PALIGN";
10713 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
10714 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
10715 case X86ISD::PSHUFHW_LD: return "X86ISD::PSHUFHW_LD";
10716 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
10717 case X86ISD::PSHUFLW_LD: return "X86ISD::PSHUFLW_LD";
10718 case X86ISD::SHUFPS: return "X86ISD::SHUFPS";
10719 case X86ISD::SHUFPD: return "X86ISD::SHUFPD";
10720 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
10721 case X86ISD::MOVLHPD: return "X86ISD::MOVLHPD";
10722 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
10723 case X86ISD::MOVHLPD: return "X86ISD::MOVHLPD";
10724 case X86ISD::MOVLPS: return "X86ISD::MOVLPS";
10725 case X86ISD::MOVLPD: return "X86ISD::MOVLPD";
10726 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
10727 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
10728 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
10729 case X86ISD::MOVSHDUP_LD: return "X86ISD::MOVSHDUP_LD";
10730 case X86ISD::MOVSLDUP_LD: return "X86ISD::MOVSLDUP_LD";
10731 case X86ISD::MOVSD: return "X86ISD::MOVSD";
10732 case X86ISD::MOVSS: return "X86ISD::MOVSS";
10733 case X86ISD::UNPCKLPS: return "X86ISD::UNPCKLPS";
10734 case X86ISD::UNPCKLPD: return "X86ISD::UNPCKLPD";
10735 case X86ISD::VUNPCKLPDY: return "X86ISD::VUNPCKLPDY";
10736 case X86ISD::UNPCKHPS: return "X86ISD::UNPCKHPS";
10737 case X86ISD::UNPCKHPD: return "X86ISD::UNPCKHPD";
10738 case X86ISD::PUNPCKLBW: return "X86ISD::PUNPCKLBW";
10739 case X86ISD::PUNPCKLWD: return "X86ISD::PUNPCKLWD";
10740 case X86ISD::PUNPCKLDQ: return "X86ISD::PUNPCKLDQ";
10741 case X86ISD::PUNPCKLQDQ: return "X86ISD::PUNPCKLQDQ";
10742 case X86ISD::PUNPCKHBW: return "X86ISD::PUNPCKHBW";
10743 case X86ISD::PUNPCKHWD: return "X86ISD::PUNPCKHWD";
10744 case X86ISD::PUNPCKHDQ: return "X86ISD::PUNPCKHDQ";
10745 case X86ISD::PUNPCKHQDQ: return "X86ISD::PUNPCKHQDQ";
10746 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
10747 case X86ISD::VPERMILPS: return "X86ISD::VPERMILPS";
10748 case X86ISD::VPERMILPSY: return "X86ISD::VPERMILPSY";
10749 case X86ISD::VPERMILPD: return "X86ISD::VPERMILPD";
10750 case X86ISD::VPERMILPDY: return "X86ISD::VPERMILPDY";
10751 case X86ISD::VPERM2F128: return "X86ISD::VPERM2F128";
10752 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
10753 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
10754 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
10755 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
10756 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
12314 case X86ISD::ADD:
12315 case X86ISD::SUB:
12316 case X86ISD::ADC:
12317 case X86ISD::SBB:
12318 case X86ISD::SMUL:
12319 case X86ISD::UMUL:
12320 case X86ISD::INC:
12321 case X86ISD::DEC:
12322 case X86ISD::OR:
12323 case X86ISD::XOR:
12324 case X86ISD::AND:
12329 case X86ISD::SETCC:
12366 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
12378 if (N->getOpcode() == X86ISD::Wrapper) {
12534 return DAG.getNode(X86ISD::MMX_MOVW2D, V.getOperand(0).getDebugLoc(),
12551 return DAG.getNode(X86ISD::MMX_MOVD2W, InputVector.getDebugLoc(),
12668 Opcode = X86ISD::FMIN;
12676 Opcode = X86ISD::FMIN;
12685 Opcode = X86ISD::FMIN;
12694 Opcode = X86ISD::FMAX;
12706 Opcode = X86ISD::FMAX;
12715 Opcode = X86ISD::FMAX;
12733 Opcode = X86ISD::FMIN;
12740 Opcode = X86ISD::FMIN;
12749 Opcode = X86ISD::FMIN;
12756 Opcode = X86ISD::FMAX;
12768 Opcode = X86ISD::FMAX;
12777 Opcode = X86ISD::FMAX;
12882 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
12898 case X86ISD::BSR:
12899 case X86ISD::BSF:
12922 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
12939 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
12977 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
13049 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
13056 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
13076 if (N00.getOpcode() == X86ISD::SETCC_CARRY ||
13079 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) {
13229 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
13269 X86ISD::NodeType NTOperator = is64BitFP ?
13270 X86ISD::FSETCCsd : X86ISD::FSETCCss;
13339 return DAG.getNode(X86ISD::ANDN, DL, VT, N0.getOperand(0), N1);
13342 return DAG.getNode(X86ISD::ANDN, DL, VT, N1.getOperand(0), N0);
13362 return DAG.getNode(X86ISD::ANDNP, DL, VT, N0.getOperand(0), N1);
13368 return DAG.getNode(X86ISD::ANDNP, DL, VT, N1.getOperand(0), N0);
13394 if (N0.getOpcode() == X86ISD::ANDNP)
13397 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) {
13453 case 8: Opc = X86ISD::PSIGNB; break;
13454 case 16: Opc = X86ISD::PSIGNW; break;
13455 case 32: Opc = X86ISD::PSIGND; break;
13496 unsigned Opc = X86ISD::SHLD;
13500 Opc = X86ISD::SHRD;
13958 return DAG.getNode(X86ISD::FHADD, N->getDebugLoc(), VT, LHS, RHS);
13973 return DAG.getNode(X86ISD::FHSUB, N->getDebugLoc(), VT, LHS, RHS);
13977 /// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and
13978 /// X86ISD::FXOR nodes.
13980 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
13992 /// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes.
14029 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
14038 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
14039 // (and (i32 x86isd::setcc_carry), 1)
14049 if (N00.getOpcode() != X86ISD::SETCC_CARRY)
14055 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
14063 // Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
14074 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
14102 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
14117 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
14140 if (SetCC.getOpcode() != X86ISD::SETCC || !SetCC.hasOneUse())
14148 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
14154 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0,
14159 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB,
14162 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC,
14202 case X86ISD::CMOV: return PerformCMOVCombine(N, DAG, DCI);
14205 case X86ISD::ADC: return PerformADCCombine(N, DAG, DCI);
14217 case X86ISD::FXOR:
14218 case X86ISD::FOR: return PerformFORCombine(N, DAG);
14219 case X86ISD::FAND: return PerformFANDCombine(N, DAG);
14220 case X86ISD::BT: return PerformBTCombine(N, DAG, DCI);
14221 case X86ISD::VZEXT_MOVL: return PerformVZEXT_MOVLCombine(N, DAG);
14223 case X86ISD::SETCC: return PerformSETCCCombine(N, DAG);
14224 case X86ISD::SHUFPS: // Handle all target specific shuffles
14225 case X86ISD::SHUFPD:
14226 case X86ISD::PALIGN:
14227 case X86ISD::PUNPCKHBW:
14228 case X86ISD::PUNPCKHWD:
14229 case X86ISD::PUNPCKHDQ:
14230 case X86ISD::PUNPCKHQDQ:
14231 case X86ISD::UNPCKHPS:
14232 case X86ISD::UNPCKHPD:
14233 case X86ISD::VUNPCKHPSY:
14234 case X86ISD::VUNPCKHPDY:
14235 case X86ISD::PUNPCKLBW:
14236 case X86ISD::PUNPCKLWD:
14237 case X86ISD::PUNPCKLDQ:
14238 case X86ISD::PUNPCKLQDQ:
14239 case X86ISD::UNPCKLPS:
14240 case X86ISD::UNPCKLPD:
14241 case X86ISD::VUNPCKLPSY:
14242 case X86ISD::VUNPCKLPDY:
14243 case X86ISD::MOVHLPS:
14244 case X86ISD::MOVLHPS:
14245 case X86ISD::PSHUFD:
14246 case X86ISD::PSHUFHW:
14247 case X86ISD::PSHUFLW:
14248 case X86ISD::MOVSS:
14249 case X86ISD::MOVSD:
14250 case X86ISD::VPERMILPS:
14251 case X86ISD::VPERMILPSY:
14252 case X86ISD::VPERMILPD:
14253 case X86ISD::VPERMILPDY:
14254 case X86ISD::VPERM2F128:
