Lines Matching refs:Op
3227 static bool MayFoldLoad(SDValue Op) {
3228 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
3231 static bool MayFoldIntoStore(SDValue Op) {
3232 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
3449 // ZF PF CF op
4573 // load folding shufps op.
5133 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
5141 SDLoc dl(Op);
5159 MVT::i16, Op.getOperand(i-1));
5162 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
5181 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
5189 SDLoc dl(Op);
5203 MVT::v8i16, V, Op.getOperand(i),
5402 X86TargetLowering::LowerVectorBroadcast(SDValue Op, SelectionDAG &DAG) const {
5406 MVT VT = Op.getValueType().getSimpleVT();
5407 SDLoc dl(Op);
5415 switch (Op.getOpcode()) {
5422 if (!isSplatVector(Op.getNode()))
5425 Ld = Op.getOperand(0);
5438 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
5445 SDValue Sc = Op.getOperand(0);
5532 X86TargetLowering::buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) const {
5533 EVT VT = Op.getValueType();
5539 SDLoc DL(Op);
5540 unsigned NumElems = Op.getNumOperands();
5548 unsigned Opc = Op.getOperand(i).getOpcode();
5562 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
5563 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
5598 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
5607 X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const {
5609 EVT VT = Op.getValueType();
5613 SDLoc dl(Op);
5614 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
5622 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
5632 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
5633 SDValue In = Op.getOperand(idx);
5651 if (!isSplatVector(Op.getNode()))
5654 SDValue In = Op.getOperand(0);
5708 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
5709 SDLoc dl(Op);
5711 MVT VT = Op.getValueType().getSimpleVT();
5713 unsigned NumElems = Op.getNumOperands();
5717 return LowerBUILD_VECTORvXi1(Op, DAG);
5720 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
5724 return Op;
5732 if (Subtarget->hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
5734 return Op;
5739 SDValue Broadcast = LowerVectorBroadcast(Op, DAG);
5751 SDValue Elt = Op.getOperand(i);
5773 SDValue Item = Op.getOperand(Idx);
5845 X86::isZeroNode(Op.getOperand(0)) &&
5846 !X86::isZeroNode(Op.getOperand(1))) {
5850 VT, Op.getOperand(1)),
5882 SDValue Item = Op.getOperand(Idx);
5883 if (Op.getNode()->isOnlyUserOf(Item.getNode()))
5899 V.push_back(Op.getOperand(i));
5918 Op.getOperand(Idx));
5926 SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
5932 SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
5945 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
5980 V[i] = Op.getOperand(i);
5988 SDValue Sh = buildFromShuffleMostly(Op, DAG);
5995 if (Op.getOperand(0).getOpcode() != ISD::UNDEF)
5996 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
6001 if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue;
6003 Op.getOperand(i), DAG.getIntPtrConstant(i));
6012 if (Op.getOperand(i).getOpcode() != ISD::UNDEF)
6013 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
6045 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
6046 SDLoc dl(Op);
6047 MVT ResVT = Op.getValueType().getSimpleVT();
6052 SDValue V1 = Op.getOperand(0);
6053 SDValue V2 = Op.getOperand(1);
6061 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
6062 assert(Op.getNumOperands() == 2);
6066 return LowerAVXCONCAT_VECTORS(Op, DAG);
6131 LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget,
6133 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
6906 SDValue getMOVDDup(SDValue &Op, SDLoc &dl, SDValue V1, SelectionDAG &DAG) {
6907 EVT VT = Op.getValueType();
6917 SDValue getMOVLowToHigh(SDValue &Op, SDLoc &dl, SelectionDAG &DAG,
6919 SDValue V1 = Op.getOperand(0);
6920 SDValue V2 = Op.getOperand(1);
6921 EVT VT = Op.getValueType();
6936 SDValue getMOVHighToLow(SDValue &Op, SDLoc &dl, SelectionDAG &DAG) {
6937 SDValue V1 = Op.getOperand(0);
6938 SDValue V2 = Op.getOperand(1);
6939 EVT VT = Op.getValueType();
6952 SDValue getMOVLP(SDValue &Op, SDLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
6953 SDValue V1 = Op.getOperand(0);
6954 SDValue V2 = Op.getOperand(1);
6955 EVT VT = Op.getValueType();
6973 else if (MayFoldVectorLoad(V1) && MayFoldIntoStore(Op))
6976 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
7010 X86TargetLowering::LowerVectorIntExtend(SDValue Op, SelectionDAG &DAG) const {
7015 EVT VT = Op.getValueType();
7021 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
7022 SDLoc DL(Op);
7023 SDValue V1 = Op.getOperand(0);
7024 SDValue V2 = Op.getOperand(1);
7098 X86TargetLowering::NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG) const {
7099 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
7100 MVT VT = Op.getValueType().getSimpleVT();
7101 SDLoc dl(Op);
7102 SDValue V1 = Op.getOperand(0);
7103 SDValue V2 = Op
7111 SDValue Broadcast = LowerVectorBroadcast(Op, DAG);
7117 SDValue NewOp = LowerVectorIntExtend(Op, DAG);
7155 X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
7156 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
7157 SDValue V1 = Op.getOperand(0);
7158 SDValue V2 = Op.getOperand(1);
7159 MVT VT = Op.getValueType().getSimpleVT();
7160 SDLoc dl(Op);
7178 assert(!V1IsUndef && "Op 1 of shuffle should not be undef");
7197 SDValue NewOp = NormalizeVectorShuffle(Op, DAG);
7212 return getMOVDDup(Op, dl, V1, DAG);
7215 return getMOVHighToLow(Op, dl, DAG);
7274 return getMOVLowToHigh(Op, dl, DAG, HasSSE2);
7277 return getMOVHighToLow(Op, dl, DAG);
7286 return getMOVLP(Op, dl, DAG, HasSSE2);
7362 // nodes, and remove one by one until they don't return Op anymore.
7440 SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, Subtarget, DAG);
7469 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
7470 MVT VT = Op.getValueType().getSimpleVT();
7471 SDLoc dl(Op);
7473 if (!Op.getOperand(0).getValueType().getSimpleVT().is128BitVector())
7478 Op.getOperand(0), Op.getOperand(1));
7485 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
7492 Op.getOperand(0)),
7493 Op.getOperand(1)));
7495 Op.getOperand(0), Op.getOperand(1));
7507 if (!Op.hasOneUse())
7509 SDNode *User = *Op.getNode()->use_begin();
7511 (isa<ConstantSDNode>(Op.getOperand(1)) &&
7512 cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) &&
7518 Op.getOperand(0)),
7519 Op.getOperand(1));
7525 if (isa<ConstantSDNode>(Op.getOperand(1)))
7526 return Op;
7532 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
7534 SDLoc dl(Op);
7535 if (!isa<ConstantSDNode>(Op.getOperand(1)))
7538 SDValue Vec = Op.getOperand(0);
7544 SDValue Idx = Op.getOperand(1);
7556 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
7563 SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
7568 MVT VT = Op.getValueType().getSimpleVT();
7571 SDValue Vec = Op.getOperand(0);
7572 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
7578 Op.getOperand(1)));
7582 Op.getOperand(0), Op.getOperand(1));
7589 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
7591 return Op;
7595 MVT VVT = Op.getOperand(0).getValueType().getSimpleVT();
7596 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
7606 unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
7608 return Op;
7614 MVT VVT = Op.getOperand(0).getValueType().getSimpleVT();
7615 SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
7624 static SDValue LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
7625 MVT VT = Op.getValueType().getSimpleVT();
7627 SDLoc dl(Op);
7629 SDValue N0 = Op.getOperand(0);
7630 SDValue N1 = Op.getOperand(1);
7631 SDValue N2 = Op.getOperand(2);
7672 return Op;
7678 X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
7679 MVT VT = Op.getValueType().getSimpleVT();
7682 SDLoc dl(Op);
7683 SDValue N0 = Op.getOperand(0);
7684 SDValue N1 = Op.getOperand(1);
7685 SDValue N2 = Op.getOperand(2);
7709 return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG);
7726 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
7728 SDLoc dl(Op);
7729 MVT OpVT = Op.getValueType().getSimpleVT();
7740 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
7743 return Insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
7747 Op.getOperand(0).getValueType() == MVT::i64)
7748 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
7750 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
7759 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
7761 SDLoc dl(Op);
7762 SDValue In = Op.getOperand(0);
7763 SDValue Idx = Op.getOperand(1);
7765 EVT ResVT = Op.getValueType();
7785 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
7788 SDLoc dl(Op.getNode());
7789 SDValue Vec = Op.getNode()->getOperand(0);
7790 SDValue SubVec = Op.getNode()->getOperand(1);
7791 SDValue Idx = Op.getNode()->getOperand(2);
7793 if ((Op.getNode()->getValueType(0).is256BitVector() ||
7794 Op.getNode()->getValueType(0).is512BitVector()) &&
7801 if (Op.getNode()->getValueType(0).is512BitVector() &&
7818 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
7819 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
7851 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
7852 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
7884 X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
7885 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
7908 SDLoc DL(Op);
7930 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
7935 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
7936 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
7937 SDLoc dl(Op);
8004 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
8005 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
8006 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
8007 return LowerGlobalAddress(GV, SDLoc(Op), Offset, DAG);
8159 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
8161 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
8198 SDLoc DL(Op);
8299 SDValue X86TargetLowering::LowerShiftParts(SDValue Op, SelectionDAG &DAG) const{
8300 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
8301 EVT VT = Op.getValueType();
8303 SDLoc dl(Op);
8304 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
8305 SDValue ShOpLo = Op.getOperand(0);
8306 SDValue ShOpHi = Op.getOperand(1);
8307 SDValue ShAmt = Op.getOperand(2);
8313 if (Op.getOpcode() == ISD::SHL_PARTS) {
8331 if (Op.getOpcode() == ISD::SHL_PARTS) {
8343 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
8345 EVT SrcVT = Op.getOperand(0).getValueType();
8355 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType()))
8356 return Op;
8357 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
8359 return Op;
8362 SDLoc dl(Op);
8367 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
8371 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
8374 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
8378 SDLoc DL(Op);
8380 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
8384 Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
8414 unsigned SSFISize = Op.getValueType().getSizeInBits()/8;
8419 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
8428 Op.getValueType(), MMO);
8429 Result = DAG.getLoad(Op.getValueType(), DL, Chain, StackSlot,
8438 SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
8453 SDLoc dl(Op);
8473 Op.getOperand(0));
8505 SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
8507 SDLoc dl(Op);
8514 Op.getOperand(0));
8539 EVT DestVT = Op.getValueType();
8551 SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op,
8553 SDValue N0 = Op.getOperand(0);
8555 SDLoc dl(Op);
8563 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
8567 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
8569 SDValue N0 = Op.getOperand(0);
8570 SDLoc dl(Op);
8572 if (Op.getValueType().isVector())
8573 return lowerUINT_TO_FP_vec(Op, DAG);
8579 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0);
8582 EVT DstVT = Op.getValueType();
8584 return LowerUINT_TO_FP_i64(Op, DAG);
8586 return LowerUINT_TO_FP_i32(Op, DAG);
8596 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
8602 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
8607 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
8631 Op.getOperand(0), DAG.getConstant(0, MVT::i64),
8657 X86TargetLowering:: FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
8659 SDLoc DL(Op);
8661 EVT DstTy = Op.getValueType();
8674 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
8678 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
8700 SDValue Value = Op.getOperand(0);
8701 EVT TheVT = Op.getOperand(0).getValueType();
8709 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
8751 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
8753 MVT VT = Op->getValueType(0).getSimpleVT();
8754 SDValue In = Op->getOperand(0);
8756 SDLoc dl(Op);
8780 bool NeedZero = Op.getOpcode() == ISD::ZERO_EXTEND;
8793 SDValue X86TargetLowering::LowerANY_EXTEND(SDValue Op,
8796 SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
8803 SDValue X86TargetLowering::LowerZERO_EXTEND(SDValue Op,
8805 SDLoc DL(Op);
8806 MVT VT = Op.getValueType().getSimpleVT();
8807 SDValue In = Op.getOperand(0);
8811 SDValue Res = LowerAVXExtend(Op, DAG, Subtarget);
8836 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
8837 SDLoc DL(Op);
8838 MVT VT = Op.getValueType().getSimpleVT();
8839 SDValue In = Op.getOperand(0);
8935 "Invalid op");
8953 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
8955 MVT VT = Op.getValueType().getSimpleVT();
8958 return DAG.getNode(ISD::TRUNCATE, SDLoc(Op), VT,
8959 DAG.getNode(ISD::FP_TO_SINT, SDLoc(Op),
8960 MVT::v8i32, Op.getOperand(0)));
8964 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
8968 if (FIST.getNode() == 0) return Op;
8972 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
8980 SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op,
8982 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
8989 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
8997 static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) {
8998 SDLoc DL(Op);
8999 MVT VT = Op.getValueType().getSimpleVT();
9000 SDValue In = Op.getOperand(0);
9010 SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) const {
9012 SDLoc dl(Op);
9013 MVT VT = Op.getValueType().getSimpleVT();
9038 Op.getOperand(0)),
9041 return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
9044 SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const {
9046 SDLoc dl(Op);
9047 MVT VT = Op.getValueType().getSimpleVT();
9072 Op.getOperand(0)),
9076 return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
9079 SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
9081 SDValue Op0 = Op.getOperand(0);
9082 SDValue Op1 = Op.getOperand(1);
9083 SDLoc dl(Op);
9084 MVT VT = Op.getValueType().getSimpleVT();
9158 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
9159 SDValue N0 = Op.getOperand(0);
9160 SDLoc dl(Op);
9161 MVT VT = Op.getValueType().getSimpleVT();
9171 SDValue X86TargetLowering::LowerVectorAllZeroTest(SDValue Op,
9173 assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
9178 if (!Op->hasOneUse())
9181 SDNode *N = Op.getNode();
9264 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
9266 SDLoc dl(Op);
9288 if (Op.getResNo() != 0 || NeedOF || NeedCF)
9290 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
9291 DAG.getConstant(0, Op.getValueType()));
9300 SDValue ArithOp = Op;
9301 if (Op->getOpcode() == ISD::TRUNCATE && Op->hasOneUse()) {
9302 SDValue Arith = Op->getOperand(0);
9303 // Both the trunc and the arithmetic op need to have one user each.
9319 // which may be the result of a CAST. We use the variable 'Op', which is the
9333 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
9334 UE = Op.getNode()->use_end(); UI != UE; ++UI)
9365 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
9366 UE = Op.getNode()->use_end(); UI != UE; ++UI) {
9390 // Due to the ISEL shortcoming noted above, be conservative if this op is
9392 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
9393 UE = Op.getNode()->use_end(); UI != UE; ++UI)
9405 SDValue EFLAGS = LowerVectorAllZeroTest(Op, DAG);
9423 return SDValue(Op.getNode(), 1);
9430 // update 'Op'.
9432 EVT VT = Op.getValueType();
9433 SDValue WideVal = Op->getOperand(0);
9453 Op = DAG.getNode(ConvertedOp, dl, VT, V0, V1);
9460 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
9461 DAG.getConstant(0, Op.getValueType()));
9463 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
9466 Ops.push_back(Op.getOperand(i));
9469 DAG.ReplaceAllUsesWith(Op, New);
9640 static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
9641 MVT VT = Op.getValueType().getSimpleVT();
9643 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&
9647 SDLoc dl(Op);
9648 SDValue CC = Op.getOperand(2);
9651 SDValue LHS = Op.getOperand(0);
9656 SDValue RHS = Op.getOperand(1);
9664 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
9665 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
9668 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
9671 SDValue Op0 = Op.getOperand(0);
9672 SDValue Op1 = Op.getOperand(1);
9673 SDValue CC = Op.getOperand(2);
9674 MVT VT = Op.getValueType().getSimpleVT();
9676 bool isFP = Op.getOperand(1).getValueType().getSimpleVT().isFloatingPoint();
9677 SDLoc dl(Op);
9711 return Lower256IntVSETCC(Op, DAG);
9842 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
9844 MVT VT = Op.getValueType().getSimpleVT();
9846 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
9849 SDValue Op0 = Op.getOperand(0);
9850 SDValue Op1 = Op.getOperand(1);
9851 SDLoc dl(Op);
9852 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
9900 static bool isX86LogicalCmp(SDValue Op) {
9901 unsigned Opc = Op.getNode()->getOpcode();
9905 if (Op.getResNo() == 1 &&
9919 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
9940 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
9942 SDValue Cond = Op.getOperand(0);
9943 SDValue Op1 = Op.getOperand(1);
9944 SDValue Op2 = Op.getOperand(2);
9945 SDLoc DL(Op);
10002 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
10013 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
10043 MVT VT = Op.getValueType().getSimpleVT();
10122 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
10133 if (Op.getValueType() == MVT::i8 &&
10141 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
10147 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
10152 SDValue X86TargetLowering::LowerSIGN_EXTEND(SDValue Op,
10154 MVT VT = Op->getValueType(0).getSimpleVT();
10155 SDValue In = Op->getOperand(0);
10157 SDLoc dl(Op);
10202 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
10203 Opc = Op.getOpcode();
10206 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
10207 Op.getOperand(0).hasOneUse() &&
10208 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
10209 Op.getOperand(1).hasOneUse());
10214 static bool isXor1OfSetCC(SDValue Op) {
10215 if (Op.getOpcode() != ISD::XOR)
10217 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
10219 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
10220 Op.getOperand(0).hasOneUse();
10225 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
10227 SDValue Chain = Op.getOperand(0);
10228 SDValue Cond = Op.getOperand(1);
10229 SDValue Dest = Op.getOperand(2);
10230 SDLoc dl(Op);
10344 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
10358 Op.getNode()->hasOneUse()) {
10363 SDNode *User = *Op.getNode()->use_begin();
10375 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
10403 if (Op.getNode()->hasOneUse()) {
10404 SDNode *User = *Op.getNode()->use_begin();
10420 Op.getValueType(),
10434 if (Op.getNode()->hasOneUse()) {
10435 SDNode *User = *Op.getNode()->use_begin();
10450 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
10483 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
10493 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
10500 SDLoc dl(Op);
10503 SDValue Chain = Op.getOperand(0);
10504 SDValue Size = Op.getOperand(1);
10555 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
10559 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
10560 SDLoc DL(Op);
10567 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
10577 SDValue FIN = Op.getOperand(1);
10579 SDValue Store = DAG.getStore(Op.getOperand(0), DL,
10588 Store = DAG.getStore(Op.getOperand(0), DL,
10599 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN,
10609 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN,
10616 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
10622 assert(Op.getNode()->getNumOperands() == 4);
10623 SDValue Chain = Op.getOperand(0);
10624 SDValue SrcPtr = Op.getOperand(1);
10625 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
10626 unsigned Align = Op.getConstantOperandVal(3);
10627 SDLoc dl(Op);
10629 EVT ArgVT = Op.getNode()->getValueType(0);
10684 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget,
10688 SDValue Chain = Op.getOperand(0);
10689 SDValue DstPtr = Op.getOperand(1);
10690 SDValue SrcPtr = Op.getOperand(2);
10691 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
10692 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
10693 SDLoc DL(Op);
10746 static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) {
10747 SDLoc dl(Op);
10748 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
10842 SDValue LHS = Op.getOperand(1);
10843 SDValue RHS = Op.getOperand(2);
10855 return DAG.getNode(X86ISD::PMULUDQ, dl, Op.getValueType(),
10856 Op.getOperand(1), Op.getOperand(2));
10863 return DAG.getNode(X86ISD::SUBUS, dl, Op.getValueType(),
10864 Op.getOperand(1), Op.getOperand(2));
10911 return DAG.getNode(Opcode, dl, Op.getValueType(),
10912 Op.getOperand(1), Op.getOperand(2));
10976 return DAG.getNode(Opcode, dl, Op.getValueType(),
10977 Op.getOperand(1), Op.getOperand(2));
11005 return DAG.getNode(Opcode, dl, Op.getValueType(),
11006 Op.getOperand(1), Op.getOperand(2));
11040 return DAG.getNode(Opcode, dl, Op.getValueType(),
11041 Op.getOperand(1), Op.getOperand(2));
11046 return DAG.getNode(X86ISD::PSHUFB, dl, Op.getValueType(),
11047 Op.getOperand(1), Op.getOperand(2));
11055 return DAG.getNode(X86ISD::PSIGN, dl, Op.getValueType(),
11056 Op.getOperand(1), Op.getOperand(2));
11059 return DAG.getNode(X86ISD::INSERTPS, dl, Op.getValueType(),
11060 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
11066 return DAG.getNode(X86ISD::VPERM2X128, dl, Op.getValueType(),
11067 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
11073 return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(),
11074 Op.getOperand(2), Op.getOperand(1));
11080 return DAG.getNode(ISD::FSQRT, dl, Op.getValueType(), Op.getOperand(1));
11139 SDValue LHS = Op.getOperand(1);
11140 SDValue RHS = Op.getOperand(2);
11191 return DAG.getNode(Opcode, dl, Op.getValueType(),
11192 Op.getOperand(1), Op.getOperand(2));
11238 return getTargetVShiftNode(Opcode, dl, Op.getValueType(),
11239 Op.getOperand(1), Op.getOperand(2), DAG);
11297 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
11298 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
11314 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
11315 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
11383 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
11384 Op.getOperand(2), Op.getOperand(3));
11389 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) {
11390 SDLoc dl(Op);
11391 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
11407 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other);
11408 SDValue Result = DAG.getNode(Opcode, dl, VTs, Op.getOperand(0));
11412 SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
11413 DAG.getConstant(1, Op->getValueType(1)),
11417 DAG.getVTList(Op->getValueType(1), MVT::Glue),
11421 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
11427 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
11428 SDValue InTrans = DAG.getNode(X86ISD::XTEST, dl, VTs, Op.getOperand(0));
11432 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
11433 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
11439 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
11444 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
11445 SDLoc dl(Op);
11449 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
11465 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
11469 EVT VT = Op.getValueType();
11470 SDLoc dl(Op); // FIXME probably not meaningful
11471 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
11486 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
11493 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
11494 SDValue Chain = Op.getOperand(0);
11495 SDValue Offset = Op.getOperand(1);
11496 SDValue Handler = Op.getOperand(2);
11497 SDLoc dl (Op);
11520 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
11522 SDLoc DL(Op);
11525 Op.getOperand(0), Op.getOperand(1));
11528 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
11530 SDLoc DL(Op);
11532 Op.getOperand(0), Op.getOperand(1));
11535 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
11536 return Op.getOperand(0);
11539 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
11541 SDValue Root = Op.getOperand(0);
11542 SDValue Trmp = Op.getOperand(1); // trampoline
11543 SDValue FPtr = Op.getOperand(2); // nested function
11544 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
11545 SDLoc dl (Op);
11547 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
11608 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
11689 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
11714 EVT VT = Op.getValueType();
11715 SDLoc DL(Op);
11758 static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) {
11759 EVT VT = Op.getValueType();
11762 SDLoc dl(Op);
11764 Op = Op.getOperand(0);
11768 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
11773 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
11777 Op,
11780 Op.getValue(1)
11782 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));
11785 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
11788 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
11789 return Op;
11792 static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
11793 EVT VT = Op.getValueType();
11796 SDLoc dl(Op);
11798 Op = Op.getOperand(0);
11802 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
11807 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
11810 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
11813 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
11814 return Op;
11817 static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
11818 EVT VT = Op.getValueType();
11820 SDLoc dl(Op);
11821 Op = Op.getOperand(0);
11825 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
11829 Op,
11832 Op.getValue(1)
11839 static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
11840 EVT VT = Op.getValueType();
11846 SDLoc dl(Op);
11849 SDValue LHS = Op.getOperand(0);
11854 SDValue RHS = Op.getOperand(1);
11862 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
11863 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
11866 static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
11867 assert(Op.getValueType().is256BitVector() &&
11868 Op.getValueType().isInteger() &&
11870 return Lower256IntArith(Op, DAG);
11873 static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) {
11874 assert(Op.getValueType().is256BitVector() &&
11875 Op.getValueType().isInteger() &&
11877 return Lower256IntArith(Op, DAG);
11880 static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
11882 SDLoc dl(Op);
11883 EVT VT = Op.getValueType();
11887 return Lower256IntArith(Op, DAG);
11889 SDValue A = Op.getOperand(0);
11890 SDValue B = Op.getOperand(1);
11953 SDValue X86TargetLowering::LowerSDIV(SDValue Op, SelectionDAG &DAG) const {
11954 EVT VT = Op.getValueType();
11957 SDValue N0 = Op.getOperand(0);
11958 SDLoc dl(Op);
11961 BuildVectorSDNode *C = dyn_cast<BuildVectorSDNode>(Op.getOperand(1));
11998 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
12000 EVT VT = Op.getValueType();
12001 SDLoc dl(Op);
12002 SDValue R = Op.getOperand(0);
12003 SDValue Amt = Op.getOperand(1);
12014 if (Op.getOpcode() == ISD::SHL)
12017 if (Op.getOpcode() == ISD::SRL)
12020 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64)
12026 if (Op.getOpcode() == ISD::SHL) {
12038 if (Op.getOpcode() == ISD::SRL) {
12050 if (Op.getOpcode() == ISD::SRA) {
12070 if (Op.getOpcode() == ISD::SHL) {
12082 if (Op.getOpcode() == ISD::SRL) {
12094 if (Op.getOpcode() == ISD::SRA) {
12146 switch (Op.getOpcode()) {
12164 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
12166 EVT VT = Op.getValueType();
12167 SDLoc dl(Op);
12168 SDValue R = Op.getOperand(0);
12169 SDValue Amt = Op.getOperand(1);
12171 if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) ||
12174 ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) ||
12231 switch (Op.getOpcode()) {
12285 switch (Op.getOpcode()) {
12289 return DAG.getNode(X86ISD::VSHL, dl, VT, R, Op.getOperand(1));
12291 return DAG.getNode(X86ISD::VSRL, dl, VT, R, Op.getOperand(1));
12293 return DAG.getNode(X86ISD::VSRA, dl, VT, R, Op.getOperand(1));
12300 SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
12302 EVT VT = Op.getValueType();
12303 SDLoc dl(Op);
12304 SDValue R = Op.getOperand(0);
12305 SDValue Amt = Op.getOperand(1);
12311 V = LowerScalarImmediateShift(Op, DAG, Subtarget);
12315 V = LowerScalarVariableShift(Op, DAG, Subtarget);
12321 if (Op.getOpcode() == ISD::SRL &&
12324 return Op;
12325 if (Op.getOpcode() == ISD::SHL &&
12328 return Op;
12329 if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32))
12330 return Op;
12334 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
12335 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT));
12337 Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT));
12338 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op);
12339 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
12340 return DAG.getNode(ISD::MUL, dl, VT, Op, R);
12342 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) {
12346 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT));
12347 Op = DAG.getNode(ISD::BITCAST, dl, VT, Op);
12351 SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
12365 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
12366 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
12377 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op);
12378 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op);
12419 V1 = DAG.getNode(Op.getOpcode(), dl, NewVT, V1, Amt1);
12420 V2 = DAG.getNode(Op.getOpcode(), dl, NewVT, V2, Amt2);
12429 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
12434 SDNode *N = Op.getNode();
12439 SDLoc DL(Op);
12440 switch (Op.getOpcode()) {
12504 SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
12506 SDLoc dl(Op);
12507 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
12508 EVT VT = Op.getValueType();
12528 SDValue LHS = Op.getOperand(0);
12541 LHS1 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, Extra);
12542 LHS2 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, Extra);
12550 SDValue Op0 = Op.getOperand(0);
12571 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget *Subtarget,
12573 SDLoc dl(Op);
12575 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
12577 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
12586 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
12588 SDValue Chain = Op.getOperand(0);
12603 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
12604 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
12607 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget,
12609 EVT T = Op.getValueType();
12610 SDLoc DL(Op);
12623 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
12624 Op.getOperand(2), SDValue());
12626 Op.getOperand(1),
12627 Op.getOperand(3),
12631 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
12639 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
12643 SDValue TheChain = Op.getOperand(0);
12644 SDLoc dl(Op);
12658 SDValue X86TargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
12659 EVT SrcVT = Op.getOperand(0).getValueType();
12660 EVT DstVT = Op.getValueType();
12668 return Op;
12670 return Op;
12673 return Op;
12678 static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
12679 SDNode *Node = Op.getNode();
12694 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
12695 SDNode *Node = Op.getNode();
12716 return Op;
12719 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
12720 EVT VT = Op.getNode()->getValueType(0);
12730 switch (Op.getOpcode()) {
12739 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
12740 Op.getOperand(1));
12741 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
12742 Op.getOperand(1), Op.getOperand(2));
12745 SDValue X86TargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
12751 SDLoc dl(Op);
12752 SDValue Arg = Op.getOperand(0);
12798 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
12799 switch (Op.getOpcode()) {
12801 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op,DAG);
12802 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
12803 case ISD::ATOMIC_CMP_SWAP: return LowerCMP_SWAP(Op, Subtarget, DAG);
12804 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG);
12805 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG);
12806 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
12807 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
12808 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
12809 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
12810 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
12811 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
12812 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
12813 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
12814 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
12815 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
12816 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
12817 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
12818 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
12821 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
12822 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
12823 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
12824 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
12825 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, DAG);
12826 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, DAG);
12827 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, DAG);
12828 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
12829 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
12830 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
12831 case ISD::FABS: return LowerFABS(Op, DAG);
12832 case ISD::FNEG: return LowerFNEG(Op, DAG);
12833 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
12834 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
12835 case ISD::SETCC: return LowerSETCC(Op, DAG);
12836 case ISD::SELECT: return LowerSELECT(Op, DAG);
12837 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
12838 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
12839 case ISD::VASTART: return LowerVASTART(Op, DAG);
12840 case ISD::VAARG: return LowerVAARG(Op, DAG);
12841 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
12842 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
12843 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG);
12844 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
12845 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
12847 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
12848 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
12849 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
12850 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
12851 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
12852 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
12853 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
12854 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
12855 case ISD::CTLZ: return LowerCTLZ(Op, DAG);
12856 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ_ZERO_UNDEF(Op, DAG);
12857 case ISD::CTTZ: return LowerCTTZ(Op, DAG);
12858 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
12861 case ISD::SHL: return LowerShift(Op, DAG);
12867 case ISD::UMULO: return LowerXALUO(Op, DAG);
12868 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
12869 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
12873 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
12874 case ISD::ADD: return LowerADD(Op, DAG);
12875 case ISD::SUB: return LowerSUB(Op, DAG);
12876 case ISD::SDIV: return LowerSDIV(Op, DAG);
12877 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG);
13562 llvm_unreachable("Unhandled atomic-load-op opcode!");
13591 llvm_unreachable("Unhandled atomic-load-op opcode!");
13609 llvm_unreachable("Unhandled atomic-load-op opcode!");
13623 llvm_unreachable("Unhandled atomic-load-op opcode!");
13642 // dst = atomic-fetch-op MI.addr, MI.val
13651 // t2 = OP MI.val, t4
13676 "Expected atomic-load-op to have one memoperand");
13709 // t1 = OP MI.val, EAX
13762 llvm_unreachable("Unhandled atomic-load-op opcode!");
13852 "Invalid atomic-load-op transformation!");
13855 assert(CC != X86::COND_INVALID && "Invalid atomic-load-op transformation!");
13907 // dst = atomic-fetch-op MI.addr, MI.val
13918 // t2L = OP MI.val.lo, t4L
13919 // t2H = OP MI.val.hi, t4H
13991 // t2L = OP MI.val.lo, t4L
13992 // t2H = OP MI.val.hi, t4H
14222 MachineOperand &Op = MI->getOperand(i);
14223 if (!(Op.isReg() && Op.isImplicit()))
14224 MIB.addOperand(Op);
14259 MachineOperand &Op = MI->getOperand(i);
14260 if (!(Op.isReg() && Op.isImplicit()))
14261 MIB.addOperand(Op);
15257 MachineOperand &Op = MI->getOperand(0);
15258 if (Op.isReg()) {
15260 AM.Base.Reg = Op.getReg();
15263 AM.Base.FrameIndex = Op.getIndex();
15265 Op = MI->getOperand(1);
15266 if (Op.isImm())
15267 AM.Scale = Op.getImm();
15268 Op = MI->getOperand(2);
15269 if (Op.isImm())
15270 AM.IndexReg = Op.getImm();
15271 Op = MI->getOperand(3);
15272 if (Op.isGlobal()) {
15273 AM.GV = Op.getGlobal();
15275 AM.Disp = Op.getImm();
15397 void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op,
15403 unsigned Opc = Op.getOpcode();
15408 "Should use MaskedValueIsZero if you don't know whether Op"
15426 if (Op.getResNo() == 0)
15433 unsigned IntId = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
15464 unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
15467 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
15468 return Op.getValueType().getScalarType().getSizeInBits();
16256 if (unsigned Op = matchIntegerMINMAX(Cond, VT, LHS, RHS, DAG, Subtarget))
16257 return DAG.getNode(Op, DL, N->getValueType(0), LHS, RHS);
16336 // (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
16337 // (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
16338 // to (Op EFLAGS Cond)
16340 // (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
16341 // (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
16342 // to (Op EFLAGS !Cond)
16344 // where Op could be BRCOND or CMOV.
16428 SDValue Op = SetCC.getOperand(0);
16430 if (Op.getOpcode() == ISD::ZERO_EXTEND ||
16431 Op.getOpcode() == ISD::TRUNCATE)
16432 Op = Op.getOperand(0);
16435 if ((Op.getOpcode() != X86ISD::RDRAND &&
16436 Op.getOpcode() != X86ISD::RDSEED) || Op
16962 SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, WideVT, N0, N1);
16966 return Op;
16972 Op, DAG.getConstant(Mask, VT));
16976 Op, DAG.getValueType(NarrowVT));
17714 /// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
17715 /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
17716 /// A horizontal-op B, for some already available A and B, and if so then LHS is
17727 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
17728 // which is A horizontal-op B.
17944 SDValue Op = N->getOperand(0);
17945 if (Op.getOpcode() == ISD::BITCAST)
17946 Op = Op.getOperand(0);
17947 EVT VT = N->getValueType(0), OpVT = Op.getValueType();
17948 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
17951 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
18442 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
18443 EVT VT = Op.getValueType();
18449 switch (Op.getOpcode()) {
18452 LoadSDNode *LD = cast<LoadSDNode>(Op);
18456 Op.hasOneUse()*/) {
18457 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
18458 UE = Op.getNode()->use_end(); UI != UE; ++UI) {
18475 SDValue N0 = Op.getOperand(0);
18477 if (MayFoldLoad(N0) && MayFoldIntoStore(Op))
18490 SDValue N0 = Op.getOperand(0);
18491 SDValue N1 = Op.getOperand(1);
18495 if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op)))
18497 if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op)))
18785 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
18798 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
18800 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
18806 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
18808 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
18814 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
18816 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
18822 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
18824 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
18831 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
18845 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
18848 Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
18858 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
18877 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) {
18880 } else if (Op.getOpcode() == ISD::ADD) {
18881 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
18883 Op = Op.getOperand(0);
18886 } else if (Op.getOpcode() == ISD::SUB) {
18887 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
18889 Op = Op.getOperand(0);
18905 Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op),
18915 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
