Lines Matching refs:Op
1625 for (ISD::NodeType Op :
1628 if (isOperationExpand(Op, MVT::f32))
1629 setOperationAction(Op, MVT::f32, Promote);
3502 unsigned Op = Arg.getOpcode();
3503 if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST) {
3507 if (Op == ISD::TRUNCATE) {
3782 static bool MayFoldLoad(SDValue Op) {
3783 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
3786 static bool MayFoldIntoStore(SDValue Op) {
3787 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
4020 // ZF PF CF op
4554 static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
4557 SDLoc dl(Op);
4558 SDValue Vec = Op.getOperand(0);
4559 SDValue SubVec = Op.getOperand(1);
4560 SDValue Idx = Op.getOperand(2);
4567 return Op;
4569 MVT OpVT = Op.getSimpleValueType();
4814 for (SDValue Op : MaskNode->ops()) {
4815 if (auto *CN = dyn_cast<ConstantSDNode>(Op.getNode()))
4817 else if (auto *CFN = dyn_cast<ConstantFPSDNode>(Op.getNode()))
5130 SDValue Op = V.getOperand(M / Scale);
5131 if (Op.isUndef())
5133 else if (X86::isZeroNode(Op))
5145 SDValue Op = V.getOperand((M * Scale) + j);
5146 AllUndef &= Op.isUndef();
5147 AllZero &= X86::isZeroNode(Op);
5165 static bool resolveTargetShuffleInputs(SDValue Op, SDValue &Op0, SDValue &Op1,
5168 if (!setTargetShuffleZeroElements(Op, Mask, Ops))
5263 static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
5271 SDLoc dl(Op);
5288 MVT::v16i8, V, Op.getOperand(i),
5312 MVT::i16, Op.getOperand(i-1));
5315 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
5333 static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
5341 SDLoc dl(Op);
5355 MVT::v8i16, V, Op.getOperand(i),
5364 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
5370 SDValue Elt = Op->getOperand(i);
5383 SDValue Elt = Op->getOperand(i);
5412 Elt = Op->getOperand(EltIdx);
5422 SDValue VZero = getZeroVector(VT, Subtarget, DAG, SDLoc(Op));
5441 SDValue Current = Op->getOperand(i);
5463 SDLoc DL(Op);
5744 static SDValue LowerVectorBroadcast(SDValue Op, const X86Subtarget &Subtarget,
5752 MVT VT = Op.getSimpleValueType();
5753 SDLoc dl(Op);
5761 switch (Op.getOpcode()) {
5767 auto *BVOp = cast<BuildVectorSDNode>(Op.getNode());
5788 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
5795 SDValue Sc = Op.getOperand(0);
5934 static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
5935 MVT VT = Op.getSimpleValueType();
5942 SDLoc DL(Op);
5943 unsigned NumElems = Op.getNumOperands();
5951 unsigned Opc = Op.getOperand(i).getOpcode();
5965 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
5966 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
5999 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
6006 static SDValue ConvertI1VectorToInteger(SDValue Op, SelectionDAG &DAG) {
6007 assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
6008 Op.getScalarValueSizeInBits() == 1 &&
6011 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
6012 SDValue In = Op.getOperand(idx);
6016 SDLoc dl(Op);
6018 MVT::getIntegerVT(std::max((int)Op.getValueType().getSizeInBits(), 8));
6023 X86TargetLowering::LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG) const {
6025 MVT VT = Op.getSimpleValueType();
6029 SDLoc dl(Op);
6030 if (ISD::isBuildVectorAllZeros(Op.getNode()))
6033 if (ISD::isBuildVectorAllOnes(Op.getNode()))
6036 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
6037 SDValue Imm = ConvertI1VectorToInteger(Op, DAG);
6051 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
6052 SDValue In = Op.getOperand(idx);
6063 else if (In != Op.getOperand(SplatIdx))
6069 return DAG.getNode(ISD::SELECT, dl, VT, Op.getOperand(SplatIdx),
6095 Op.getOperand(InsertIdx),
6134 SDValue Op = N->getOperand(i + BaseIdx);
6137 if (Op->isUndef()) {
6145 CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
6150 SDValue Op0 = Op.getOperand(0);
6151 SDValue Op1 = Op.getOperand(1);
6292 SDValue Op = BV->getOperand(i);
6295 unsigned Opcode = Op.getOpcode();
6305 SDValue Op0 = Op.getOperand(0);
6306 SDValue Op1 = Op.getOperand(1);
6502 static SDValue lowerBuildVectorToBitOp(SDValue Op, SelectionDAG &DAG) {
6503 SDLoc DL(Op);
6504 MVT VT = Op.getSimpleValueType();
6510 unsigned Opcode = Op.getOperand(0).getOpcode();
6512 if (Opcode != Op.getOperand(i).getOpcode())
6528 for (SDValue Elt : Op->ops()) {
6547 static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
6549 SDLoc DL(Op);
6550 MVT VT = Op.getSimpleValueType();
6553 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
6557 return Op;
6565 if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
6568 return Op;
6577 X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
6578 SDLoc dl(Op);
6580 MVT VT = Op.getSimpleValueType();
6582 unsigned NumElems = Op.getNumOperands();
6586 return LowerBUILD_VECTORvXi1(Op, DAG);
6588 if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
6591 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
6596 if (SDValue Broadcast = LowerVectorBroadcast(Op, Subtarget, DAG))
6598 if (SDValue BitOp = lowerBuildVectorToBitOp(Op, DAG))
6609 SDValue Elt = Op.getOperand(i);
6632 SDValue Item = Op.getOperand(Idx);
6704 X86::isZeroNode(Op.getOperand(0)) &&
6705 !X86::isZeroNode(Op.getOperand(1))) {
6709 VT, Op.getOperand(1)),
6735 SDValue Item = Op.getOperand(Idx);
6736 if (Op.getNode()->isOnlyUserOf(Item.getNode()))
6749 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
6757 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
6779 Op.getOperand(Idx));
6787 if (SDValue V = LowerBuildVectorv16i8(Op, NonZeros, NumNonZero, NumZero,
6792 if (SDValue V = LowerBuildVectorv8i16(Op, NonZeros, NumNonZero, NumZero,
6798 if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget, *this))
6809 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
6843 if (SDValue Sh = buildFromShuffleMostly(Op, DAG))
6849 if (!Op.getOperand(0).isUndef())
6850 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
6855 if (Op.getOperand(i).isUndef()) continue;
6857 Op.getOperand(i), DAG.getIntPtrConstant(i, dl));
6867 if (!Op.getOperand(i).isUndef())
6868 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
6900 static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
6901 SDLoc dl(Op);
6902 MVT ResVT = Op.getSimpleValueType();
6907 SDValue V1 = Op.getOperand(0);
6908 SDValue V2 = Op.getOperand(1);
6913 if (Op.getNumOperands() == 4) {
6916 SDValue V3 = Op.getOperand(2);
6917 SDValue V4 = Op.getOperand(3);
6926 static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
6929 SDLoc dl(Op);
6930 MVT ResVT = Op.getSimpleValueType();
6931 unsigned NumOfOperands = Op.getNumOperands();
6942 if (!Op.getOperand(i).isUndef()) {
6950 Op.getOperand(OpIdx).getValueType().getVectorNumElements();
6953 Op.getOperand(OpIdx), IdxVal);
6960 Ops.push_back(Op.getOperand(i));
6964 Ops.push_back(Op.getOperand(i));
6970 SDValue V1 = Op.getOperand(0);
6971 SDValue V2 = Op.getOperand(1);
6978 return Op; // The operation is legal with KUNPCK
7003 static SDValue LowerCONCAT_VECTORS(SDValue Op,
7006 MVT VT = Op.getSimpleValueType();
7008 return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
7010 assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||
7011 (VT.is512BitVector() && (Op.getNumOperands() == 2 ||
7012 Op.getNumOperands() == 4)));
7018 return LowerAVXCONCAT_VECTORS(Op, DAG);
7032 /// \brief Tiny helper function to identify a no-op mask.
7038 /// in-place shuffle are 'no-op's.
7267 SDValue Op = V.getOperand(M / Scale);
7268 if (Op.isUndef() || X86::isZeroNode(Op))
7270 else if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
7275 } else if (ConstantFPSDNode *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
7290 SDValue Op = V.getOperand((M * Scale) + j);
7291 Op.isUndef() || X86::isZeroNode(Op));
7682 // input shuffles would be a no-op. We prefer to shuffle inputs as the
7717 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
11215 if (SDValue Op =
11217 return Op;
11726 // Insure elements came from the same Op.
11734 SDValue Op = WidenedMask[i] > MaxOp1Index ? V2 : V1;
11737 Ops[OpIndex] = Op;
11738 else if (Ops[OpIndex] != Op)
11814 if (SDValue Op =
11816 return Op;
12114 static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget,
12116 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
12118 SDValue V1 = Op.getOperand(0);
12119 SDValue V2 = Op.getOperand(1);
12120 MVT VT = Op.getSimpleValueType();
12122 SDLoc DL(Op);
12251 static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
12254 SDValue Cond = Op.getOperand(0);
12255 SDValue LHS = Op.getOperand(1);
12256 SDValue RHS = Op.getOperand(2);
12257 SDLoc dl(Op);
12258 MVT VT = Op.getSimpleValueType();
12276 SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
12279 if (ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(0).getNode()) &&
12280 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(1).getNode()) &&
12281 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(2).getNode()))
12286 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
12294 // VSELECT-matching blend, return Op, and but if we need to expand, return
12296 switch (Op.getSimpleValueType().SimpleTy) {
12299 return Op;
12304 return Op;
12312 return Op;
12320 static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
12321 MVT VT = Op.getSimpleValueType();
12322 SDLoc dl(Op);
12324 if (!Op.getOperand(0).getSimpleValueType().is128BitVector())
12329 Op.getOperand(0), Op.getOperand(1));
12337 if (isNullConstant(Op.getOperand(1)))
12341 DAG.getBitcast(MVT::v4i32, Op.getOperand(0)),
12342 Op.getOperand(1)));
12344 Op.getOperand(0), Op.getOperand(1));
12356 if (!Op.hasOneUse())
12358 SDNode *User = *Op.getNode()->use_begin();
12360 isNullConstant(Op.getOperand(1))) &&
12365 DAG.getBitcast(MVT::v4i32, Op.getOperand(0)),
12366 Op.getOperand(1));
12372 if (isa<ConstantSDNode>(Op.getOperand(1)))
12373 return Op;
12381 X86TargetLowering::ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG) const {
12382 SDValue Vec = Op.getOperand(0);
12385 SDValue Idx = Op.getOperand(1);
12386 MVT EltVT = Op.getSimpleValueType();
12421 X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
12423 SDLoc dl(Op);
12424 SDValue Vec = Op.getOperand(0);
12426 SDValue Idx = Op.getOperand(1);
12428 if (Op.getSimpleValueType() == MVT::i1)
12429 return ExtractBitFromMaskVector(Op, DAG);
12447 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Perm,
12468 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
12475 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
12478 MVT VT = Op.getSimpleValueType();
12496 return Op;
12510 return Op;
12527 X86TargetLowering::InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG) const {
12528 SDLoc dl(Op);
12529 SDValue Vec = Op.getOperand(0);
12530 SDValue Elt = Op.getOperand(1);
12531 SDValue Idx = Op.getOperand(2);
12555 SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
12557 MVT VT = Op.getSimpleValueType();
12562 return InsertBitToMaskVector(Op, DAG);
12564 SDLoc dl(Op);
12565 SDValue N0 = Op.getOperand(0);
12566 SDValue N1 = Op.getOperand(1);
12567 SDValue N2 = Op.getOperand(2);
12670 return Op;
12689 static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
12690 SDLoc dl(Op);
12691 MVT OpVT = Op.getSimpleValueType();
12701 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
12704 return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
12708 Op.getOperand(0).getValueType() == MVT::i64)
12709 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
12711 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
12720 static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
12722 SDLoc dl(Op);
12723 SDValue In = Op.getOperand(0);
12724 SDValue Idx = Op.getOperand(1);
12726 MVT ResVT = Op.getSimpleValueType();
12746 static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
12751 SDLoc dl(Op);
12752 SDValue Vec = Op.getOperand(0);
12753 SDValue SubVec = Op.getOperand(1);
12754 SDValue Idx = Op.getOperand(2);
12760 MVT OpVT = Op.getSimpleValueType();
12797 return insert1BitVector(Op, DAG, Subtarget);
12809 X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
12810 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
12837 SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
12838 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
12865 X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
12866 const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
12882 SDLoc DL(Op);
12903 X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
12908 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
12909 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
12910 SDLoc dl(Op);
12976 X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
12977 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
12978 int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
12979 return LowerGlobalAddress(GV, SDLoc(Op), Offset, DAG);
13131 X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
13133 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
13173 SDLoc DL(Op);
13283 static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
13284 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
13285 MVT VT = Op.getSimpleValueType();
13287 SDLoc dl(Op);
13288 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
13289 SDValue ShOpLo = Op.getOperand(0);
13290 SDValue ShOpHi = Op.getOperand(1);
13291 SDValue ShAmt = Op.getOperand(2);
13302 if (Op.getOpcode() == ISD::SHL_PARTS) {
13323 if (Op.getOpcode() == ISD::SHL_PARTS) {
13335 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
13337 SDValue Src = Op.getOperand(0);
13339 MVT VT = Op.getSimpleValueType();
13340 SDLoc dl(Op);
13350 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
13361 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType()))
13362 return Op;
13363 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
13365 return Op;
13368 SDValue ValueToStore = Op.getOperand(0);
13369 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
13385 return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
13388 SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
13392 SDLoc DL(Op);
13394 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
13398 Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
13426 unsigned SSFISize = Op.getValueType().getSizeInBits()/8;
13432 Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
13439 Ops, Op.getValueType(), MMO);
13441 Op.getValueType(), DL, Chain, StackSlot,
13450 SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
13465 SDLoc dl(Op);
13486 Op.getOperand(0));
13519 SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
13521 SDLoc dl(Op);
13528 Op.getOperand(0));
13553 MVT DestVT = Op.getSimpleValueType();
13565 static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
13588 SDLoc DL(Op);
13589 SDValue V = Op->getOperand(0);
13595 if (VecFloatVT != Op->getSimpleValueType(0))
13659 SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op,
13661 SDValue N0 = Op.getOperand(0);
13663 SDLoc dl(Op);
13673 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
13678 return lowerUINT_TO_FP_vXi32(Op, DAG, Subtarget);
13682 return DAG.getNode(ISD::UINT_TO_FP, dl, Op.getValueType(),
13687 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
13689 SDValue N0 = Op.getOperand(0);
13690 SDLoc dl(Op);
13693 if (Op.getSimpleValueType().isVector())
13694 return lowerUINT_TO_FP_vec(Op, DAG);
13700 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0);
13703 MVT DstVT = Op.getSimpleValueType();
13709 return Op;
13713 return LowerUINT_TO_FP_i64(Op, DAG);
13715 return LowerUINT_TO_FP_i32(Op, DAG);
13723 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
13729 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
13734 SDValue ValueToStore = Op.getOperand(0);
13735 if (isScalarFPTypeInSSEReg(Op.getValueType()) && !Subtarget.is64Bit())
13763 Op.getOperand(0), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
13799 X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
13801 SDLoc DL(Op);
13803 EVT DstTy = Op.getValueType();
13804 EVT TheVT = Op.getOperand(0).getValueType();
13834 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
13838 isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
13857 SDValue Value = Op.getOperand(0);
13914 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
13977 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
13979 MVT VT = Op->getSimpleValueType(0);
13980 SDValue In = Op->getOperand(0);
13982 SDLoc dl(Op);
14010 bool NeedZero = Op.getOpcode() == ISD::ZERO_EXTEND;
14023 static SDValue LowerZERO_EXTEND_AVX512(SDValue Op,
14025 MVT VT = Op->getSimpleValueType(0);
14026 SDValue In = Op->getOperand(0);
14028 SDLoc DL(Op);
14054 static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
14057 if (SDValue Res = LowerAVXExtend(Op, DAG, Subtarget))
14063 static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
14065 SDLoc DL(Op);
14066 MVT VT = Op.getSimpleValueType();
14067 SDValue In = Op.getOperand(0);
14071 return LowerZERO_EXTEND_AVX512(Op, Subtarget, DAG);
14074 if (SDValue Res = LowerAVXExtend(Op, DAG, Subtarget))
14082 static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
14085 SDLoc DL(Op);
14086 MVT VT = Op.getSimpleValueType();
14087 SDValue In = Op.getOperand(0);
14120 SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
14121 SDLoc DL(Op);
14122 MVT VT = Op.getSimpleValueType();
14123 SDValue In = Op.getOperand(0);
14138 return LowerTruncateVecI1(Op, DAG, Subtarget);
14244 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
14246 assert(!Op.getSimpleValueType().isVector());
14248 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
14253 return Op;
14257 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
14265 SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op,
14267 std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
14272 return Op;
14276 return DAG.getLoad(Op.getValueType(), SDLoc(Op),
14284 static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) {
14285 SDLoc DL(Op);
14286 MVT VT = Op.getSimpleValueType();
14287 SDValue In = Op.getOperand(0);
14297 /// The only differences between FABS and FNEG are the mask and the logic op.
14299 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
14300 assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&
14303 bool IsFABS = (Op.getOpcode() == ISD::FABS);
14308 for (SDNode *User : Op->uses())
14310 return Op;
14312 SDLoc dl(Op);
14313 MVT VT = Op.getSimpleValueType();
14336 // generate a 16-byte vector constant and logic op even for the scalar case.
14338 // the logic op, so it can save (~4 bytes) on code size.
14359 SDValue Op0 = Op.getOperand(0);
14368 // For the scalar case extend to a 128-bit vector, perform the logic op,
14376 static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
14379 SDValue Op0 = Op.getOperand(0);
14380 SDValue Op1 = Op.getOperand(1);
14381 SDLoc dl(Op);
14382 MVT VT = Op.getSimpleValueType();
14465 static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
14466 SDValue N0 = Op.getOperand(0);
14467 SDLoc dl(Op);
14468 MVT VT = Op.getSimpleValueType();
14484 static SDValue LowerVectorAllZeroTest(SDValue Op, const X86Subtarget &Subtarget,
14486 assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
14491 if (!Op->hasOneUse())
14494 SDNode *N = Op.getNode();
14575 /// \brief return true if \c Op has a use that doesn't just read flags.
14576 static bool hasNonFlagsUse(SDValue Op) {
14577 for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
14595 static SDValue EmitKTEST(SDValue Op, SelectionDAG &DAG,
14597 if (Op.getOpcode() == ISD::BITCAST) {
14603 SDValue Op0 = Op.getOperand(0);
14607 return DAG.getNode(X86ISD::KTEST, SDLoc(Op), Op0VT, Op0, Op0);
14614 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
14616 if (Op.getValueType() == MVT::i1) {
14617 SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op);
14637 switch (Op->getOpcode()) {
14642 const auto *BinNode = cast<BinaryWithFlagsSDNode>(Op.getNode());
14656 if (Op.getResNo() != 0 || NeedOF || NeedCF) {
14658 if (auto Node = EmitKTEST(Op, DAG, Subtarget))
14661 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
14662 DAG.getConstant(0, dl, Op.getValueType()));
14671 SDValue ArithOp = Op;
14672 if (Op->getOpcode() == ISD::TRUNCATE && Op->hasOneUse()) {
14673 SDValue Arith = Op->getOperand(0);
14674 // Both the trunc and the arithmetic op need to have one user each.
14690 // which may be the result of a CAST. We use the variable 'Op', which is the
14704 for (SDNode::use_iterator UI = Op
14705 UE = Op.getNode()->use_end(); UI != UE; ++UI)
14737 if ((X86CC == X86::COND_E || X86CC == X86::COND_NE) && Op->hasOneUse() &&
14738 isa<ConstantSDNode>(Op->getOperand(1)) && !hasNonFlagsUse(Op)) {
14739 EVT VT = Op.getValueType();
14741 unsigned ShAmt = Op->getConstantOperandVal(1);
14749 Op = DAG.getNode(ISD::AND, dl, VT, Op->getOperand(0),
14757 if (!hasNonFlagsUse(Op)) {
14773 // Due to the ISEL shortcoming noted above, be conservative if this op is
14775 for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
14776 UE = Op.getNode()->use_end(); UI != UE; ++UI)
14788 if (SDValue EFLAGS = LowerVectorAllZeroTest(Op, Subtarget, DAG))
14805 return SDValue(Op.getNode(), 1);
14812 // update 'Op'.
14814 EVT VT = Op.getValueType();
14815 SDValue WideVal = Op->getOperand(0);
14835 Op = DAG.getNode(ConvertedOp, dl, VT, V0, V1);
14842 if (auto Node = EmitKTEST(Op, DAG, Subtarget))
14846 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
14847 DAG.getConstant(0, dl, Op.getValueType()));
14849 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
14850 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
14853 DAG.ReplaceAllUsesWith(Op, New);
14918 SDValue X86TargetLowering::getRsqrtEstimate(SDValue Op,
14922 EVT VT = Op.getValueType();
14946 return DCI.DAG.getNode(X86ISD::FRSQRT, SDLoc(Op), VT, Op);
14951 SDValue X86TargetLowering::getRecipEstimate(SDValue Op,
14954 EVT VT = Op.getValueType();
14977 return DCI.DAG.getNode(X86ISD::FRCP, SDLoc(Op), VT, Op);
15106 static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
15107 MVT VT = Op.getSimpleValueType();
15109 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&
15113 SDLoc dl(Op);
15114 SDValue CC = Op.getOperand(2);
15117 SDValue LHS = Op.getOperand(0);
15122 SDValue RHS = Op.getOperand(1);
15130 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
15131 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
15134 static SDValue LowerBoolVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
15135 SDValue Op0 = Op.getOperand(0);
15136 SDValue Op1 = Op.getOperand(1);
15137 SDValue CC = Op.getOperand(2);
15138 MVT VT = Op.getSimpleValueType();
15139 SDLoc dl(Op);
15177 static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
15179 SDValue Op0 = Op.getOperand(0);
15180 SDValue Op1 = Op.getOperand(1);
15181 SDValue CC = Op.getOperand(2);
15182 MVT VT = Op.getSimpleValueType();
15183 SDLoc dl(Op);
15246 static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
15248 SDValue Op0 = Op.getOperand(0);
15249 SDValue Op1 = Op.getOperand(1);
15250 SDValue CC = Op.getOperand(2);
15251 MVT VT = Op.getSimpleValueType();
15253 bool isFP = Op.getOperand(1).getSimpleValueType().isFloatingPoint();
15254 SDLoc dl(Op);
15275 // emit two comparisons and a logic op to tie them together.
15312 Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp);
15348 return Lower256IntVSETCC(Op, DAG);
15353 return LowerBoolVSETCC_AVX512(Op, DAG);
15366 return LowerIntVSETCC_AVX512(Op, DAG);
15569 SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
15571 MVT VT = Op.getSimpleValueType();
15573 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
15577 SDValue Op0 = Op.getOperand(0);
15578 SDValue Op1 = Op.getOperand(1);
15579 SDLoc dl(Op);
15580 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
15652 SDValue X86TargetLowering::LowerSETCCE(SDValue Op, SelectionDAG &DAG) const {
15653 SDValue LHS = Op.getOperand(0);
15654 SDValue RHS = Op.getOperand(1);
15655 SDValue Carry = Op.getOperand(2);
15656 SDValue Cond = Op.getOperand(3);
15657 SDLoc DL(Op);
15667 if (Op.getSimpleValueType() == MVT::i1) {
15676 static bool isX86LogicalCmp(SDValue Op) {
15677 unsigned Opc = Op.getNode()->getOpcode();
15681 if (Op.getResNo() == 1 &&
15695 if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
15720 SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
15722 SDValue Cond = Op.getOperand(0);
15723 SDValue Op1 = Op.getOperand(1);
15724 SDValue Op2 = Op.getOperand(2);
15725 SDLoc DL(Op);
15853 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
15864 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
15891 MVT VT = Op.getSimpleValueType();
15969 SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
15981 if (Op.getValueType() == MVT::i8 &&
15989 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
15995 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
16000 static SDValue LowerSIGN_EXTEND_AVX512(SDValue Op,
16003 MVT VT = Op->getSimpleValueType(0);
16004 SDValue In = Op->getOperand(0);
16008 SDLoc dl(Op);
16050 static SDValue LowerSIGN_EXTEND_VECTOR_INREG(SDValue Op,
16053 SDValue In = Op->getOperand(0);
16054 MVT VT = Op->getSimpleValueType(0);
16070 SDLoc dl(Op);
16116 static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
16118 MVT VT = Op->getSimpleValueType(0);
16119 SDValue In = Op->getOperand(0);
16121 SDLoc dl(Op);
16124 return LowerSIGN_EXTEND_AVX512(Op, Subtarget, DAG);
16177 SDValue Op = St->getValue();
16178 MVT OpVT = Op.getValueType().getSimpleVT();
16183 Op = DAG.getNode(ISD::TRUNCATE, dl, MemVT, Op);
16185 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
16186 DAG.getUNDEF(MVT::v8i1), Op,
16188 return DAG.getStore(St->getChain(), dl, Op, St->getBasePtr(),
16197 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ExtVT,
16198 DAG.getUNDEF(ExtVT), Op, DAG.getIntPtrConstant(0, dl));
16200 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i1, Op);
16201 return DAG.getStore(St->getChain(), dl, Op, St->getBasePtr(),
16207 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v16i8, Op,
16213 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v16i8, Op,
16226 static SDValue LowerExtended1BitVectorLoad(SDValue Op,
16230 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
16237 MVT VT = Op.getValueType().getSimpleVT();
16264 return DAG.getNode(ExtOpcode, dl, Op.getValueType(), Load);
16325 static SDValue LowerExtendedLoad(SDValue Op, const X86Subtarget &Subtarget,
16327 MVT RegVT = Op.getSimpleValueType();
16335 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
16339 return LowerExtended1BitVectorLoad(Op, Subtarget, DAG);
16510 static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
16511 Opc = Op.getOpcode();
16514 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16515 Op.getOperand(0).hasOneUse() &&
16516 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
16517 Op.getOperand(1).hasOneUse());
16522 static bool isXor1OfSetCC(SDValue Op) {
16523 if (Op.getOpcode() != ISD::XOR)
16525 if (isOneConstant(Op.getOperand(1)))
16526 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
16527 Op.getOperand(0).hasOneUse();
16531 SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
16533 SDValue Chain = Op.getOperand(0);
16534 SDValue Cond = Op.getOperand(1);
16535 SDValue Dest = Op.getOperand(2);
16536 SDLoc dl(Op);
16659 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16673 Op.getNode()->hasOneUse()) {
16678 SDNode *User = *Op.getNode()->use_begin();
16690 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16718 if (Op.getNode()->hasOneUse()) {
16719 SDNode *User = *Op.getNode()->use_begin();
16735 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16749 if (Op.getNode()->hasOneUse()) {
16750 SDNode *User = *Op.getNode()->use_begin();
16765 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16797 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
16807 X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
16813 SDLoc dl(Op);
16816 SDNode *Node = Op.getNode();
16817 SDValue Chain = Op.getOperand(0);
16818 SDValue Size = Op.getOperand(1);
16819 unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
16890 SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
16895 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
16896 SDLoc DL(Op);
16903 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
16913 SDValue FIN = Op.getOperand(1);
16915 SDValue Store = DAG.getStore(Op.getOperand(0), DL,
16923 Store = DAG.getStore(Op.getOperand(0), DL,
16932 Store = DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN,
16941 Store = DAG.getStore(Op.getOperand(0), DL, RSFIN, FIN, MachinePointerInfo(
16947 SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
16950 assert(Op.getNode()->getNumOperands() == 4);
16955 return DAG.expandVAArg(Op.getNode());
16957 SDValue Chain = Op.getOperand(0);
16958 SDValue SrcPtr = Op.getOperand(1);
16959 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
16960 unsigned Align = Op.getConstantOperandVal(3);
16961 SDLoc dl(Op);
16963 EVT ArgVT = Op.getNode()->getValueType(0);
17011 static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
17019 return DAG.expandVACopy(Op.getNode());
17021 SDValue Chain = Op.getOperand(0);
17022 SDValue DstPtr = Op.getOperand(1);
17023 SDValue SrcPtr = Op.getOperand(2);
17024 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
17025 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
17026 SDLoc DL(Op);
17214 /// \brief Return (and \p Op, \p Mask) for compare instructions or
17215 /// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
17217 static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
17221 MVT VT = Op.getSimpleValueType();
17224 SDLoc dl(Op);
17227 return Op;
17231 switch (Op.getOpcode()) {
17237 return DAG.getNode(ISD::AND, dl, VT, Op, VMask);
17240 return DAG.getNode(ISD::OR, dl, VT, Op, VMask);
17253 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
17257 /// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
17263 static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
17268 return Op;
17270 MVT VT = Op.getSimpleValueType();
17271 SDLoc dl(Op);
17275 if (Op.getOpcode() == X86ISD::FSETCC)
17276 return DAG.getNode(ISD::AND, dl, VT, Op, IMask);
17277 if (Op.getOpcode() == X86ISD::VFPCLASS ||
17278 Op.getOpcode() == X86ISD::VFPCLASSS)
17279 return DAG.getNode(ISD::OR, dl, VT, Op, IMask);
17283 return DAG.getNode(X86ISD::SELECT, dl, VT, IMask, Op, PreservedSrc);
17347 static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
17349 SDLoc dl(Op);
17350 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
17351 MVT VT = Op.getSimpleValueType();
17356 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1));
17358 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17359 Op.getOperand(2));
17361 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17362 DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(2)));
17364 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17365 Op.getOperand(2), Op.getOperand(3));
17367 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
17368 Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
17370 SDValue Src = Op.getOperand(1);
17371 SDValue PassThru = Op.getOperand(2);
17372 SDValue Mask = Op.getOperand(3);
17377 if (Op.getNumOperands() == 4)
17381 RoundingMode = Op.getOperand(4);
17387 dl, Op.getValueType(), Src, RoundingMode),
17394 SDValue Src = Op.getOperand(1);
17395 SDValue PassThru = Op.getOperand(2);
17396 SDValue Mask = Op.getOperand(3);
17402 SDValue Rnd = Op.getOperand(4);
17406 dl, Op.getValueType(),
17415 SDValue Src1 = Op.getOperand(1);
17416 SDValue Src2 = Op.getOperand(2);
17417 SDValue passThru = Op.getOperand(3);
17418 SDValue Mask = Op.getOperand(4);
17423 SDValue Src1 = Op.getOperand(1);
17424 SDValue Src2 = Op.getOperand(2);
17425 SDValue Src0 = Op.getOperand(3);
17426 SDValue Mask = Op.getOperand(4);
17430 if (Op.getNumOperands() == 6) {
17431 SDValue Sae = Op.getOperand(5);
17437 assert(Op.getNumOperands() == 7 && "Unexpected intrinsic form");
17438 SDValue RoundingMode = Op.getOperand(5);
17439 SDValue Sae = Op.getOperand(6);
17446 SDValue Src1 = Op.getOperand(1);
17447 SDValue Src2 = Op.getOperand(2);
17448 SDValue PassThru = Op.getOperand(3);
17449 SDValue Mask = Op.getOperand(4);
17459 SDValue Rnd = Op.getOperand(5);
17463 dl, Op.getValueType(),
17473 SDValue Src1 = Op.getOperand(1);
17474 SDValue Src2 = Op.getOperand(2);
17475 SDValue PassThru = Op.getOperand(3);
17476 SDValue Mask = Op.getOperand(4);
17482 if (Op.getNumOperands() == 6)
17483 Rnd = Op.getOperand(5);
17491 SDValue Src1 = Op.getOperand(1);
17492 SDValue Src2 = Op.getOperand(2);
17493 SDValue Src3 = Op.getOperand(3);
17494 SDValue PassThru = Op.getOperand(4);
17495 SDValue Mask = Op.getOperand(5);
17496 SDValue Sae = Op.getOperand(6);
17503 SDValue Src1 = Op.getOperand(1);
17504 SDValue Src2 = Op.getOperand(2);
17505 SDValue Imm = Op.getOperand(3);
17506 SDValue PassThru = Op.getOperand(4);
17507 SDValue Mask = Op.getOperand(5);
17513 if (Op.getNumOperands() == 7)
17514 Rnd = Op.getOperand(6);
17524 SDValue Src1 = Op.getOperand(1);
17525 SDValue Src2 = Op.getOperand(2);
17526 SDValue Src3 = Op.getOperand(3);
17527 SDValue PassThru = Op.getOperand(4);
17528 SDValue Mask = Op.getOperand(5);
17545 SDValue Rnd = Op.getOperand(6);
17549 dl, Op.getValueType(),
17559 SDValue Src1 = Op.getOperand(1);
17560 SDValue Src2 = Op.getOperand(2);
17561 SDValue PassThru = Op.getOperand(3);
17562 SDValue Mask = Op.getOperand(4);
17571 SDValue Src1 = Op.getOperand(1);
17572 SDValue Src2 = Op.getOperand(2);
17573 SDValue Src3 = Op.getOperand(3);
17574 SDValue Mask = Op.getOperand(4);
17575 MVT VT = Op.getSimpleValueType();
17586 dl, Op.getValueType(),
17593 SDValue Src1 = Op.getOperand(1);
17594 SDValue Src2 = Op.getOperand(2);
17595 SDValue Src3 = Op.getOperand(3);
17596 SDValue Mask = Op.getOperand(4);
17597 MVT VT = Op.getSimpleValueType();
17613 SDValue Rnd = Op.getOperand(5);
17617 dl, Op.getValueType(),
17622 dl, Op.getValueType(),
17629 SDValue Src1 = Op.getOperand(1);
17630 SDValue Src2 = Op.getOperand(2);
17631 SDValue Src3 = Op.getOperand(3);
17632 SDValue Mask = Op.getOperand(4);
17633 MVT VT = Op.getSimpleValueType();
17644 SDValue Rnd = Op.getOperand(5);
17646 Op.getValueType(), Src1, Src2,
17652 SDValue Src1 = Op.getOperand(1);
17653 SDValue Src2 = Op.getOperand(2);
17654 SDValue Src3 = Op.getOperand(3);
17655 SDValue Src4 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(4));
17656 SDValue Mask = Op.getOperand(5);
17657 MVT VT = Op.getSimpleValueType();
17669 SDValue Src1 = Op.getOperand(1);
17672 SDValue Imm = Op.getOperand(2);
17673 SDValue Mask = Op.getOperand(3);
17683 return DAG.getBitcast(Op.getValueType(), Res);
17686 SDValue Src1 = Op.getOperand(1);
17687 SDValue Imm = Op.getOperand(2);
17688 SDValue Mask = Op.getOperand(3);
17705 MVT VT = Op.getOperand(1).getSimpleValueType();
17707 SDValue Mask = Op.getOperand((IntrData->Type == CMP_MASK_CC) ? 4 : 3);
17712 SDValue CC = Op.getOperand(3);
17718 SDValue Rnd = Op.getOperand(5);
17721 Cmp = DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
17722 Op.getOperand(2), CC, Rnd);
17726 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
17727 Op.getOperand(2), CC);
17731 Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
17732 Op.getOperand(2));
17741 return DAG.getBitcast(Op.getValueType(), Res);
17744 SDValue Src1 = Op.getOperand(1);
17745 SDValue Src2 = Op.getOperand(2);
17746 SDValue CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(3));
17747 SDValue Mask = Op.getOperand(4);
17751 SDValue Rnd = Op.getOperand(5);
17769 SDValue LHS = Op.getOperand(1);
17770 SDValue RHS = Op.getOperand(2);
17816 SDValue LHS = Op.getOperand(1);
17817 SDValue RHS = Op.getOperand(2);
17818 unsigned CondVal = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
17819 SDValue Sae = Op.getOperand(4);
17833 return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
17834 Op.getOperand(1), Op.getOperand(2), DAG);
17836 SDValue Mask = Op.getOperand(3);
17837 SDValue DataToCompress = Op.getOperand(1);
17838 SDValue PassThru = Op.getOperand(2);
17840 return Op.getOperand(1);
17847 SDValue Mask = Op.getOperand(1);
17851 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Mask);
17854 MVT VT = Op.getSimpleValueType();
17857 SDValue Src1 = getMaskNode(Op.getOperand(1), MaskVT, Subtarget, DAG, dl);
17858 SDValue Src2 = getMaskNode(Op.getOperand(2), MaskVT, Subtarget, DAG, dl);
17869 SDValue Src1 = Op.getOperand(1);
17870 SDValue Src2 = Op.getOperand(2);
17871 SDValue Src3 = Op.getOperand(3);
17872 SDValue Imm = Op.getOperand(4);
17873 SDValue Mask = Op.getOperand(5);
17881 if (Op.getNumOperands() == 7)
17882 Rnd = Op.getOperand(6);
17895 MVT SrcVT = Op.getOperand(1).getSimpleValueType();
17900 Op.getOperand(1));
17904 return DAG.getBitcast(Op.getValueType(), Res);
17907 SDValue Mask = Op.getOperand(1);
17913 SDValue Src = Op.getOperand(1);
17914 SDValue Passthru = Op.getOperand(2);
17915 SDValue Mask = Op.getOperand(3);
17930 SDValue Src = Op.getOperand(1);
17931 SDValue PassThru = Op.getOperand(2);
17932 SDValue Mask = Op.getOperand(3);
17956 return DAG.getNode(X86ISD::VPERMV, dl, Op.getValueType(),
17957 Op.getOperand(2), Op.getOperand(1));
18016 SDValue LHS = Op.getOperand(1);
18017 SDValue RHS = Op.getOperand(2);
18027 SDValue LHS = DAG.getBitcast(MVT::v16i1, Op.getOperand(1));
18028 SDValue RHS = DAG.getBitcast(MVT::v16i1, Op.getOperand(2));
18090 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
18091 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
18107 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
18108 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
18125 SDValue Op1 = Op.getOperand(1);
18137 SDValue FnOp = Op.getOperand(1);
18138 SDValue IncomingFPOp = Op.getOperand(2);
18162 static SDValue getGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
18166 SDLoc dl(Op);
18173 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
18177 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
18184 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
18188 SDLoc dl(Op);
18203 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
18207 SDLoc dl(Op);
18322 static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
18325 SDLoc DL(Op);
18326 getReadTimeStampCounter(Op.getNode(), DL, X86ISD::RDTSC_DAG, DAG, Subtarget,
18331 static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
18333 SDValue Chain = Op.getOperand(0);
18334 SDValue RegNode = Op.getOperand(2);
18349 static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
18351 SDValue Chain = Op.getOperand(0);
18352 SDValue EHGuard = Op.getOperand(2);
18367 static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
18369 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
18374 return MarkEHRegistrationNode(Op, DAG);
18376 return MarkEHGuard(Op, DAG);
18392 SDLoc dl(Op);
18398 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other);
18399 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
18403 SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
18404 DAG.getConstant(1, dl, Op->getValueType(1)),
18408 DAG.getVTList(Op->getValueType(1), MVT::Glue),
18412 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
18417 SDValue Chain = Op.getOperand(0);
18418 SDValue Src = Op.getOperand(2);
18419 SDValue Base = Op.getOperand(3);
18420 SDValue Index = Op.getOperand(4);
18421 SDValue Mask = Op.getOperand(5);
18422 SDValue Scale = Op.getOperand(6);
18423 return getGatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale,
18428 SDValue Chain = Op.getOperand(0);
18429 SDValue Base = Op.getOperand(2);
18430 SDValue Mask = Op.getOperand(3);
18431 SDValue Index = Op.getOperand(4);
18432 SDValue Src = Op.getOperand(5);
18433 SDValue Scale = Op.getOperand(6);
18434 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
18438 SDValue Hint = Op.getOperand(6);
18442 SDValue Chain = Op.getOperand(0);
18443 SDValue Mask = Op.getOperand(2);
18444 SDValue Index = Op.getOperand(3);
18445 SDValue Base = Op.getOperand(4);
18446 SDValue Scale = Op.getOperand(5);
18447 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
18453 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
18460 getReadPerformanceCounter(Op.getNode(), dl, DAG, Subtarget, Results);
18465 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
18466 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
18470 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
18471 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
18476 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
18477 SDVTList VTs = DAG.getVTList(Op.getOperand(3)->getValueType(0), MVT::Other);
18478 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(2),
18480 SDValue Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(3),
18481 Op.getOperand(4), GenCF.getValue(1));
18482 SDValue Store = DAG.getStore(Op.getOperand(0), dl, Res.getValue(0),
18483 Op.getOperand(5), MachinePointerInfo(),
18492 SDValue Mask = Op.getOperand(4);
18493 SDValue DataToCompress = Op.getOperand(3);
18494 SDValue Addr = Op.getOperand(2);
18495 SDValue Chain = Op.getOperand(0);
18498 MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
18514 SDValue Mask = Op.getOperand(4);
18515 SDValue DataToTruncate = Op.getOperand(3);
18516 SDValue Addr = Op.getOperand(2);
18517 SDValue Chain = Op.getOperand(0);
18519 MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
18535 SDValue Mask = Op.getOperand(4);
18536 SDValue PassThru = Op.getOperand(3);
18537 SDValue Addr = Op.getOperand(2);
18538 SDValue Chain = Op.getOperand(0);
18539 MVT VT = Op.getSimpleValueType();
18541 MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
18558 SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
18563 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
18566 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
18567 SDLoc dl(Op);
18571 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
18586 SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
18591 EVT VT = Op.getValueType();
18612 SDLoc dl(Op); // FIXME probably not meaningful
18613 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
18660 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
18663 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
18685 Op, SelectionDAG &DAG) const {
18686 SDValue Chain = Op.getOperand(0);
18687 SDValue Offset = Op.getOperand(1);
18688 SDValue Handler = Op.getOperand(2);
18689 SDLoc dl (Op);
18712 SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
18714 SDLoc DL(Op);
18727 Op.getOperand(0), Op.getOperand(1));
18730 SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
18732 SDLoc DL(Op);
18734 Op.getOperand(0), Op.getOperand(1));
18737 SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
18739 SDLoc DL(Op);
18741 Op.getOperand(0));
18744 static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
18745 return Op.getOperand(0);
18748 SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
18750 SDValue Root = Op.getOperand(0);
18751 SDValue Trmp = Op.getOperand(1); // trampoline
18752 SDValue FPtr = Op.getOperand(2); // nested function
18753 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
18754 SDLoc dl (Op);
18756 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
18817 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
18900 SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
18924 MVT VT = Op.getSimpleValueType();
18925 SDLoc DL(Op);
18976 static SDValue LowerVectorCTLZ_AVX512(SDValue Op, SelectionDAG &DAG) {
18977 assert(Op.getOpcode() == ISD::CTLZ);
18978 SDLoc dl(Op);
18979 MVT VT = Op.getSimpleValueType();
18991 Op.getOperand(0),
19005 std::tie(Lo, Hi) = DAG.SplitVector(Op.getOperand(0), dl);
19020 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
19021 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
19029 static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
19032 MVT VT = Op.getSimpleValueType();
19053 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
19098 static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
19101 MVT VT = Op.getSimpleValueType();
19102 SDValue Op0 = Op.getOperand(0);
19105 return LowerVectorCTLZ_AVX512(Op, DAG);
19121 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
19124 static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
19126 MVT VT = Op.getSimpleValueType();
19129 SDLoc dl(Op);
19130 unsigned Opc = Op.getOpcode();
19133 return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
19135 Op = Op.getOperand(0);
19139 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
19144 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
19149 Op,
19152 Op.getValue(1)
19154 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
19158 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
19162 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
19163 return Op;
19166 static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
19167 MVT VT = Op.getSimpleValueType();
19169 SDLoc dl(Op);
19172 SDValue N0 = Op.getOperand(0);
19180 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
19192 assert(Op.getOpcode() == ISD::CTTZ &&
19197 Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op.getOperand(0));
19201 Op,
19204 Op.getValue(1)
19211 static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
19212 MVT VT = Op.getSimpleValueType();
19218 SDLoc dl(Op);
19221 SDValue LHS = Op.getOperand(0);
19226 SDValue RHS = Op.getOperand(1);
19234 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
19235 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
19240 static SDValue Lower512IntArith(SDValue Op, SelectionDAG &DAG) {
19241 MVT VT = Op.getSimpleValueType();
19247 SDLoc dl(Op);
19250 SDValue LHS = Op.getOperand(0);
19255 SDValue RHS = Op.getOperand(1);
19263 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
19264 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
19267 static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
19268 if (Op.getValueType() == MVT::i1)
19269 return DAG.getNode(ISD::XOR, SDLoc(Op), Op.getValueType(),
19270 Op.getOperand(0), Op.getOperand(1));
19271 assert(Op.getSimpleValueType().is256BitVector() &&
19272 Op.getSimpleValueType().isInteger() &&
19274 return Lower256IntArith(Op, DAG);
19277 static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) {
19278 if (Op.getValueType() == MVT::i1)
19279 return DAG.getNode(ISD::XOR, SDLoc(Op), Op.getValueType(),
19280 Op.getOperand(0), Op.getOperand(1));
19281 assert(Op.getSimpleValueType().is256BitVector() &&
19282 Op.getSimpleValueType().isInteger() &&
19284 return Lower256IntArith(Op, DAG);
19287 static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) {
19288 assert(Op.getSimpleValueType().is256BitVector() &&
19289 Op.getSimpleValueType().isInteger() &&
19291 return Lower256IntArith(Op, DAG);
19294 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
19296 SDLoc dl(Op);
19297 MVT VT = Op.getSimpleValueType();
19300 return DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), Op.getOperand(1));
19304 return Lower256IntArith(Op, DAG);
19306 SDValue A = Op.getOperand(0);
19307 SDValue B = Op.getOperand(1);
19316 return Lower512IntArith(Op, DAG);
19322 return Lower256IntArith(Op, DAG);
19446 static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
19448 SDLoc dl(Op);
19449 MVT VT = Op.getSimpleValueType();
19453 return Lower256IntArith(Op, DAG);
19461 SDValue A = Op.getOperand(0);
19462 SDValue B = Op.getOperand(1);
19466 unsigned Opcode = Op.getOpcode();
19560 SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
19562 EVT VT = Op.getValueType();
19568 switch (Op->getOpcode()) {
19578 SDLoc dl(Op);
19583 for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
19584 EVT ArgVT = Op->getOperand(i).getValueType();
19589 InChain = DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MachinePointerInfo(),
19612 static SDValue LowerMUL_LOHI(SDValue Op, const X86Subtarget &Subtarget,
19614 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
19616 SDLoc dl(Op);
19620 unsigned Opcode = Op.getOpcode();
19662 bool IsSigned = Op->getOpcode() == ISD::SMUL_LOHI;
19754 static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
19756 MVT VT = Op.getSimpleValueType();
19757 SDLoc dl(Op);
19758 SDValue R = Op.getOperand(0);
19759 SDValue Amt = Op.getOperand(1);
19761 unsigned X86Opc = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
19762 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
19801 if (SupportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode()))
19806 Op.getOpcode() == ISD::SRA && !Subtarget.hasXOP())
19816 if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1)
19820 if (Op.getOpcode() == ISD::SRA && ShiftAmt == 7) {
19834 if (Op.getOpcode() == ISD::SHL) {
19843 if (Op.getOpcode() == ISD::SRL) {
19852 if (Op.getOpcode() == ISD::SRA) {
19914 if (SupportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode()))
19917 if (Op.getOpcode() == ISD::SRA)
19924 static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
19926 MVT VT = Op.getSimpleValueType();
19927 SDLoc dl(Op);
19928 SDValue R = Op.getOperand(0);
19929 SDValue Amt = Op.getOperand(1);
19931 unsigned X86OpcI = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHLI :
19932 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRLI : X86ISD::VSRAI;
19934 unsigned X86OpcV = (Op.getOpcode() == ISD::SHL) ? X86ISD::VSHL :
19935 (Op.getOpcode() == ISD::SRL) ? X86ISD::VSRL : X86ISD::VSRA;
19937 if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Op.getOpcode())) {
20001 if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Op.getOpcode()))
20002 return DAG.getNode(X86OpcV, dl, VT, R, Op.getOperand(1));
20007 static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
20009 MVT VT = Op.getSimpleValueType();
20010 SDLoc dl(Op);
20011 SDValue R = Op.getOperand(0);
20012 SDValue Amt = Op.getOperand(1);
20018 if (SDValue V = LowerScalarImmediateShift(Op, DAG, Subtarget))
20021 if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget))
20024 if (SupportedVectorVarShift(VT, Subtarget, Op.getOpcode()))
20025 return Op;
20032 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) {
20036 if (Op.getOpcode() == ISD::SHL || Op.getOpcode() == ISD::SRL)
20038 if (Op.getOpcode() == ISD::SRA)
20044 if (VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) {
20048 SDValue R0 = DAG.getNode(Op->getOpcode(), dl, VT, R, Amt0);
20049 SDValue R1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Amt1);
20057 Op.getOpcode() == ISD::SRA) {
20069 if (ConstantAmt && Op.getOpcode() == ISD::SHL &&
20079 SDValue Op = Amt->getOperand(i);
20080 if (Op->isUndef()) {
20081 Elts.push_back(Op);
20085 ConstantSDNode *ND = cast<ConstantSDNode>(Op);
20099 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) {
20100 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
20102 Op = DAG.getNode(ISD::ADD, dl, VT, Op,
20104 Op = DAG.getBitcast(MVT::v4f32, Op);
20105 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
20106 return DAG.getNode(ISD::MUL, dl, VT, Op, R);
20166 SDValue Shift1 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat1);
20169 SDValue Shift2 = DAG.getNode(Op->getOpcode(), dl, VT, R, Splat2);
20186 unsigned Opc = Op.getOpcode();
20230 unsigned ShiftOpcode = Op->getOpcode();
20257 if (Op->getOpcode() == ISD::SHL || Op->getOpcode() == ISD::SRL) {
20279 if (Op->getOpcode() == ISD::SRA) {
20341 Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
20345 DAG.getNode(Op.getOpcode(), dl, ExtVT, R, Amt));
20359 SDValue Lo = DAG.getNode(Op.getOpcode(), dl, ExtVT, RLo, ALo);
20360 SDValue Hi = DAG.getNode(Op.getOpcode(), dl, ExtVT, RHi, AHi);
20367 unsigned ShiftOpcode = Op->getOpcode();
20434 return Lower256IntArith(Op, DAG);
20439 static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
20441 MVT VT = Op.getSimpleValueType();
20442 SDLoc DL(Op);
20443 SDValue R = Op.getOperand(0);
20444 SDValue Amt = Op.getOperand(1);
20448 assert((Op.getOpcode() == ISD::ROTL) && "Only ROTL supported");
20455 return Lower256IntArith(Op, DAG);
20473 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
20478 SDNode *N = Op.getNode();
20483 SDLoc DL(Op);
20484 switch (Op.getOpcode()) {
20601 AtomicRMWInst::BinOp Op = AI->getOperation();
20602 switch (Op) {
20687 static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
20689 SDLoc dl(Op);
20691 cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
20693 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
20700 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
20702 SDValue Chain = Op.getOperand(0);
20717 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
20718 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
20721 static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
20723 MVT T = Op.getSimpleValueType();
20724 SDLoc DL(Op);
20737 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
20738 Op.getOperand(2), SDValue());
20740 Op.getOperand(1),
20741 Op.getOperand(3),
20745 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
20753 SDValue Success = DAG.getNode(X86ISD::SETCC, DL, Op->getValueType(1),
20757 DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), cpOut);
20758 DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
20759 DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), EFLAGS.getValue(1));
20763 static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
20765 MVT SrcVT = Op.getOperand(0).getSimpleValueType();
20766 MVT DstVT = Op.getSimpleValueType();
20775 SDValue Op0 = Op->getOperand(0);
20777 SDLoc dl(Op);
20816 return Op;
20818 return Op;
20821 return Op;
20895 static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
20898 MVT VT = Op.getSimpleValueType();
20923 SDValue In = DAG.getBitcast(ByteVecVT, Op);
20952 static SDValue LowerVectorCTPOPBitmath(SDValue Op, const SDLoc &DL,
20955 MVT VT = Op.getSimpleValueType();
20987 SDValue V = Op;
21017 static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
21019 MVT VT = Op.getSimpleValueType();
21022 SDLoc DL(Op.getNode());
21023 SDValue Op0 = Op.getOperand(0);
21058 static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
21060 assert(Op.getSimpleValueType().isVector() &&
21062 return LowerVectorCTPOP(Op, Subtarget, DAG);
21065 static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
21066 MVT VT = Op.getSimpleValueType();
21067 SDValue In = Op.getOperand(0);
21068 SDLoc DL(Op);
21098 // VPPERM reverses the bits of a byte with the permute Op (2 << 5), and we
21118 static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
21121 return LowerBITREVERSE_XOP(Op, DAG);
21125 MVT VT = Op.getSimpleValueType();
21126 SDValue In = Op.getOperand(0);
21127 SDLoc DL(Op);
21213 // We can lower atomic_load_add into LXADD. However, any other atomicrmw op
21238 static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) {
21239 SDNode *Node = Op.getNode();
21261 return Op;
21264 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
21265 MVT VT = Op.getNode()->getSimpleValueType(0);
21275 switch (Op.getOpcode()) {
21284 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
21285 Op.getOperand(1));
21286 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
21287 Op.getOperand(1), Op.getOperand(2));
21290 static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
21297 SDLoc dl(Op);
21298 SDValue Arg = Op.getOperand(0);
21394 static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
21402 if (Op.getNode()->getNumValues() == 2)
21403 return Op;
21405 MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode());
21408 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op");
21409 SDLoc dl(Op);
21484 DAG.ReplaceAllUsesWith(Op, SDValue(NewScatter.getNode(), 1));
21488 static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
21491 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
21492 MVT VT = Op.getSimpleValueType();
21495 SDLoc dl(Op);
21498 "Cannot lower masked load op.");
21503 "Unsupported masked load op.");
21525 static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
21527 MaskedStoreSDNode *N = cast<MaskedStoreSDNode>(Op.getNode());
21532 SDLoc dl(Op);
21535 "Cannot lower masked store op.");
21540 "Unsupported masked store op.");
21554 static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
21559 MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode());
21560 SDLoc dl(Op);
21561 MVT VT = Op.getSimpleValueType();
21569 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op");
21581 return Op;
21614 return Op;
21617 SDValue X86TargetLowering::LowerGC_TRANSITION_START(SDValue Op,
21627 Ops.push_back(Op.getOperand(0));
21628 if (Op->getGluedNode())
21629 Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
21631 SDLoc OpDL(Op);
21633 SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
21638 SDValue X86TargetLowering::LowerGC_TRANSITION_END(SDValue Op,
21648 Ops.push_back(Op.getOperand(0));
21649 if (Op->getGluedNode())
21650 Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
21652 SDLoc OpDL(Op);
21654 SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
21660 SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
21661 switch (Op.getOpcode()) {
21663 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
21665 return LowerCMP_SWAP(Op, Subtarget, DAG);
21666 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG);
21671 case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget);
21672 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG);
21673 case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG);
21674 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
21675 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
21676 case ISD::VECTOR_SHUFFLE: return lowerVectorShuffle(Op, Subtarget, DAG);
21677 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
21678 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
21679 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
21680 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
21681 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
21682 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
21683 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
21684 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
21685 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
21686 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
21687 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
21690 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
21691 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
21692 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
21693 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
21694 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG);
21695 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG);
21696 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG);
21698 return LowerSIGN_EXTEND_VECTOR_INREG(Op, Subtarget, DAG);
21699 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
21700 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
21701 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
21702 case ISD::LOAD: return LowerExtendedLoad(Op, Subtarget, DAG);
21704 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG);
21705 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
21706 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
21707 case ISD::SETCC: return LowerSETCC(Op, DAG);
21708 case ISD::SETCCE: return LowerSETCCE(Op, DAG);
21709 case ISD::SELECT: return LowerSELECT(Op, DAG);
21710 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
21711 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
21712 case ISD::VASTART: return LowerVASTART(Op, DAG);
21713 case ISD::VAARG: return LowerVAARG(Op, DAG);
21714 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
21715 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, Subtarget, DAG);
21717 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
21718 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
21719 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
21721 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
21722 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
21723 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
21724 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
21725 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
21727 return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
21728 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
21729 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
21730 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
21732 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ(Op, Subtarget, DAG);
21734 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op, DAG);
21735 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
21737 case ISD::MULHU: return LowerMULH(Op, Subtarget, DAG);
21739 case ISD::SMUL_LOHI: return LowerMUL_LOHI(Op, Subtarget, DAG);
21740 case ISD::ROTL: return LowerRotate(Op, Subtarget, DAG);
21743 case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
21749 case ISD::UMULO: return LowerXALUO(Op, DAG);
21750 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
21751 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG);
21755 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
21756 case ISD::ADD: return LowerADD(Op, DAG);
21757 case ISD::SUB: return LowerSUB(Op, DAG);
21761 case ISD::UMIN: return LowerMINMAX(Op, DAG);
21762 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG);
21763 case ISD::MLOAD: return LowerMLOAD(Op, Subtarget, DAG);
21764 case ISD::MSTORE: return LowerMSTORE(Op, Subtarget, DAG);
21765 case ISD::MGATHER: return LowerMGATHER(Op, Subtarget, DAG);
21766 case ISD::MSCATTER: return LowerMSCATTER(Op, Subtarget, DAG);
21768 return LowerGC_TRANSITION_START(Op, DAG);
21769 case ISD::GC_TRANSITION_END: return LowerGC_TRANSITION_END(Op, DAG);
21770 case ISD::STORE: return LowerTruncatingStore(Op, Subtarget, DAG);
22620 MachineOperand &Op = MI.getOperand(i);
22621 if (!(Op.isReg() && Op.isImplicit()))
22622 MIB.addOperand(Op);
22656 MachineOperand &Op = MI.getOperand(i);
22657 if (!(Op.isReg() && Op.isImplicit()))
22658 MIB.addOperand(Op);
23411 // a.store(reg OP a.load(acquire), release)
23950 unsigned Op = 0;
23957 Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
23962 Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
23982 MachineInstrBuilder MIB = BuildMI(*MBB, MI, DL, TII->get(Op));
24083 unsigned Op = FPIs64Bit ? X86::MOV64rm : X86::MOV32rm;
24084 addRegOffset(BuildMI(DispatchBB, DL, TII->get(Op), BP), FP, true,
24600 void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
24606 unsigned Opc = Op.getOpcode();
24611 "Should use MaskedValueIsZero if you don't know whether Op"
24629 if (Op.getResNo() == 0)
24636 unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
24644 SDValue Op,
24648 if (Op.getOpcode() == X86ISD::SETCC_CARRY)
24649 return Op.getValueType().getScalarSizeInBits();
25327 static bool combineX86ShufflesRecursively(SDValue Op, SDValue Root,
25339 while (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).hasOneUse())
25340 Op = Op.getOperand(0);
25342 MVT VT = Op.getSimpleValueType();
25354 if (!resolveTargetShuffleInputs(Op, Input0, Input1, OpMask))
25370 "Must not have a ratio for both incoming and op masks!");
25431 HasVariableMask |= isTargetShuffleVariableMask(Op.getOpcode());
25434 if (Op->isOnlyUserOf(Input0.getNode()) &&
25899 // Nuke no-op shuffles that show up after combining.
25917 // dwords as otherwise it would have been removed as a no-op.
26939 /// Note that this is only legal for some op/cc combinations.
27006 // (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
27007 // (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
27008 // to (Op EFLAGS Cond)
27010 // (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
27011 // (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
27012 // to (Op EFLAGS !Cond)
27014 // where Op could be BRCOND or CMOV.
27096 SDValue Op = SetCC.getOperand(0);
27098 if (Op.getOpcode() == ISD::ZERO_EXTEND ||
27099 Op.getOpcode() == ISD::TRUNCATE)
27100 Op = Op.getOperand(0);
27103 if ((Op.getOpcode() != X86ISD::RDRAND &&
27104 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
28056 SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, WideVT, N0, N1);
28060 return Op;
28066 Op, DAG.getConstant(Mask, DL, VT));
28070 Op, DAG.getValueType(NarrowVT));
28165 /// If both input operands of a logic op are being cast from floating point
28847 const SDValue &Op = BV->getOperand(i);
28848 if (Op.isUndef())
28850 auto *ConstNode = dyn_cast<ConstantSDNode>(Op);
29435 /// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
29436 /// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
29437 /// A horizontal-op B, for some already available A and B, and if so then LHS is
29448 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
29449 // which is A horizontal-op B.
29790 default: llvm_unreachable("Unexpected FP logic op");
29947 SDValue Op = peekThroughBitcasts(N->getOperand(0));
29948 EVT VT = N->getValueType(0), OpVT = Op.getValueType();
29949 if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
29952 return DAG.getBitcast(VT, Op);
30662 // The two operands of reduction add are from PHI and a select-op
30822 SDValue Op = N->getOperand(0);
30823 MVT OpVT = Op.getSimpleValueType();
30828 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
30831 SDValue OpElt = Op.getOperand(i);
30845 SDValue V = peekThroughBitcasts(Op);
30846 if (V != Op && V.getOpcode() == X86ISD::VZEXT) {
30886 Op = DAG.getBitcast(OpVT, OrigV);
30887 return DAG.getNode(X86ISD::VZEXT, DL, VT, Op);
31077 bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
31078 EVT VT = Op.getValueType();
31084 switch (Op.getOpcode()) {
31093 SDValue N0 = Op.getOperand(0);
31095 if (MayFoldLoad(N0) && MayFoldIntoStore(Op))
31108 SDValue N0 = Op.getOperand(0);
31109 SDValue N1 = Op.getOperand(1);
31113 if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op)))
31115 if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op)))
31406 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
31419 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31421 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31422 Op.getValueType());
31428 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31430 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31431 Op.getValueType());
31437 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31439 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31440 Op.getValueType());
31446 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31449 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
31450 Op.getValueType());
31456 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31458 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31459 Op.getValueType());
31465 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31467 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31468 Op.getValueType());
31474 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31476 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31477 Op.getValueType());
31484 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31488 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
31498 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
31501 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
31502 Op.getValueType());
31512 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
31514 Result = DAG.getTargetConstant(CST->getSExtValue(), SDLoc(Op), MVT::i64);
31531 if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) {
31534 } else if (Op.getOpcode() == ISD::ADD) {
31535 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
31537 Op = Op.getOperand(0);
31540 } else if (Op.getOpcode() == ISD::SUB) {
31541 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
31543 Op = Op.getOperand(0);
31558 Result = DAG.getTargetGlobalAddress(GV, SDLoc(Op),
31568 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
