Lines Matching refs:Op
738 const SDValue Op, APInt &KnownZero, APInt &KnownOne,
740 switch (Op.getOpcode()) {
745 DAG.computeKnownBits(Op
746 DAG.computeKnownBits(Op->getOperand(1), KnownZero2, KnownOne2, Depth + 1);
752 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
759 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
769 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
779 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1616 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
1617 assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
1620 SDLoc DL(Op);
1621 SDValue LHS = Op.getOperand(0);
1622 SDValue RHS = Op.getOperand(1);
1624 switch (Op.getOpcode()) {
1647 bool IsSigned = Op.getOpcode() == ISD::SMULO;
1648 if (Op.getValueType() == MVT::i32) {
1696 assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
1721 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
1730 SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
1732 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
1733 return makeLibCall(DAG, Call, MVT::f128, Ops, false, SDLoc(Op)).first;
1736 static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
1737 SDValue Sel = Op.getOperand(0);
1738 SDValue Other = Op.getOperand(1);
1744 return Op;
1762 return Op;
1769 return Op;
1792 return Op;
1795 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
1796 EVT VT = Op.getValueType();
1806 switch (Op.getOpcode()) {
1826 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
1827 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
1828 Op.getOperand(2));
1831 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
1833 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
1836 SDLoc dl(Op);
1840 std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
1853 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
1862 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
1863 SDLoc DL(Op);
1864 unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
1865 unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
1866 unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
1884 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
1885 DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
1888 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
1890 assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
1893 LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
1895 return LowerF128Call(Op, DAG, LC);
1898 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
1900 if (Op.getOperand(0).getValueType() != MVT::f128) {
1902 return Op;
1906 LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
1911 SDValue SrcVal = Op.getOperand(0);
1912 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
1913 SDLoc(Op)).first;
1916 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
1920 EVT InVT = Op.getOperand(0).getValueType();
1921 EVT VT = Op.getValueType();
1927 SDLoc dl(Op);
1929 Op.getOpcode(), dl, Op.getValueType(),
1930 DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
1934 SDLoc dl(Op);
1936 DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
1937 Op.getOperand(0));
1942 SDLoc dl(Op);
1946 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
1947 return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
1951 return Op;
1954 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
1956 if (Op.getOperand(0).getValueType().isVector())
1957 return LowerVectorFP_TO_INT(Op, DAG);
1960 if (Op.getOperand(0).getValueType() == MVT::f16) {
1961 SDLoc dl(Op);
1963 Op.getOpcode(), dl, Op.getValueType(),
1964 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0)));
1967 if (Op.getOperand(0).getValueType() != MVT::f128) {
1969 return Op;
1973 if (Op.getOpcode() == ISD::FP_TO_SINT)
1974 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), Op.getValueType());
1976 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), Op.getValueType());
1978 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
1979 return makeLibCall(DAG, LC, Op.getValueType(), Ops, false, SDLoc(Op)).first;
1982 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
1986 EVT VT = Op.getValueType();
1987 SDLoc dl(Op);
1988 SDValue In = Op.getOperand(0);
1995 In = DAG.getNode(Op.getOpcode(), dl, CastVT, In);
2001 Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2004 return DAG.getNode(Op.getOpcode(), dl, VT, In);
2007 return Op;
2010 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2012 if (Op.getValueType().isVector())
2013 return LowerVectorINT_TO_FP(Op, DAG);
2016 if (Op.getValueType() == MVT::f16) {
2017 SDLoc dl(Op);
2020 DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)),
2025 if (Op.getOperand(0).getValueType() == MVT::i128)
2030 if (Op.getValueType() != MVT::f128)
2031 return Op;
2034 if (Op.getOpcode() == ISD::SINT_TO_FP)
2035 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2037 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2039 return LowerF128Call(Op, DAG, LC);
2042 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2046 SDLoc dl(Op);
2047 SDValue Arg = Op.getOperand(0);
2074 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2075 if (Op.getValueType() != MVT::f16)
2078 assert(Op.getOperand(0).getValueType() == MVT::i16);
2079 SDLoc DL(Op);
2081 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
2082 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
2084 DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op,
2211 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
2214 EVT VT = Op.getValueType();
2217 SDNode *N0 = Op.getOperand(0).getNode();
2218 SDNode *N1 = Op.getOperand(1).getNode();
2252 return Op;
2257 SDLoc DL(Op);
2280 SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2282 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2283 SDLoc dl(Op);
2291 return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
2292 Op.getOperand(1), Op.getOperand(2));
2294 return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
2295 Op.getOperand(1), Op.getOperand(2));
2297 return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
2298 Op.getOperand(1), Op.getOperand(2));
2300 return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
2301 Op.getOperand(1), Op.getOperand(2));
2305 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
2307 switch (Op.getOpcode()) {
2312 return LowerBITCAST(Op, DAG);
2314 return LowerGlobalAddress(Op, DAG);
2316 return LowerGlobalTLSAddress(Op, DAG);
2318 return LowerSETCC(Op, DAG);
2320 return LowerBR_CC(Op, DAG);
2322 return LowerSELECT(Op, DAG);
2324 return LowerSELECT_CC(Op, DAG);
2326 return LowerJumpTable(Op, DAG);
2328 return LowerConstantPool(Op, DAG);
2330 return LowerBlockAddress(Op, DAG);
2332 return LowerVASTART(Op, DAG);
2334 return LowerVACOPY(Op, DAG);
2336 return LowerVAARG(Op, DAG);
2341 return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
2348 return LowerXALUO(Op, DAG);
2350 return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
2352 return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
2354 return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
2356 return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
2358 return LowerFP_ROUND(Op, DAG);
2360 return LowerFP_EXTEND(Op, DAG);
2362 return LowerFRAMEADDR(Op, DAG);
2364 return LowerRETURNADDR(Op, DAG);
2366 return LowerINSERT_VECTOR_ELT(Op, DAG);
2368 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
2370 return LowerBUILD_VECTOR(Op, DAG);
2372 return LowerVECTOR_SHUFFLE(Op, DAG);
2374 return LowerEXTRACT_SUBVECTOR(Op, DAG);
2378 return LowerVectorSRA_SRL_SHL(Op, DAG);
2380 return LowerShiftLeftParts(Op, DAG);
2383 return LowerShiftRightParts(Op, DAG);
2385 return LowerCTPOP(Op, DAG);
2387 return LowerFCOPYSIGN(Op, DAG);
2389 return LowerVectorAND(Op, DAG);
2391 return LowerVectorOR(Op, DAG);
2393 return LowerXOR(Op, DAG);
2395 return LowerPREFETCH(Op, DAG);
2398 return LowerINT_TO_FP(Op, DAG);
2401 return LowerFP_TO_INT(Op, DAG);
2403 return LowerFSINCOS(Op, DAG);
2405 return LowerMUL(Op, DAG);
2407 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3358 SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
3361 SDLoc DL(Op);
3362 const GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
3367 assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
3428 AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
3432 SDLoc DL(Op);
3434 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3503 AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
3513 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
3527 SDLoc DL(Op);
3603 SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
3606 return LowerDarwinGlobalTLSAddress(Op, DAG);
3608 return LowerELFGlobalTLSAddress(Op, DAG);
3612 SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3613 SDValue Chain = Op.getOperand(0);
3614 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
3615 SDValue LHS = Op.getOperand(2);
3616 SDValue RHS = Op.getOperand(3);
3617 SDValue Dest = Op.getOperand(4);
3618 SDLoc dl(Op);
3743 SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
3745 EVT VT = Op.getValueType();
3746 SDLoc DL(Op);
3748 SDValue In1 = Op.getOperand(0);
3749 SDValue In2 = Op.getOperand(1);
3818 SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
3834 SDValue Val = Op.getOperand(0);
3835 SDLoc DL(Op);
3836 EVT VT = Op.getValueType();
3852 SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3854 if (Op.getValueType().isVector())
3855 return LowerVSETCC(Op, DAG);
3857 SDValue LHS = Op.getOperand(0);
3858 SDValue RHS = Op.getOperand(1);
3859 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
3860 SDLoc dl(Op);
3863 EVT VT = Op.getValueType();
3874 assert(LHS.getValueType() == Op.getValueType() &&
4065 SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
4067 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
4068 SDValue LHS = Op.getOperand(0);
4069 SDValue RHS = Op.getOperand(1);
4070 SDValue TVal = Op.getOperand(2);
4071 SDValue FVal = Op.getOperand(3);
4072 SDLoc DL(Op);
4076 SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
4078 SDValue CCVal = Op->getOperand(0);
4079 SDValue TVal = Op->getOperand(1);
4080 SDValue FVal = Op->getOperand(2);
4081 SDLoc DL(Op);
4098 return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
4117 SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
4121 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
4123 SDLoc DL(Op);
4145 SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
4147 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
4149 SDLoc DL(Op);
4186 SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
4188 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
4190 SDLoc DL(Op);
4209 SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
4214 SDLoc DL(Op);
4217 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4218 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
4222 SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
4229 SDLoc DL(Op);
4231 SDValue Chain = Op.getOperand(0);
4232 SDValue VAList = Op.getOperand(1);
4233 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4291 SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
4293 return Subtarget->isTargetDarwin() ? LowerDarwin_VASTART(Op, DAG)
4294 : LowerAAPCS_VASTART(Op, DAG);
4297 SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
4301 SDLoc DL(Op);
4303 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
4304 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
4306 return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1),
4307 Op.getOperand(2),
4313 SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
4317 const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4318 EVT VT = Op.getValueType();
4319 SDLoc DL(Op);
4320 SDValue Chain = Op.getOperand(0);
4321 SDValue Addr = Op.getOperand(1);
4322 unsigned Align = Op.getConstantOperandVal(3);
4376 SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
4381 EVT VT = Op.getValueType();
4382 SDLoc DL(Op);
4383 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4405 SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
4411 EVT VT = Op.getValueType();
4412 SDLoc DL(Op);
4413 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4415 Op, DAG);
4429 SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
4431 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
4432 EVT VT = Op.getValueType();
4434 SDLoc dl(Op);
4435 SDValue ShOpLo = Op.getOperand(0);
4436 SDValue ShOpHi = Op.getOperand(1);
4437 SDValue ShAmt = Op.getOperand(2);
4438 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
4440 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
4487 SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
4489 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
4490 EVT VT = Op.getValueType();
4492 SDLoc dl(Op);
4493 SDValue ShOpLo = Op.getOperand(0);
4494 SDValue ShOpHi = Op.getOperand(1);
4495 SDValue ShAmt = Op.getOperand(2);
4497 assert(Op.getOpcode() == ISD::SHL_PARTS);
4755 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
4772 if (!isNullConstant(Op))
4775 if (Op.getValueType() == MVT::i64)
4788 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
4877 Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64);
4886 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
4927 SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
4929 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
4930 SDLoc dl(Op);
4931 EVT VT = Op.getValueType();
4959 SDValue V = Op.getOperand(i);
5083 SDValue Entry = Op.getOperand(i);
5370 static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
5371 SDLoc DL(Op);
5372 EVT VT = Op.getValueType();
5373 SDValue V0 = Op.getOperand(0);
5374 SDValue V1 = Op.getOperand(1);
5375 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
5503 static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
5506 SDValue V1 = Op.getOperand(0);
5507 SDValue V2 = Op.getOperand(1);
5508 SDLoc DL(Op);
5510 EVT EltVT = Op.getValueType().getVectorElementType();
5523 if (Op.getValueType().getSizeInBits() == 128) {
5562 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
5578 SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
5580 SDLoc dl(Op);
5581 EVT VT = Op.getValueType();
5583 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
5591 SDValue V1 = Op.getOperand(0);
5592 SDValue V2 = Op.getOperand(1);
5676 if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
5727 return GenerateTBL(Op, ShuffleMask, DAG);
5752 SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
5755 dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
5756 SDValue LHS = Op.getOperand(0);
5757 SDLoc dl(Op);
5758 EVT VT = Op.getValueType();
5761 return Op;
5842 return Op;
5891 // Is the first op an AND?
5896 // Is the second op an shl or lshr?
5944 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
5948 if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
5953 dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
5954 SDValue LHS = Op.getOperand(1);
5955 SDLoc dl(Op);
5956 EVT VT = Op.getValueType();
5960 LHS = Op.getOperand(0);
5961 BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
5964 return Op;
6042 return Op;
6047 static SDValue NormalizeBuildVector(SDValue Op,
6049 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
6050 SDLoc dl(Op);
6051 EVT VT = Op.getValueType();
6055 return Op;
6058 for (SDValue Lane : Op->ops()) {
6069 SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
6071 SDLoc dl(Op);
6072 EVT VT = Op.getValueType();
6073 Op = NormalizeBuildVector(Op, DAG);
6074 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
6093 return Op;
6310 SDValue V = Op.getOperand(i);
6365 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i)));
6382 SDValue V = Op.getOperand(i);
6401 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
6413 SDValue Op0 = Op.getOperand(0);
6432 SDValue V = Op.getOperand(i);
6445 SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
6447 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!");
6450 EVT VT = Op.getOperand(0).getValueType();
6451 ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(2));
6460 return Op;
6468 SDLoc DL(Op);
6469 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
6473 Op.getOperand(1), Op.getOperand(2));
6479 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
6481 assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!");
6484 EVT VT = Op.getOperand(0).getValueType();
6485 ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6494 return Op;
6502 SDLoc DL(Op);
6503 SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
6512 Op.getOperand(1));
6515 Op,
6517 EVT VT = Op.getOperand(0).getValueType();
6518 SDLoc dl(Op);
6523 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6528 unsigned Size = Op.getValueType().getSizeInBits();
6532 return Op;
6537 return Op;
6584 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6586 while (Op.getOpcode() == ISD::BITCAST)
6587 Op = Op.getOperand(0);
6588 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
6604 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6607 if (!getVShiftImm(Op, ElementBits, Cnt))
6615 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
6618 if (!getVShiftImm(Op, ElementBits, Cnt))
6623 SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
6625 EVT VT = Op.getValueType();
6626 SDLoc DL(Op);
6629 if (!Op.getOperand(1).getValueType().isVector())
6630 return Op;
6633 switch (Op.getOpcode()) {
6638 if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
6639 return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
6644 Op.getOperand(0), Op.getOperand(1));
6648 if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
6650 (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR;
6651 return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
6658 unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl
6661 SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
6664 DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0),
6767 SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
6769 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
6770 SDValue LHS = Op.getOperand(0);
6771 SDValue RHS = Op.getOperand(1);
6773 SDLoc dl(Op);
6780 return DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
6810 Cmp = DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
7691 SDValue Op = N->getOperand(0);
7692 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
7693 Op.getOpcode() != ISD::FMUL)
7696 SDValue ConstVec = Op->getOperand(1);
7700 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
7722 unsigned NumLanes = Op.getValueType().getVectorNumElements();
7747 Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32));
7762 SDValue Op = N->getOperand(0);
7763 unsigned Opc = Op->getOpcode();
7764 if (!Op.getValueType().isVector() ||
7772 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
7793 unsigned NumLanes = Op.getValueType().getVectorNumElements();
7806 SDValue ConvInput = Op.getOperand(0);
7814 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
8202 // integer op (addl, smull, ...).
8255 /// \brief Check whether or not \p Op is a SET_CC operation, either a generic or
8260 /// \return True when Op is a kind of SET_CC operation.
8261 static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
8263 if (Op.getOpcode() == ISD::SETCC) {
8264 SetCCInfo.Info.Generic.Opnd0 = &Op.getOperand(0);
8265 SetCCInfo.Info.Generic.Opnd1 = &Op.getOperand(1);
8266 SetCCInfo.Info.Generic.CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
8274 if (Op.getOpcode() != AArch64ISD::CSEL)
8278 SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3);
8281 cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
8286 ConstantSDNode *TValue = dyn_cast<ConstantSDNode>(Op.getOperand(0));
8287 ConstantSDNode *FValue = dyn_cast<ConstantSDNode>(Op.getOperand(1));
8303 // Returns true if Op is setcc or zext of setcc.
8304 static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
8305 if (isSetCC(Op, Info))
8307 return ((Op.getOpcode() == ISD::ZERO_EXTEND) &&
8308 isSetCC(Op->getOperand(0), Info));
8317 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
8318 assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!");
8319 SDValue LHS = Op->getOperand(0);
8320 SDValue RHS = Op->getOperand(1);
8339 SDLoc dl(Op);
8351 EVT VT = Op->getValueType(0);
8945 unsigned Op,
8952 if (Op == ISD::FMAXNUM || Op == ISD::FMINNUM) {
8978 if (CurOp.getOpcode() != Op && (CurStep != (NumExpectedSteps - 1)))
9011 switch (Op) {
9095 unsigned Op = VectorOp->getOpcode();
9097 if (Op != ISD::SMAX && Op != ISD::UMAX && Op != ISD::SMIN &&
9098 Op != ISD::UMIN && Op != ISD::FMAXNUM && Op != ISD::FMINNUM)
9109 if (Op == ISD::FMAXNUM || Op == ISD::FMINNUM) {
9129 if ((Op == ISD::SMAX && CC != ISD::SETGT && CC != ISD::SETGE) ||
9130 (Op == ISD::UMAX && CC != ISD::SETUGT && CC != ISD::SETUGE) ||
9131 (Op == ISD::SMIN && CC != ISD::SETLT && CC != ISD::SETLE) ||
9132 (Op == ISD::UMIN && CC != ISD::SETULT && CC != ISD::SETULE) ||
9133 (Op == ISD::FMAXNUM && CC != ISD::SETOGT && CC != ISD::SETOGE &&
9136 (Op == ISD::FMINNUM && CC != ISD::SETOLT && CC != ISD::SETOLE &&
9153 return tryMatchAcrossLaneShuffleForReduction(N, SetCC, Op, DAG);
9648 static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
9651 if (!Op->hasOneUse())
9652 return Op;
9660 if (Op->getOpcode() == ISD::TRUNCATE &&
9661 Bit < Op->getValueType(0).getSizeInBits()) {
9662 return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
9665 if (Op->getNumOperands() != 2)
9666 return Op;
9668 auto *C = dyn_cast<ConstantSDNode>(Op->getOperand(1));
9670 return Op;
9672 switch (Op->getOpcode()) {
9674 return Op;
9679 return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
9680 return Op;
9685 (Bit - C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
9687 return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
9689 return Op;
9694 if (Bit >= Op->getValueType(0).getSizeInBits())
9695 Bit = Op->getValueType(0).getSizeInBits() - 1;
9696 return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
9700 if ((Bit + C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
9702 return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
9704 return Op;
9710 return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
9984 bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
9989 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
9992 Base = Op->getOperand(0);
9995 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1))) {
9999 IsInc = (Op->getOpcode() == ISD::ADD);
10000 Offset = Op->getOperand(1);
10029 SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
10043 if (!getIndexedAddressParts(Op, Base, Offset, AM, IsInc, DAG))
10056 SDValue Op = N->getOperand(0);
10058 if (N->getValueType(0) != MVT::i16 || Op.getValueType() != MVT::f16)
10061 Op = SDValue(
10063 DAG.getUNDEF(MVT::i32), Op,
10066 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
10067 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
