Lines Matching defs:DAG
1 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
494 SDLoc DL, SelectionDAG &DAG) const {
495 return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain);
505 SelectionDAG &DAG = CLI.DAG;
507 const Function &Fn = *DAG.getMachineFunction().getFunction();
517 DAG.getContext()->diagnose(NoCalls);
522 SelectionDAG &DAG) const {
529 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
530 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
531 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
532 case ISD::FrameIndex: return LowerFrameIndex(Op, DAG);
533 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
534 case ISD::SDIV: return LowerSDIV(Op, DAG);
535 case ISD::SREM: return LowerSREM(Op, DAG);
536 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
537 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
538 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
539 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
540 case ISD::FRINT: return LowerFRINT(Op, DAG);
541 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
542 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
543 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
550 SelectionDAG &DAG) const {
561 SDNode *Node = LowerLOAD(SDValue(N, 0), DAG).getNode();
569 DAG.ReplaceAllUsesOfValueWith(SDValue(N,1), SDValue(Node, 1));
573 SDValue Lowered = LowerSTORE(SDValue(N, 0), DAG);
597 SelectionDAG &DAG) const {
605 return DAG.getStore(Chain, DL, DAG.getConstant(*CI, VT), InitPtr,
613 return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, VT), InitPtr,
625 SDValue Offset = DAG.getConstant(SL->getElementOffset(I), PtrVT);
626 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
629 Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG));
632 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
649 SDValue Offset = DAG.getConstant(i * EltSize, PtrVT);
650 SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset);
653 Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG));
656 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
662 return DAG.getStore(Chain, DL, DAG.getUNDEF(VT), InitPtr,
673 SelectionDAG &DAG) const {
698 return DAG.getConstant(Offset, getPointerTy(G->getAddressSpace()));
701 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
710 SDValue InitPtr = DAG.getFrameIndex(FI, PrivPtrVT);
715 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT);
721 for (SDNode::use_iterator I = DAG.getEntryNode()->use_begin(),
722 E = DAG.getEntryNode()->use_end(); I != E; ++I) {
727 SDValue Chain = LowerConstantInitializer(Init, GV, InitPtr, DAG.getEntryNode(), DAG);
735 DAG.UpdateNodeOperands(*I, Ops);
737 return DAG.getZExtOrTrunc(InitPtr, SDLoc(Op), ConstPtrVT);
743 SelectionDAG &DAG) const {
748 DAG.ExtractVectorElements(A, Args);
749 DAG.ExtractVectorElements(B, Args);
751 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
755 SelectionDAG &DAG) const {
760 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
763 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), Args);
767 SelectionDAG &DAG) const {
769 MachineFunction &MF = DAG.getMachineFunction();
777 return DAG.getConstant(Offset * 4 * TFL->getStackWidth(MF),
782 SelectionDAG &DAG) const {
791 return LowerIntrinsicIABS(Op, DAG);
793 return LowerIntrinsicLRP(Op, DAG);
796 return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1));
800 return DAG.getNode(AMDGPUISD::CLAMP, DL, VT,
807 return DAG.getUNDEF(VT);
815 return DAG.getNode(AMDGPUISD::DIV_SCALE, DL, VT,
820 return DAG.getNode(AMDGPUISD::DIV_FMAS, DL, VT,
824 return DAG.getNode(AMDGPUISD::DIV_FIXUP, DL, VT,
828 return DAG.getNode(AMDGPUISD::TRIG_PREOP, DL, VT,
832 return DAG.getNode(AMDGPUISD::RCP, DL, VT, Op.getOperand(1));
835 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
838 return DAG.getNode(AMDGPUISD::RSQ_LEGACY, DL, VT, Op.getOperand(1));
841 return DAG.getNode(AMDGPUISD::RSQ_CLAMPED, DL, VT, Op.getOperand(1));
844 return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1),
847 return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1),
850 return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1),
853 return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1),
857 return DAG.getNode(AMDGPUISD::MUL_U24, DL, VT,
861 return DAG.getNode(AMDGPUISD::MUL_I24, DL, VT,
865 return DAG.getNode(AMDGPUISD::MAD_U24, DL, VT,
869 return DAG.getNode(AMDGPUISD::MAD_I24, DL, VT,
873 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE0, DL, VT, Op.getOperand(1));
876 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE1, DL, VT, Op.getOperand(1));
879 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE2, DL, VT, Op.getOperand(1));
882 return DAG.getNode(AMDGPUISD::CVT_F32_UBYTE3, DL, VT, Op.getOperand(1));
885 return DAG.getNode(AMDGPUISD::BFE_I32, DL, VT,
891 return DAG.getNode(AMDGPUISD::BFE_U32, DL, VT,
897 return DAG.getNode(AMDGPUISD::BFI, DL, VT,
903 return DAG.getNode(AMDGPUISD::BFM, DL, VT,
908 return DAG.getNode(AMDGPUISD::BREV, DL, VT, Op.getOperand(1));
911 return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1));
914 return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1));
916 return DAG.getNode(ISD::FTRUNC, DL, VT, Op.getOperand(1));
922 SelectionDAG &DAG) const {
925 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
928 return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1));
934 SelectionDAG &DAG) const {
937 SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT,
938 DAG.getConstantFP(1.0f, MVT::f32),
940 SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA,
942 return DAG.getNode(ISD::FADD, DL, VT,
943 DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)),
949 SelectionDAG &DAG) const {
986 return DAG.getNode(Opc, DL, VT, LHS, RHS);
995 return DAG.getNode(Opc, DL, VT, LHS, RHS);
1004 SelectionDAG &DAG) const {
1018 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT, Load->getBasePtr(),
1019 DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8), PtrVT));
1022 = DAG.getExtLoad(Load->getExtensionType(), SL, EltVT,
1032 DAG.getNode(ISD::BUILD_VECTOR, SL, LoadVT, Loads),
1033 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains)
1036 return DAG.getMergeValues(Ops, SL);
1040 SelectionDAG &DAG) const {
1061 SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, MVT::i32);
1067 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value,
1068 DAG.getConstant(i, MVT::i32));
1069 Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32);
1070 Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg
1072 SDValue Shift = DAG.getConstant(MemEltBits * i, MVT::i32);
1073 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift);
1078 PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt);
1083 EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), PackedSize);
1084 return DAG.getTruncStore(Store->getChain(), DL, PackedValue, Ptr,
1091 return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr,
1098 SelectionDAG &DAG) const {
1109 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
1110 Store->getValue(), DAG.getConstant(i, MVT::i32));
1111 SDValue Ptr = DAG.getNode(ISD::ADD, SL, PtrVT,
1113 DAG.getConstant(i * (MemEltVT.getSizeInBits() / 8),
1115 Chains.push_back(DAG.getTruncStore(Store->getChain(), SL, Val, Ptr,
1120 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains);
1123 SDValue AMDGPUTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1134 SDValue ExtLoad32 = DAG.getExtLoad(ExtType, DL, MVT::i32,
1141 DAG.getNode(ISD::getExtForLoadExtType(ExtType), DL, VT, ExtLoad32),
1145 return DAG.getMergeValues(Ops, DL);
1157 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain,
1161 DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD),
1165 return DAG.getMergeValues(Ops, DL);
1174 SDValue Ptr = DAG.getZExtOrTrunc(Load->getBasePtr(), DL,
1176 Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Ptr,
1177 DAG.getConstant(2, MVT::i32));
1178 return DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op->getVTList(),
1180 DAG.getTargetConstant(0, MVT::i32), Op.getOperand(2));
1188 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(),
1189 DAG.getConstant(2, MVT::i32));
1190 SDValue Ret = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, Op.getValueType(),
1192 DAG.getTargetConstant(0, MVT::i32),
1194 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32,
1196 DAG.getConstant(0x3, MVT::i32));
1197 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1198 DAG.getConstant(3, MVT::i32));
1200 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt);
1204 SDValue MemEltVTNode = DAG.getValueType(MemEltVT);
1207 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode),
1211 return DAG.getMergeValues(Ops, DL);
1215 DAG.getZeroExtendInReg(Ret, DL, MemEltVT),
1219 return DAG.getMergeValues(Ops, DL);
1222 SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1224 SDValue Result = AMDGPUTargetLowering::MergeVectorStore(Op, DAG);
1234 return SplitVectorStore(Op, DAG);
1247 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr,
1248 DAG.getConstant(2, MVT::i32));
1249 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32,
1250 Chain, Ptr, DAG.getTargetConstant(0, MVT::i32));
1252 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr,
1253 DAG.getConstant(0x3, MVT::i32));
1255 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx,
1256 DAG.getConstant(3, MVT::i32));
1258 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32,
1261 SDValue MaskedValue = DAG.getZeroExtendInReg(SExtValue, DL, MemVT);
1263 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32,
1266 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, DAG.getConstant(Mask, MVT::i32),
1268 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask,
1269 DAG.getConstant(0xffffffff, MVT::i32));
1270 Dst = DAG
1272 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue);
1273 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other,
1274 Chain, Value, Ptr, DAG.getTargetConstant(0, MVT::i32));
1279 SDValue AMDGPUTargetLowering::LowerSDIV24(SDValue Op, SelectionDAG &DAG) const {
1298 SDValue jq = DAG.getNode(ISD::XOR, DL, OVT, LHS, RHS);
1301 jq = DAG.getNode(ISD::SRA, DL, OVT, jq, DAG.getConstant(bitsize - 2, OVT));
1304 jq = DAG.getNode(ISD::OR, DL, OVT, jq, DAG.getConstant(1, OVT));
1307 jq = DAG.getSExtOrTrunc(jq, DL, INTTY);
1310 SDValue ia = DAG.getSExtOrTrunc(LHS, DL, INTTY);
1313 SDValue ib = DAG.getSExtOrTrunc(RHS, DL, INTTY);
1316 SDValue fa = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ia);
1319 SDValue fb = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ib);
1322 SDValue fq = DAG.getNode(ISD::FMUL, DL, FLTTY,
1323 fa, DAG.getNode(AMDGPUISD::RCP, DL, FLTTY, fb));
1326 fq = DAG.getNode(ISD::FTRUNC, DL, FLTTY, fq);
1329 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FLTTY, fq);
1332 SDValue fr = DAG.getNode(ISD::FADD, DL, FLTTY,
1333 DAG.getNode(ISD::MUL, DL, FLTTY, fqneg, fb), fa);
1336 SDValue iq = DAG.getNode(ISD::FP_TO_SINT, DL, INTTY, fq);
1339 fr = DAG.getNode(ISD::FABS, DL, FLTTY, fr);
1342 fb = DAG.getNode(ISD::FABS, DL, FLTTY, fb);
1347 cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE);
1349 cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE);
1352 jq = DAG.getNode(ISD::SELECT, DL, OVT, cv, jq,
1353 DAG.getConstant(0, OVT));
1355 iq = DAG.getSExtOrTrunc(iq, DL, OVT);
1356 iq = DAG.getNode(ISD::ADD, DL, OVT, iq, jq);
1360 SDValue AMDGPUTargetLowering::LowerSDIV32(SDValue Op, SelectionDAG &DAG) const {
1386 SDValue r10 = DAG.getSelectCC(DL,
1387 r0, DAG.getConstant(0, OVT),
1388 DAG.getConstant(-1, OVT),
1389 DAG.getConstant(0, OVT),
1393 SDValue r11 = DAG.getSelectCC(DL,
1394 r1, DAG.getConstant(0, OVT),
1395 DAG.getConstant(-1, OVT),
1396 DAG.getConstant(0, OVT),
1400 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
1403 r1 = DAG.getNode(ISD::ADD, DL, OVT, r1, r11);
1406 r0 = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
1409 r1 = DAG.getNode(ISD::XOR, DL, OVT, r1, r11);
1412 r0 = DAG.getNode(ISD::UDIV, DL, OVT, r0, r1);
1415 r10 = DAG.getNode(ISD::XOR, DL, OVT, r10, r11);
1418 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
1421 SDValue DST = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
1425 SDValue AMDGPUTargetLowering::LowerSDIV64(SDValue Op, SelectionDAG &DAG) const {
1429 SDValue AMDGPUTargetLowering::LowerSDIV(SDValue Op, SelectionDAG &DAG) const {
1433 return LowerSDIV64(Op, DAG);
1436 return LowerSDIV32(Op, DAG);
1441 return LowerSDIV24(Op, DAG);
1447 SDValue AMDGPUTargetLowering::LowerSREM32(SDValue Op, SelectionDAG &DAG) const {
1474 SDValue r10 = DAG.getSetCC(DL, OVT, r0, DAG.getConstant(0, OVT), ISD::SETLT);
1477 SDValue r11 = DAG.getSetCC(DL, OVT, r1, DAG.getConstant(0, OVT), ISD::SETLT);
1480 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
1483 r1 = DAG.getNode(ISD::ADD, DL, OVT, r1, r11);
1486 r0 = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
1489 r1 = DAG.getNode(ISD::XOR, DL, OVT, r1, r11);
1492 SDValue r20 = DAG.getNode(ISD::UREM, DL, OVT, r0, r1);
1495 r20 = DAG.getNode(AMDGPUISD::UMUL, DL, OVT, r20, r1);
1498 r0 = DAG.getNode(ISD::SUB, DL, OVT, r0, r20);
1501 r0 = DAG.getNode(ISD::ADD, DL, OVT, r0, r10);
1504 SDValue DST = DAG.getNode(ISD::XOR, DL, OVT, r0, r10);
1508 SDValue AMDGPUTargetLowering::LowerSREM64(SDValue Op, SelectionDAG &DAG) const {
1512 SDValue AMDGPUTargetLowering::LowerSREM(SDValue Op, SelectionDAG &DAG) const {
1516 return LowerSREM64(Op, DAG);
1519 return LowerSREM32(Op, DAG);
1525 SelectionDAG &DAG) const {
1534 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1537 SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den);
1540 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1543 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT),
1547 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
1552 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1555 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1558 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1561 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT),
1565 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1568 SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den);
1571 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1574 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
1575 DAG.getConstant(-1, VT),
1576 DAG.getConstant(0, VT),
1579 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1581 DAG.getConstant(-1, VT),
1582 DAG.getConstant(0, VT),
1585 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1591 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
1592 DAG.getConstant(1, VT));
1595 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
1596 DAG.getConstant(1, VT));
1599 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
1603 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
1609 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1612 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1615 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT),
1619 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT),
1625 return DAG.getMergeValues(Ops, DL);
1629 SelectionDAG &DAG) const {
1633 SDValue Zero = DAG.getConstant(0, VT);
1634 SDValue NegOne = DAG.getConstant(-1, VT);
1639 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
1640 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
1641 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
1644 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
1645 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
1647 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
1648 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
1650 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
1653 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
1654 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
1656 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
1657 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
1663 return DAG.getMergeValues(Res, DL);
1666 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
1674 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1676 const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64);
1677 const SDValue One = DAG.getConstantFP(1.0, MVT::f64);
1679 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
1681 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
1682 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1683 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1685 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
1686 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1689 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
1695 const SDValue Zero = DAG.getConstant(0, MVT::i32);
1696 const SDValue One = DAG.getConstant(1, MVT::i32);
1698 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
1702 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
1708 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_I32, SL, MVT::i32,
1710 DAG.getConstant(FractBits - 32, MVT::i32),
1711 DAG.getConstant(ExpBits, MVT::i32));
1712 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
1713 DAG.getConstant(1023, MVT::i32));
1716 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, MVT::i32);
1717 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
1720 SDValue SignBit64 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
1722 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
1724 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
1726 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, MVT::i64);
1728 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
1729 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
1730 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
1732 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::i32);
1734 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, MVT::i32);
1736 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
1737 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
1739 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
1740 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
1742 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
1745 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
1752 SDValue C1 = DAG.getConstantFP(C1Val, MVT::f64);
1753 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
1755 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
1756 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
1758 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
1761 SDValue C2 = DAG.getConstantFP(C2Val, MVT::f64);
1763 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
1764 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
1766 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
1769 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
1773 return DAG
1776 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
1784 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
1786 const SDValue Zero = DAG.getConstantFP(0.0, MVT::f64);
1787 const SDValue NegOne = DAG.getConstantFP(-1.0, MVT::f64);
1789 EVT SetCCVT = getSetCCResultType(*DAG.getContext(), MVT::f64);
1791 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
1792 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
1793 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
1795 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
1796 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
1800 SelectionDAG &DAG) const {
1807 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
1808 DAG.getConstant(0, MVT::i32));
1809 SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo);
1810 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0,
1811 DAG.getConstant(1, MVT::i32));
1812 SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi);
1813 FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi,
1814 DAG.getConstantFP(4294967296.0f, MVT::f32)); // 2^32
1815 return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi);
1820 SelectionDAG &DAG) const {
1823 SDValue Shift = DAG.getConstant(BitsDiff, VT);
1825 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, Op.getOperand(0), Shift);
1827 return DAG.getNode(ISD::SRA, DL, VT, Shl, Shift);
1831 SelectionDAG &DAG) const {
1845 DAG.ExtractVectorElements(Src, Args, 0, NElts);
1847 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
1849 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
1851 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Args);
1855 // Custom DAG optimizations
1858 static bool isU24(SDValue Op, SelectionDAG &DAG) {
1861 DAG.computeKnownBits(Op, KnownZero, KnownOne);
1866 static bool isI24(SDValue Op, SelectionDAG &DAG) {
1873 (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24;
1878 SelectionDAG &DAG = DCI.DAG;
1879 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1884 TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
1890 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0,
1894 return DAG.getConstant(Result, MVT::i32);
1897 return DAG.getConstant(Src0 >> Offset, MVT::i32);
1907 SelectionDAG &DAG = DCI.DAG;
1914 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
1915 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
1916 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
1917 Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1);
1918 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
1919 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
1920 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
1921 Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1);
1928 return DAG.getSExtOrTrunc(Mul, DL, VT);
1933 SelectionDAG &DAG = DCI.DAG;
1949 return CombineMinMax(N, DAG);
1961 return DAG.getConstant(0, MVT::i32);
1976 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
1980 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
1983 // DAG Combines. If not eliminated, we will match back to BFE during
1988 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
1989 DAG.getValueType(SmallVT));
1992 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
1997 return constantFoldBFE<int32_t>(DAG,
2003 return constantFoldBFE<uint32_t>(DAG,
2014 SDValue ShiftVal = DAG.getConstant(OffsetVal, MVT::i32);
2015 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
2020 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2022 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2039 SelectionDAG &DAG,
2090 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
2093 MachineFunction &MF = DAG.getMachineFunction();
2102 return DAG.getRegister(VirtualRegister, VT);
2110 // AMDIL DAG nodes
2116 // AMDGPU DAG nodes
2169 const SelectionDAG &DAG,
2173 DAG.computeKnownBits(Op0, Op0Zero, Op0One, Depth);
2174 DAG.computeKnownBits(Op1, Op1Zero, Op1One, Depth);
2184 const SelectionDAG &DAG,
2204 KnownZero, KnownOne, DAG, Depth);
2217 KnownZero, KnownOne, DAG, Depth);
2248 const SelectionDAG &DAG,
2262 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
