Lines Matching refs:ISD
136 if (N->getOpcode() == ISD::HANDLENODE)
195 /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
198 /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
216 ISD::NodeType ExtType);
337 SDValue N3, ISD::CondCode CC,
339 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
573 if (Op.getOpcode() == ISD::FNEG) return 2;
583 case ISD::ConstantFP:
587 case ISD::FADD:
593 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType()))
603 case ISD::FSUB:
610 case ISD::FMUL:
611 case ISD::FDIV:
622 case ISD::FP_EXTEND:
623 case ISD::FP_ROUND:
624 case ISD::FSIN:
635 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
646 case ISD::ConstantFP: {
651 case ISD::FADD:
658 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
663 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
667 case ISD::FSUB:
677 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
680 case ISD::FMUL:
681 case ISD::FDIV:
698 case ISD::FP_EXTEND:
699 case ISD::FSIN:
703 case ISD::FP_ROUND:
704 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
718 if (N.getOpcode() == ISD::SETCC) {
725 if (N.getOpcode() != ISD::SELECT_CC ||
771 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode()))
781 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode()))
948 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
967 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
968 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
969 : ISD::EXTLOAD)
980 case ISD::AssertSext:
981 return DAG.getNode(ISD::AssertSext, dl, PVT,
984 case ISD::AssertZext:
985 return DAG.getNode(ISD::AssertZext, dl, PVT,
988 case ISD::Constant: {
990 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
995 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
997 return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
1001 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
1013 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
1083 return DAG.getNode(ISD::TRUNCATE, dl, VT,
1114 if (Opc == ISD::SRA)
1116 else if (Opc == ISD::SRL)
1130 return DAG.getNode(ISD::TRUNCATE, dl, VT,
1189 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1190 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD
1191 : ISD::EXTLOAD)
1196 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
1322 assert(N->getOpcode() != ISD::DELETED_NODE &&
1323 RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
1359 case ISD::TokenFactor: return visitTokenFactor(N);
1360 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1361 case ISD::ADD: return visitADD(N);
1362 case ISD::SUB: return visitSUB(N);
1363 case ISD::ADDC: return visitADDC(N);
1364 case ISD::SUBC: return visitSUBC(N);
1365 case ISD::ADDE: return visitADDE(N);
1366 case ISD::SUBE: return visitSUBE(N);
1367 case ISD::MUL: return visitMUL(N);
1368 case ISD::SDIV: return visitSDIV(N);
1369 case ISD::UDIV: return visitUDIV(N);
1370 case ISD::SREM:
1371 case ISD::UREM: return visitREM(N);
1372 case ISD::MULHU: return visitMULHU(N);
1373 case ISD::MULHS: return visitMULHS(N);
1374 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
1375 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
1376 case ISD::SMULO: return visitSMULO(N);
1377 case ISD::UMULO: return visitUMULO(N);
1378 case ISD::SMIN:
1379 case ISD::SMAX:
1380 case ISD::UMIN:
1381 case ISD::UMAX: return visitIMINMAX(N);
1382 case ISD::AND: return visitAND(N);
1383 case ISD::OR: return visitOR(N);
1384 case ISD::XOR: return visitXOR(N);
1385 case ISD::SHL: return visitSHL(N);
1386 case ISD::SRA: return visitSRA(N);
1387 case ISD::SRL: return visitSRL(N);
1388 case ISD::ROTR:
1389 case ISD::ROTL: return visitRotate(N);
1390 case ISD::BSWAP: return visitBSWAP(N);
1391 case ISD::CTLZ: return visitCTLZ(N);
1392 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N);
1393 case ISD::CTTZ: return visitCTTZ(N);
1394 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N);
1395 case ISD::CTPOP: return visitCTPOP(N);
1396 case ISD::SELECT: return visitSELECT(N);
1397 case ISD::VSELECT: return visitVSELECT(N);
1398 case ISD::SELECT_CC: return visitSELECT_CC(N);
1399 case ISD::SETCC: return visitSETCC(N);
1400 case ISD::SETCCE: return visitSETCCE(N);
1401 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
1402 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
1403 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
1404 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
1405 case ISD::SIGN_EXTEND_VECTOR_INREG: return visitSIGN_EXTEND_VECTOR_INREG(N);
1406 case ISD::TRUNCATE: return visitTRUNCATE(N);
1407 case ISD::BITCAST: return visitBITCAST(N);
1408 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
1409 case ISD::FADD: return visitFADD(N);
1410 case ISD::FSUB: return visitFSUB(N);
1411 case ISD::FMUL: return visitFMUL(N);
1412 case ISD::FMA: return visitFMA(N);
1413 case ISD::FDIV: return visitFDIV(N);
1414 case ISD::FREM: return visitFREM(N);
1415 case ISD::FSQRT: return visitFSQRT(N);
1416 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
1417 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
1418 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
1419 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
1420 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
1421 case ISD::FP_ROUND: return visitFP_ROUND(N);
1422 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N);
1423 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
1424 case ISD::FNEG: return visitFNEG(N);
1425 case ISD::FABS: return visitFABS(N);
1426 case ISD::FFLOOR: return visitFFLOOR(N);
1427 case ISD::FMINNUM: return visitFMINNUM(N);
1428 case ISD::FMAXNUM: return visitFMAXNUM(N);
1429 case ISD::FCEIL: return visitFCEIL(N);
1430 case ISD::FTRUNC: return visitFTRUNC(N);
1431 case ISD::BRCOND: return visitBRCOND(N);
1432 case ISD::BR_CC: return visitBR_CC(N);
1433 case ISD::LOAD: return visitLOAD(N);
1434 case ISD::STORE: return visitSTORE(N);
1435 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
1436 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1437 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
1438 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
1439 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
1440 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
1441 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
1442 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
1443 case ISD::MGATHER: return visitMGATHER(N);
1444 case ISD::MLOAD: return visitMLOAD(N);
1445 case ISD::MSCATTER: return visitMSCATTER(N);
1446 case ISD::MSTORE: return visitMSTORE(N);
1447 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
1448 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
1458 assert(N->getOpcode() != ISD::DELETED_NODE &&
1461 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1462 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1476 case ISD::ADD:
1477 case ISD::SUB:
1478 case ISD::MUL:
1479 case ISD::AND:
1480 case ISD::OR:
1481 case ISD::XOR:
1484 case ISD::SHL:
1485 case ISD::SRA:
1486 case ISD::SRL:
1489 case ISD::SIGN_EXTEND:
1490 case ISD::ZERO_EXTEND:
1491 case ISD::ANY_EXTEND:
1494 case ISD::LOAD:
1563 case ISD::EntryToken:
1569 case ISD::TokenFactor:
1601 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1649 if (ISD::isBuildVectorAllZeros(N1.getNode()))
1651 if (ISD::isBuildVectorAllZeros(N0.getNode()))
1656 if (N0.getOpcode() == ISD::UNDEF)
1658 if (N1.getOpcode() == ISD::UNDEF)
1664 return DAG.FoldConstantArithmetic(ISD::ADD, SDLoc(N), VT, N0C, N1C);
1668 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, N0);
1675 GA->getOpcode() == ISD::GlobalAddress)
1680 if (N1C && N0.getOpcode() == ISD::SUB)
1683 return DAG.getNode(ISD::SUB, DL, VT,
1689 if (SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1))
1692 if (N0.getOpcode() == ISD::SUB && isNullConstant(N0.getOperand(0)))
1693 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, N0.getOperand(1));
1695 if (N1.getOpcode() == ISD::SUB && isNullConstant(N1.getOperand(0)))
1696 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1.getOperand(1));
1698 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
1701 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
1704 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
1706 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0),
1709 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
1711 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0),
1714 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
1715 N1.getOperand(0).getOpcode() == ISD::SUB &&
1721 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
1728 return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1729 DAG.getNode(ISD
1730 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
1737 if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
1739 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1);
1742 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::SUB &&
1744 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0,
1745 DAG.getNode(ISD::SHL, SDLoc(N), VT,
1748 if (N0.getOpcode() == ISD::SHL && N0.getOperand(0).getOpcode() == ISD::SUB &&
1750 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1,
1751 DAG.getNode(ISD::SHL, SDLoc(N), VT,
1755 if (N1.getOpcode() == ISD::AND) {
1764 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0);
1769 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
1771 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
1773 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
1774 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
1778 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1782 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
1784 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
1798 return CombineTo(N, DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N1),
1799 DAG.getNode(ISD::CARRY_FALSE,
1806 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N1, N0);
1810 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
1824 return CombineTo(N, DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1),
1825 DAG.getNode(ISD::CARRY_FALSE,
1841 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
1845 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
1846 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
1858 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
1874 if (ISD::isBuildVectorAllZeros(N1.getNode()))
1886 return DAG.FoldConstantArithmetic(ISD::SUB, SDLoc(N), VT, N0C, N1C);
1890 return DAG.getNode(ISD::ADD, DL, VT, N0,
1895 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
1897 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
1900 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
1903 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
1906 ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? nullptr :
1908 if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
1912 return DAG.getNode(ISD::SUB, DL, VT, NewC,
1916 if (N0.getOpcode() == ISD::ADD &&
1917 (N0.getOperand(1).getOpcode() == ISD::SUB ||
1918 N0.getOperand(1).getOpcode() == ISD::ADD) &&
1923 if (N0.getOpcode() == ISD::ADD &&
1924 N0.getOperand(1).getOpcode() == ISD::ADD &&
1926 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
1929 if (N0.getOpcode() == ISD::SUB &&
1930 N0.getOperand(1).getOpcode() == ISD::SUB &&
1932 return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1936 if (N0.getOpcode() == ISD::UNDEF)
1938 if (N1.getOpcode() == ISD::UNDEF)
1945 if (N1C && GA->getOpcode() == ISD::GlobalAddress)
1957 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1961 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
1963 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
1978 return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
1979 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
1984 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
1988 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
1992 return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
1993 DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
2004 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
2005 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
2016 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2046 return DAG.FoldConstantArithmetic(ISD::MUL, SDLoc(N), VT,
2052 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
2066 return DAG.getNode(ISD::SUB, DL, VT,
2073 return DAG.getNode(ISD::SHL, DL, VT, N0,
2084 return DAG.getNode(ISD::SUB, DL, VT,
2086 DAG.getNode(ISD::SHL, DL, VT, N0,
2093 if (N1IsConst && N0.getOpcode() == ISD::SHL &&
2096 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT,
2099 return DAG.getNode(ISD::MUL, SDLoc(N), VT,
2108 if (N0.getOpcode() == ISD::SHL &&
2113 } else if (N1.getOpcode() == ISD::SHL &&
2120 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT,
2122 return DAG.getNode(ISD::SHL, SDLoc(N), VT,
2129 N0.getOpcode() == ISD::ADD &&
2132 return DAG.getNode(ISD::ADD, SDLoc(N), VT,
2133 DAG.getNode(ISD::MUL, SDLoc(N0), VT,
2135 DAG.getNode(ISD::MUL, SDLoc(N1), VT,
2139 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1))
2171 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
2173 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2182 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
2183 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
2187 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
2218 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
2220 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
2243 return DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, N0C, N1C);
2249 return DAG.getNode(ISD::SUB, DL, VT,
2256 return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
2275 DAG.getNode(ISD::SRA, DL, VT, N0,
2282 DAG.getNode(ISD::SRL, DL, VT, SGN,
2285 SDValue ADD = DAG.getNode(ISD::ADD, DL, VT, N0, SRL);
2288 SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, ADD,
2298 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
2317 if (N0.getOpcode() == ISD::UNDEF)
2320 if (N1.getOpcode() == ISD::UNDEF)
2342 if (SDValue Folded = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT,
2347 return DAG.getNode(ISD::SRL, DL, VT, N0,
2352 if (N1.getOpcode() == ISD::SHL) {
2356 SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT,
2362 return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
2381 if (N0.getOpcode() == ISD::UNDEF)
2384 if (N1.getOpcode() == ISD::UNDEF)
2390 // handles ISD::SREM and ISD::UREM
2396 bool isSigned = (Opcode == ISD::SREM);
2411 return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
2417 return DAG.getNode(ISD::AND, DL, VT, N0,
2421 if (N1.getOpcode() == ISD::SHL) {
2425 DAG.getNode(ISD::ADD, DL, VT, N1,
2429 return DAG.getNode(ISD::AND, DL, VT, N0, Add);
2446 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
2451 assert((OptimizedDiv.getOpcode() != ISD::UDIVREM) &&
2452 (OptimizedDiv.getOpcode() != ISD::SDIVREM));
2453 SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
2454 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
2465 if (N0.getOpcode() == ISD::UNDEF)
2468 if (N1.getOpcode() == ISD::UNDEF)
2486 return DAG.getNode(ISD::SRA, DL, N0.getValueType(), N0,
2492 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2501 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2502 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
2503 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
2504 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2505 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
2508 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2528 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2537 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2538 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
2539 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
2540 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2541 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
2544 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2603 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
2615 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2616 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
2617 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
2618 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2620 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
2623 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2625 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2634 if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
2646 if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2647 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
2648 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
2649 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2651 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
2654 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2656 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2668 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
2678 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
2728 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
2729 N0.getOpcode() == ISD::SIGN_EXTEND ||
2730 N0.getOpcode() == ISD::BSWAP ||
2732 (N0.getOpcode() == ISD::ANY_EXTEND &&
2734 (N0.getOpcode() == ISD::TRUNCATE &&
2752 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
2753 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
2770 if ((N0.getOpcode() == ISD::BITCAST ||
2771 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
2800 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
2817 if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) {
2838 if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) {
2870 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2875 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
2876 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
2881 if (isNullConstant(LR) && Op1 == ISD::SETEQ) {
2882 SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
2889 if (Op1 == ISD::SETEQ) {
2890 SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0),
2896 if (Op1 == ISD::SETGT) {
2897 SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
2907 Op0 == ISD::SETNE && ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
2910 SDValue ADDNode = DAG.getNode(ISD::ADD, DL, LL.getValueType(),
2916 ISD::SETUGE);
2920 Op1 = ISD::getSetCCSwappedOperands(Op1);
2925 ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger);
2926 if (Result != ISD::SETCC_INVALID &&
2929 TLI.isOperationLegal(ISD::SETCC, LL.getValueType())))) {
2939 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
2956 DAG.getNode(ISD::ADD, DL, VT,
2984 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))) {
3001 !TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, ExtVT))
3004 if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT))
3022 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3028 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3036 if (ISD::isBuildVectorAllOnes(N0.getNode()))
3038 if (ISD::isBuildVectorAllOnes(N1.getNode()))
3046 return DAG.FoldConstantArithmetic(ISD::AND, SDLoc(N), VT, N0C, N1C);
3050 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
3060 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1))
3063 if (N1C && N0.getOpcode() == ISD::OR)
3068 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
3073 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
3092 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
3094 N0.getOperand(0).getOpcode() == ISD::LOAD) ||
3095 N0.getOpcode() == ISD::LOAD) {
3096 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
3145 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
3158 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
3159 case ISD::ZEXTLOAD:
3160 case ISD::NON_EXTLOAD: B = true; break;
3167 if (Load->getExtensionType() == ISD::EXTLOAD) {
3168 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
3195 if (N1C && (N0.getOpcode() == ISD::LOAD ||
3196 (N0.getOpcode() == ISD::ANY_EXTEND &&
3197 N0.getOperand(0).getOpcode() == ISD::LOAD))) {
3198 bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND;
3202 if (LN0->getExtensionType() != ISD::SEXTLOAD &&
3211 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy,
3231 NewPtr = DAG.getNode(ISD::ADD, DL, PtrType,
3239 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy,
3267 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
3276 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
3277 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
3286 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
3296 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) {
3297 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
3306 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
3325 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
3331 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
3333 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
3335 if (N0.getOpcode() == ISD::AND) {
3345 if (N1.getOpcode() == ISD::AND) {
3355 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
3357 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
3372 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
3383 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
3414 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
3417 Res = DAG.getNode(ISD::SRL, DL, VT, Res,
3435 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
3454 if (Opc == ISD::AND) {
3458 if (N0.getOpcode() != ISD::SRL)
3466 if (N0.getOpcode() != ISD::SHL)
3472 } else if (Opc == ISD::SHL) {
3480 } else { // Opc == ISD::SRL
3510 if (!TLI.isOperationLegal(ISD::BSWAP, VT))
3516 if (N0.getOpcode() != ISD::OR)
3522 if (N1.getOpcode() == ISD::OR &&
3544 if (N00.getOpcode() != ISD::OR)
3559 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
3565 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
3566 return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
3567 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
3568 return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
3569 return DAG.getNode(ISD::OR, DL, VT,
3570 DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
3571 DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
3580 (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) {
3588 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
3589 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
3594 if (isNullConstant(LR) && (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
3595 SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR),
3602 if (isAllOnesConstant(LR) && (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
3603 SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR),
3611 Op1 = ISD::getSetCCSwappedOperands(Op1);
3616 ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger);
3617 if (Result != ISD::SETCC_INVALID &&
3620 TLI.isOperationLegal(ISD::SETCC, LL.getValueType())))) {
3631 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
3647 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
3650 return DAG.getNode(ISD::AND, DL, VT, X,
3658 if (N0.getOpcode() == ISD::AND &&
3659 N1.getOpcode() == ISD::AND &&
3663 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
3665 return DAG.getNode(ISD::AND, SDLoc(LocReference), VT, N0.getOperand(0), X);
3682 if (ISD::isBuildVectorAllZeros(N0.getNode()))
3684 if (ISD::isBuildVectorAllZeros(N1.getNode()))
3688 if (ISD::isBuildVectorAllOnes(N0.getNode()))
3694 if (ISD::isBuildVectorAllOnes(N1.getNode()))
3709 ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode())) {
3762 return DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N), VT, N0C, N1C);
3766 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
3787 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1))
3791 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
3795 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT,
3798 ISD::AND, SDLoc(N), VT,
3799 DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1), COR);
3822 if (Op.getOpcode() == ISD::AND) {
3831 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
3881 if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
3891 if (Neg.getOpcode() != ISD::SUB)
3900 if (MaskLoBits && Pos.getOpcode() == ISD::AND)
3927 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
3977 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
3978 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
3999 if (RHSShift.getOpcode() == ISD::SHL) {
4019 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
4029 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4030 DAG.getNode(ISD::OR, DL, VT, LHSMask,
4035 Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
4036 DAG.getNode(ISD::OR, DL, VT, RHSMask,
4040 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
4054 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
4055 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
4056 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
4057 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
4058 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
4059 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
4060 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
4061 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
4067 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
4072 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
4090 if (ISD::isBuildVectorAllZeros(N0.getNode()))
4092 if (ISD::isBuildVectorAllZeros(N1.getNode()))
4097 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
4100 if (N0.getOpcode() == ISD::UNDEF)
4102 if (N1.getOpcode() == ISD::UNDEF)
4108 return DAG.FoldConstantArithmetic(ISD::XOR, SDLoc(N), VT, N0C, N1C);
4112 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
4117 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1))
4124 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
4132 case ISD::SETCC:
4134 case ISD::SELECT_CC:
4142 if (isOneConstant(N1) && N0.getOpcode() == ISD::ZERO_EXTEND &&
4147 V = DAG.getNode(ISD::XOR, DL, V.getValueType(), V,
4150 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
4155 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
4158 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
4159 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
4160 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
4167 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
4170 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
4171 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
4172 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
4178 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
4183 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
4186 if (N1C && N0.getOpcode() == ISD::XOR) {
4189 return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1),
4195 return DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0),
4222 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0.getOpcode() == ISD::SHL
4225 return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
4256 case ISD::OR:
4257 case ISD::XOR:
4260 case ISD::AND:
4263 case ISD::ADD:
4264 if (N->getOpcode() != ISD::SHL)
4280 if ((BinOpLHSVal->getOpcode() != ISD::SHL &&
4281 BinOpLHSVal->getOpcode() != ISD::SRA &&
4282 BinOpLHSVal->getOpcode() != ISD::SRL) ||
4292 if (N->getOpcode() == ISD::SRA) {
4317 assert(N->getOpcode() == ISD::TRUNCATE);
4318 assert(N->getOperand(0).getOpcode() == ISD::AND);
4332 return DAG.getNode(ISD::AND, DL, TruncVT,
4333 DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00),
4344 if (N->getOperand(1).getOpcode() == ISD::TRUNCATE &&
4345 N->getOperand(1).getOperand(0).getOpcode() == ISD::AND) {
4370 if (N0.getOpcode() == ISD::AND) {
4375 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
4378 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT,
4380 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
4391 return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C);
4402 if (N0.getOpcode() == ISD::UNDEF)
4409 if (N1.getOpcode() == ISD::TRUNCATE &&
4410 N1.getOperand(0).getOpcode() == ISD::AND) {
4413 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
4420 if (N1C && N0.getOpcode() == ISD::SHL) {
4427 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
4437 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
4438 N0.getOpcode() == ISD::ANY_EXTEND ||
4439 N0.getOpcode() == ISD::SIGN_EXTEND) &&
4440 N0.getOperand(0).getOpcode() == ISD::SHL) {
4451 return DAG.getNode(ISD::SHL, DL, VT,
4462 if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
4463 N0.getOperand(0).getOpcode() == ISD::SRL) {
4473 SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(),
4477 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
4485 if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) &&
4492 return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
4503 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
4513 Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0),
4518 Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
4522 return DAG.getNode(ISD::AND, DL, VT, Shift,
4528 if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) {
4535 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0),
4543 if (N1C && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() &&
4546 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
4547 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1);
4548 return DAG.getNode(ISD::ADD, SDLoc(N), VT, Shl0, Shl1);
4552 if (N1C && N0.getOpcode() == ISD::MUL && N0.getNode()->hasOneUse()) {
4555 DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, N0C1, N1C))
4556 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N0.getOperand(0), Folded);
4585 return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C);
4600 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
4607 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
4608 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
4613 if (N1C && N0.getOpcode() == ISD::SRA) {
4619 return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
4629 if (N0.getOpcode() == ISD::SHL && N1C) {
4648 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
4649 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
4655 SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
4657 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
4659 return DAG.getNode(ISD::SIGN_EXTEND, DL,
4666 if (N1.getOpcode() == ISD::TRUNCATE &&
4667 N1.getOperand(0).getOpcode() == ISD::AND) {
4670 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
4675 if (N0.getOpcode() == ISD::TRUNCATE &&
4676 (N0.getOperand(0).getOpcode() == ISD::SRL ||
4677 N0.getOperand(0).getOpcode() == ISD::SRA) &&
4691 SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT,
4693 return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
4705 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
4732 return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C);
4748 if (N1C && N0.getOpcode() == ISD::SRL) {
4755 return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0),
4761 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
4762 N0.getOperand(0).getOpcode() == ISD::SRL &&
4775 return DAG.getNode(ISD::TRUNCATE, DL, VT,
4776 DAG.getNode(ISD::SRL, DL, InnerShiftVT,
4784 if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1) {
4789 return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0),
4795 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4802 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
4805 SDValue SmallShift = DAG.getNode(ISD::SRL, DL0, SmallVT,
4812 return DAG.getNode(ISD::AND, DL, VT,
4813 DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
4821 if (N0.getOpcode() == ISD::SRA)
4822 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
4826 if (N1C && N0.getOpcode() == ISD::CTLZ &&
4851 Op = DAG.getNode(ISD::SRL, DL, VT, Op,
4858 return DAG.getNode(ISD::XOR, DL, VT,
4864 if (N1.getOpcode() == ISD::TRUNCATE &&
4865 N1.getOperand(0).getOpcode() == ISD::AND) {
4867 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
4902 if (Use->getOpcode() == ISD::BRCOND)
4904 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
4907 if (Use->getOpcode() == ISD::BRCOND)
4921 return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N0);
4923 if (N0.getOpcode() == ISD::BSWAP)
4934 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
4944 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
4954 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
4964 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
4974 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
4982 ISD::CondCode CC, const TargetLowering &TLI,
4988 case ISD::SETOLT:
4989 case ISD::SETOLE:
4990 case ISD::SETLT:
4991 case ISD::SETLE:
4992 case ISD::SETULT:
4993 case ISD::SETULE: {
4994 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
4999 case ISD::SETOGT:
5000 case ISD::SETOGE:
5001 case ISD::SETGT:
5002 case ISD::SETGE:
5003 case ISD::SETUGT:
5004 case ISD::SETUGE: {
5005 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
5032 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2);
5052 return DAG.getNode(ISD::XOR, DL, VT0,
5056 XORNode = DAG.getNode(ISD::XOR, DL0, VT0,
5060 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, XORNode);
5061 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, XORNode);
5067 return DAG.getNode(ISD::AND, SDLoc(N), VT, NOTNode, N2);
5073 return DAG.getNode(ISD::OR, SDLoc(N), VT, NOTNode, N1);
5077 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1);
5081 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2);
5085 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1);
5104 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
5107 SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N),
5110 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0,
5114 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
5117 SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N),
5120 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, N1,
5125 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
5132 SDValue And = DAG.getNode(ISD::AND, SDLoc(N), N0.getValueType(),
5134 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), And,
5139 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined,
5144 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
5151 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N), N0.getValueType(),
5153 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Or,
5158 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined,
5165 if (N0.getOpcode() == ISD::SETCC) {
5179 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
5188 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) ||
5189 TLI.isOperationLegal(ISD::SELECT_CC, VT))
5190 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT,
5225 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
5226 RHS.getOpcode() == ISD::CONCAT_VECTORS &&
5227 Cond.getOpcode() == ISD::BUILD_VECTOR);
5241 if (Cond->getOperand(i)->getOpcode() == ISD::UNDEF)
5253 if (Cond->getOperand(i)->getOpcode() == ISD::UNDEF)
5266 ISD::CONCAT_VECTORS, dl, VT,
5285 if (Mask.getOpcode() != ISD::SETCC)
5329 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
5346 if (Mask.getOpcode() == ISD::SETCC) {
5386 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
5401 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
5420 if (Mask.getOpcode() != ISD::SETCC)
5470 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
5477 SDValue GatherRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
5497 if (Mask.getOpcode() == ISD::SETCC) {
5535 ISD::NON_EXTLOAD);
5538 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
5547 ISD::NON_EXTLOAD);
5554 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
5561 SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
5580 if (N0.getOpcode() == ISD::SETCC) {
5582 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
5584 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
5586 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
5587 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
5588 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
5589 isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode());
5590 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
5591 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
5592 isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
5597 ISD::SRA, DL, VT, LHS,
5599 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
5602 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
5613 if (N0.getOpcode() == ISD::SETCC) {
5634 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
5638 if (ISD::isBuildVectorAllOnes(N0.getNode()))
5641 if (ISD::isBuildVectorAllZeros(N0.getNode()))
5647 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
5648 N2.getOpcode() == ISD::CONCAT_VECTORS &&
5649 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
5663 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
5680 } else if (SCC->getOpcode() == ISD::UNDEF) {
5684 } else if (SCC.getOpcode() == ISD::SETCC) {
5686 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(),
5713 if (Carry.getOpcode() == ISD::CARRY_FALSE)
5714 return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
5732 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
5733 Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
5748 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())))
5760 if (Op->getOpcode() == ISD::UNDEF) {
5769 if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
5775 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts).getNode();
5797 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
5798 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
5799 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
5820 if (User->getOpcode() == ISD::CopyToReg)
5829 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
5844 ISD::NodeType ExtType) {
5859 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
5869 assert((N->getOpcode() == ISD::SIGN_EXTEND ||
5870 N->getOpcode() == ISD::ZERO_EXTEND) &&
5890 if (N0->getOpcode() != ISD::LOAD)
5895 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
5904 ISD::LoadExtType ExtType =
5905 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
5937 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
5944 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
5945 SDValue NewValue = DAG.getNode(ISD
5952 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
5955 (ISD::NodeType)N->getOpcode());
5969 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
5970 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT,
5973 if (N0.getOpcode() == ISD::TRUNCATE) {
6003 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, Op);
6008 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op);
6012 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
6015 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
6017 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
6018 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, Op,
6026 if (ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
6029 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, N0.getValueType()))) {
6033 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI);
6038 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
6043 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
6047 ISD::SIGN_EXTEND);
6059 if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
6060 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
6064 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT)) {
6065 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
6071 DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
6080 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
6081 N0.getOpcode() == ISD::XOR) &&
6083 N0.getOperand(1).getOpcode() == ISD::Constant &&
6084 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, N0.getValueType()) &&
6087 if (LN0->getExtensionType() != ISD::ZEXTLOAD && LN0->isUnindexed()) {
6091 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND,
6094 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN0), VT,
6103 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
6109 ISD::SIGN_EXTEND);
6115 if (N0.getOpcode() == ISD::SETCC) {
6163 TLI.isOperationLegal(ISD::SETCC, N0.getOperand(0).getValueType())) {
6165 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
6175 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
6177 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0);
6189 if (N->getOpcode() == ISD::TRUNCATE) {
6195 if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 ||
6196 cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE)
6228 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
6229 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT,
6248 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Op);
6250 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op);
6258 if (N0.getOpcode() == ISD::TRUNCATE) {
6271 if (N0.getOpcode() == ISD::TRUNCATE) {
6290 if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
6291 TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) {
6299 if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
6302 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Op);
6305 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op);
6314 if (N0.getOpcode() == ISD::AND &&
6315 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
6316 N0.getOperand(1).getOpcode() == ISD::Constant &&
6322 X = DAG.getNode(ISD::ANY_EXTEND, SDLoc(X), VT, X);
6324 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
6329 return DAG.getNode(ISD::AND, DL, VT,
6336 if (ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
6339 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, N0.getValueType()))) {
6343 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI);
6348 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT,
6353 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
6358 ISD::ZERO_EXTEND);
6372 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR ||
6373 N0.getOpcode() == ISD::XOR) &&
6375 N0.getOperand(1).getOpcode() == ISD::Constant &&
6376 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, N0.getValueType()) &&
6379 if (LN0->getExtensionType() != ISD::SEXTLOAD && LN0->isUnindexed()) {
6383 if (N0.getOpcode() == ISD::AND) {
6394 ISD::ZERO_EXTEND, SetCCs, TLI);
6397 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), VT,
6406 SDValue Trunc = DAG.getNode(ISD::TRUNCATE,
6412 ISD::ZERO_EXTEND);
6420 if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) &&
6421 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
6425 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT)) {
6426 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT,
6432 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(),
6439 if (N0.getOpcode() == ISD::SETCC) {
6458 return DAG.getNode(ISD::AND, DL, VT,
6462 DAG.getNode(ISD::BUILD_VECTOR, DL, VT,
6478 return DAG.getNode(ISD::AND, DL, VT,
6480 DAG.getNode(ISD::BUILD_VECTOR, DL, VT, OneOps));
6493 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
6495 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
6499 if (N0.getOpcode() == ISD::SHL) {
6513 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
6516 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)),
6534 if (N0.getOpcode() == ISD::ANY_EXTEND ||
6535 N0.getOpcode() == ISD::ZERO_EXTEND ||
6536 N0.getOpcode() == ISD::SIGN_EXTEND)
6541 if (N0.getOpcode() == ISD::TRUNCATE) {
6554 if (N0.getOpcode() == ISD::TRUNCATE) {
6559 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, TruncOp);
6560 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, TruncOp);
6565 if (N0.getOpcode() == ISD::AND &&
6566 N0.getOperand(0).getOpcode() == ISD::TRUNCATE &&
6567 N0.getOperand(1).getOpcode() == ISD::Constant &&
6572 X = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, X);
6574 X = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X);
6579 return DAG.getNode(ISD::AND, DL, VT,
6587 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() &&
6588 ISD::isUNINDEXEDLoad(N0.getNode()) &&
6589 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) {
6593 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
6596 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
6601 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
6605 ISD::ANY_EXTEND);
6613 if (N0.getOpcode() == ISD::LOAD &&
6614 !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
6617 ISD::LoadExtType ExtType = LN0->getExtensionType();
6625 DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
6632 if (N0.getOpcode() == ISD::SETCC) {
6681 case ISD::Constant: {
6690 case ISD::OR:
6691 case ISD::XOR:
6698 case ISD::SRL:
6710 return DAG.getNode(ISD::SRL, SDLoc(V), V.getValueType(),
6725 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
6736 if (Opc == ISD::SIGN_EXTEND_INREG) {
6737 ExtType = ISD::SEXTLOAD;
6739 } else if (Opc == ISD::SRL) {
6741 ExtType = ISD::ZEXTLOAD;
6759 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
6776 if (cast<LoadSDNode>(N0)->getExtensionType() == ISD::SEXTLOAD)
6790 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
6823 if (LN0->getExtensionType() != ISD::NON_EXTLOAD &&
6847 SDValue NewPtr = DAG.getNode(ISD::ADD, DL,
6853 if (ExtType == ISD::NON_EXTLOAD)
6882 Result = DAG.getNode(ISD::SHL, DL, VT,
6903 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1);
6910 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
6912 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
6918 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
6921 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
6922 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
6942 if (N0.getOpcode() == ISD::SRL) {
6949 return DAG.getNode(ISD::SRA, SDLoc(N), VT,
6955 if (ISD::isEXTLoad(N0.getNode()) &&
6956 ISD::isUNINDEXEDLoad(N0.getNode()) &&
6959 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
6961 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
6971 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
6975 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) {
6977 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT,
6987 if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) {
6991 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
7002 if (N0.getOpcode() == ISD::UNDEF)
7022 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0);
7024 if (N0.getOpcode() == ISD::TRUNCATE)
7025 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
7027 if (N0.getOpcode() == ISD::ZERO_EXTEND ||
7028 N0.getOpcode() == ISD::SIGN_EXTEND ||
7029 N0.getOpcode() == ISD::ANY_EXTEND) {
7036 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0));
7052 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7071 SDValue V = DAG.getNode(ISD::BITCAST, SDLoc(N),
7075 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
7082 if (N0.getOpcode() == ISD::SELECT) {
7084 if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) &&
7088 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
7089 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
7090 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1);
7099 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
7100 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
7121 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Opnds);
7135 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter);
7145 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
7160 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
7168 if (X.getOpcode() != ISD::UNDEF) {
7192 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
7196 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds);
7210 if (Elt.getOpcode() != ISD::MERGE_VALUES)
7218 assert(N->getOpcode() == ISD::BUILD_PAIR);
7222 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() ||
7227 if (ISD::isNON_EXTLoad(LD2) &&
7239 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)))
7263 N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() &&
7281 TLI.isOperationLegal(ISD::ConstantFP, VT)) ||
7283 TLI.isOperationLegal(ISD::Constant, VT)))
7284 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, N0);
7288 if (N0.getOpcode() == ISD::BITCAST)
7289 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT,
7294 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
7300 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
7329 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) ||
7330 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) &&
7333 SDValue NewConv = DAG.getNode(ISD::BITCAST, SDLoc(N0), VT,
7343 if (N0.getOpcode() == ISD::FNEG) {
7347 assert(N0.getOpcode() == ISD::FABS);
7349 DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv,
7353 FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit);
7357 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
7359 return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits);
7362 if (N0.getOpcode() == ISD::FNEG)
7363 return DAG.getNode(ISD::XOR, DL, VT,
7365 assert(N0.getOpcode() == ISD::FABS);
7366 return DAG.getNode(ISD::AND, DL, VT,
7381 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() &&
7387 SDValue X = DAG.getNode(ISD::BITCAST, SDLoc(N0),
7394 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X);
7400 X = DAG.getNode(ISD::SRL, DL,
7405 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
7411 SDValue Cst = DAG.getNode(ISD::BITCAST, SDLoc(N0.getOperand(0)), VT,
7414 SDValue X = DAG.getNode(ISD::BITCAST, SDLoc(N0.getOperand(1)), VT,
7417 SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
7420 ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult,
7425 DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64,
7429 DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
7431 return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits);
7434 X = DAG.getNode(ISD::AND, SDLoc(X), VT,
7438 SDValue Cst = DAG.getNode(ISD::BITCAST, SDLoc(N0),
7440 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
7444 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
7449 if (N0.getOpcode() == ISD::BUILD_PAIR)
7458 N0->getOpcode() == ISD::VECTOR_SHUFFLE &&
7466 if (Op.getOpcode() == ISD::BITCAST &&
7469 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
7470 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
7471 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
7526 if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR)
7527 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BV), VT,
7528 DAG.getNode(ISD::BITCAST, SDLoc(BV),
7536 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op);
7537 Ops.push_back(DAG.getNode(ISD::BITCAST, SDLoc(BV),
7541 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, Ops);
7583 if (Op.getOpcode() == ISD::UNDEF) continue;
7597 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
7608 if (Op.getOpcode() == ISD::UNDEF) {
7627 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
7642 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT));
7647 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT));
7654 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
7660 if (Aggressive && N0.getOpcode() == ISD::FMUL &&
7661 N1.getOpcode() == ISD::FMUL) {
7667 if (N0.getOpcode() == ISD::FMUL &&
7675 if (N1.getOpcode() == ISD::FMUL &&
7684 if (N0.getOpcode() == ISD::FP_EXTEND) {
7686 if (N00.getOpcode() == ISD::FMUL)
7688 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7690 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7696 if (N1.getOpcode() == ISD::FP_EXTEND) {
7698 if (N10.getOpcode() == ISD::FMUL)
7700 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7702 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7711 N0.getOperand(2).getOpcode() == ISD::FMUL) {
7722 N1.getOperand(2).getOpcode() == ISD::FMUL) {
7738 DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
7739 DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
7744 if (N02.getOpcode() == ISD::FP_EXTEND) {
7746 if (N020.getOpcode() == ISD::FMUL)
7761 DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
7762 DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
7764 DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
7765 DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
7768 if (N0.getOpcode() == ISD::FP_EXTEND) {
7772 if (N002.getOpcode() == ISD::FMUL)
7783 if (N12.getOpcode() == ISD::FP_EXTEND) {
7785 if (N120.getOpcode() == ISD::FMUL)
7797 if (N1.getOpcode() == ISD::FP_EXTEND) {
7801 if (N102.getOpcode() == ISD::FMUL)
7825 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT));
7830 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT));
7837 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
7842 if (N0.getOpcode() == ISD::FMUL &&
7846 DAG.getNode(ISD::FNEG, SL, VT, N1));
7851 if (N1.getOpcode() == ISD::FMUL &&
7854 DAG.getNode(ISD::FNEG, SL, VT,
7859 if (N0.getOpcode() == ISD::FNEG &&
7860 N0.getOperand(0).getOpcode() == ISD::FMUL &&
7865 DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
7866 DAG.getNode(ISD::FNEG, SL, VT, N1));
7873 if (N0.getOpcode() == ISD::FP_EXTEND) {
7875 if (N00.getOpcode() == ISD::FMUL)
7877 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7879 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7881 DAG.getNode(ISD::FNEG, SL, VT, N1));
7887 if (N1.getOpcode() == ISD::FP_EXTEND) {
7889 if (N10.getOpcode() == ISD::FMUL)
7891 DAG.getNode(ISD::FNEG, SL, VT,
7892 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7894 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7905 if (N0.getOpcode() == ISD::FP_EXTEND) {
7907 if (N00.getOpcode() == ISD::FNEG) {
7909 if (N000.getOpcode() == ISD::FMUL) {
7910 return DAG.getNode(ISD::FNEG, SL, VT,
7912 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7914 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7927 if (N0.getOpcode() == ISD::FNEG) {
7929 if (N00.getOpcode() == ISD::FP_EXTEND) {
7931 if (N000.getOpcode() == ISD::FMUL) {
7932 return DAG.getNode(ISD::FNEG, SL, VT,
7934 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7936 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7950 N0.getOperand(2).getOpcode() == ISD::FMUL) {
7956 DAG.getNode(ISD::FNEG, SL, VT,
7963 N1.getOperand(2).getOpcode() == ISD::FMUL) {
7967 DAG.getNode(ISD::FNEG, SL, VT,
7971 DAG.getNode(ISD::FNEG, SL, VT, N20),
7981 if (N02.getOpcode() == ISD::FP_EXTEND) {
7983 if (N020.getOpcode() == ISD::FMUL)
7987 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7989 DAG.getNode(ISD::FP_EXTEND, SL, VT,
7991 DAG.getNode(ISD::FNEG, SL, VT,
8002 if (N0.getOpcode() == ISD::FP_EXTEND) {
8006 if (N002.getOpcode() == ISD::FMUL)
8008 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8010 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8013 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8015 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8017 DAG.getNode(ISD::FNEG, SL, VT,
8025 N1.getOperand(2).getOpcode() == ISD::FP_EXTEND) {
8027 if (N120.getOpcode() == ISD::FMUL) {
8031 DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)),
8034 DAG.getNode(ISD::FNEG, SL, VT,
8035 DAG.getNode(ISD::FP_EXTEND, SL,
8037 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8049 if (N1.getOpcode() == ISD::FP_EXTEND &&
8054 if (N102.getOpcode() == ISD::FMUL) {
8058 DAG.getNode(ISD::FNEG, SL, VT,
8059 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8061 DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
8063 DAG.getNode(ISD::FNEG, SL, VT,
8064 DAG.getNode(ISD::FP_EXTEND, SL,
8066 DAG.getNode(ISD::FP_EXTEND, SL, VT,
8084 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation");
8091 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT));
8096 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT));
8103 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
8109 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) {
8115 DAG.getNode(ISD::FNEG, SL, VT, Y));
8130 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) {
8134 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
8138 DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
8139 DAG.getNode(ISD::FNEG, SL, VT, Y));
8144 DAG.getNode(ISD::FNEG, SL, VT, Y));
8176 return DAG.getNode(ISD::FADD, DL, VT, N0, N1, Flags);
8180 return DAG.getNode(ISD::FADD, DL, VT, N1, N0, Flags);
8183 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
8185 return DAG.getNode(ISD::FSUB, DL, VT, N0,
8189 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
8191 return DAG.getNode(ISD::FSUB, DL, VT, N1,
8206 if (N1CFP && N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() &&
8208 return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0),
8209 DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1,
8214 if (AllowNewConst && N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1)
8218 if (AllowNewConst && N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0)
8224 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) {
8225 if (N0.getOpcode() == ISD::FMUL) {
8231 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
8233 return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP, Flags);
8237 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD &&
8240 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
8242 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP, Flags);
8246 if (N1.getOpcode() == ISD::FMUL) {
8252 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
8254 return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP, Flags);
8258 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD &&
8261 SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
8263 return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP, Flags);
8267 if (N0.getOpcode() == ISD::FADD && AllowNewConst) {
8272 return DAG.getNode(ISD::FMUL, DL, VT,
8277 if (N1.getOpcode() == ISD::FADD && AllowNewConst) {
8282 return DAG.getNode(ISD::FMUL, DL, VT,
8289 N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD &&
8293 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0),
8325 return DAG.getNode(ISD::FSUB, dl, VT, N0, N1, Flags);
8329 return DAG.getNode(ISD::FADD, dl, VT, N0,
8342 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
8343 return DAG.getNode(ISD::FNEG, dl, VT, N1);
8352 if (N1.getOpcode() == ISD::FADD) {
8392 return DAG.getNode(ISD::FMUL, DL, VT, N0, N1, Flags);
8397 return DAG.getNode(ISD::FMUL, DL, VT, N1, N0, Flags);
8409 if (N0.getOpcode() == ISD::FMUL) {
8426 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1, Flags);
8427 return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts, Flags);
8436 if (N0.getOpcode() == ISD::FADD &&
8440 SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1, Flags);
8441 return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts, Flags);
8447 return DAG.getNode(ISD::FADD, DL, VT, N0, N0, Flags);
8451 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
8452 return DAG.getNode(ISD::FNEG, DL, VT, N0);
8460 return DAG.getNode(ISD::FMUL, DL, VT,
8490 return DAG.getNode(ISD::FMA, dl, VT, N0, N1, N2);
8501 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2);
8503 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2);
8508 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2);
8517 if (N2.getOpcode() == ISD::FMUL && N0 == N2.getOperand(0) &&
8520 return DAG.getNode(ISD::FMUL, dl, VT, N0,
8521 DAG.getNode(ISD::FADD, dl, VT, N1, N2.getOperand(1),
8526 if (N0.getOpcode() == ISD::FMUL &&
8529 return DAG.getNode(ISD::FMA, dl, VT,
8531 DAG.getNode(ISD::FMUL, dl, VT, N1, N0.getOperand(1),
8542 return DAG.getNode(ISD::FADD, dl, VT, N0, N2);
8545 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) {
8546 SDValue RHSNeg = DAG.getNode(ISD::FNEG, dl, VT, N0);
8549 return DAG.getNode(ISD::FADD, dl, VT, N2, RHSNeg);
8556 return DAG.getNode(ISD::FMUL, dl, VT, N0,
8557 DAG.getNode(ISD::FADD, dl, VT,
8563 if (N1CFP && N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) {
8564 return DAG.getNode(ISD::FMUL, dl, VT, N0,
8565 DAG.getNode(ISD::FADD, dl, VT,
8604 if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1) {
8620 SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags);
8626 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
8655 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1, Flags);
8670 // TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT) ||
8671 TLI.isOperationLegal(llvm::ISD::ConstantFP, VT) ||
8673 return DAG.getNode(ISD::FMUL, DL, VT, N0,
8679 if (N1.getOpcode() == ISD::FSQRT) {
8681 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
8683 } else if (N1.getOpcode() == ISD::FP_EXTEND &&
8684 N1.getOperand(0).getOpcode() == ISD::FSQRT) {
8687 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV);
8689 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
8691 } else if (N1.getOpcode() == ISD::FP_ROUND &&
8692 N1.getOperand(0).getOpcode() == ISD::FSQRT) {
8695 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1));
8697 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
8699 } else if (N1.getOpcode() == ISD::FMUL) {
8704 if (N1.getOperand(0).getOpcode() == ISD::FSQRT) {
8707 } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) {
8715 RV = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, RV, OtherOp, Flags);
8717 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
8725 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV, Flags);
8735 return DAG.getNode(ISD::FDIV, SDLoc(N), VT,
8757 return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1,
8779 RV = DAG.getNode(ISD::FMUL, DL, VT, N->getOperand(0), RV, &Flags);
8786 SDValue ZeroCmp = DAG.getSetCC(DL, CCVT, N->getOperand(0), Zero, ISD::SETEQ);
8790 return DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL, VT,
8805 return (N1.getOpcode() == ISD::FP_EXTEND ||
8806 N1.getOpcode() == ISD::FP_ROUND) &&
8818 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1);
8825 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT))
8826 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
8828 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
8829 return DAG.getNode(ISD::FNEG, SDLoc(N), VT,
8830 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0));
8837 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG ||
8838 N0.getOpcode() == ISD::FCOPYSIGN)
8839 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT,
8843 if (N1.getOpcode() == ISD::FABS)
8844 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
8847 if (N1.getOpcode() == ISD::FCOPYSIGN)
8848 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT,
8854 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT,
8869 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
8870 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0);
8874 if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) &&
8875 TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) {
8878 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0);
8882 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT) || !LegalOperations) {
8884 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 &&
8887 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
8893 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
8898 if (N0.getOpcode() == ISD::ZERO_EXTEND &&
8899 N0.getOperand(0).getOpcode() == ISD::SETCC &&!VT.isVector() &&
8901 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
8907 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
8923 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
8924 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0);
8928 if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) &&
8929 TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) {
8932 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0);
8936 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT) || !LegalOperations) {
8939 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() &&
8941 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) {
8947 return DAG.getNode(ISD::SELECT_CC, DL, VT, Ops);
8959 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP)
8964 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP;
8965 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT;
8985 unsigned ExtOp = IsInputSigned && IsOutputSigned ? ISD::SIGN_EXTEND
8986 : ISD::ZERO_EXTEND;
8990 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src);
8993 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Src);
9004 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0);
9015 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0);
9028 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1);
9031 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType())
9035 if (N0.getOpcode() == ISD::FP_ROUND) {
9056 return DAG.getNode(ISD::FP_ROUND, DL, VT, N0.getOperand(0),
9062 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) {
9063 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT,
9066 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT,
9083 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Round);
9095 N->use_begin()->getOpcode() == ISD::FP_ROUND)
9100 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0);
9103 if (N0.getOpcode() == ISD::FP16_TO_FP &&
9104 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal)
9105 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0));
9109 if (N0.getOpcode() == ISD::FP_ROUND
9114 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT,
9116 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In);
9120 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
9121 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) {
9123 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT,
9129 DAG.getNode(ISD::FP_ROUND, SDLoc(N0),
9145 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0);
9156 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0);
9167 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0);
9179 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0);
9188 N0.getOpcode() == ISD::BITCAST &&
9204 Int = DAG.getNode(ISD::XOR, DL0, IntVT, Int,
9207 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Int);
9212 if (N0.getOpcode() == ISD::FMUL &&
9220 TLI.isOperationLegal(ISD::ConstantFP, VT)))
9221 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0.getOperand(0),
9222 DAG.getNode(ISD::FNEG, SDLoc(N), VT,
9247 return DAG.getNode(ISD::FMINNUM, SDLoc(N), VT, N1, N0);
9268 return DAG.getNode(ISD::FMAXNUM, SDLoc(N), VT, N1, N0);
9279 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0);
9282 if (N0.getOpcode() == ISD::FABS)
9287 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN)
9288 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0));
9293 N0.getOpcode() == ISD::BITCAST &&
9309 Int = DAG.getNode(ISD::AND, DL, IntVT, Int,
9312 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Int);
9332 if (N1.getOpcode() == ISD::SETCC &&
9333 TLI.isOperationLegalOrCustom(ISD::BR_CC,
9335 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other,
9340 if ((N1.hasOneUse() && N1.getOpcode() == ISD::SRL) ||
9341 ((N1.getOpcode() == ISD::TRUNCATE && N1.hasOneUse()) &&
9343 N1.getOperand(0).getOpcode() == ISD::SRL))) {
9345 if (N1.getOpcode() == ISD::TRUNCATE) {
9371 if (Op0.getOpcode() == ISD::AND &&
9372 Op1.getOpcode() == ISD::Constant) {
9375 if (AndOp1.getOpcode() == ISD::Constant) {
9385 ISD::SETNE);
9387 SDValue NewBRCond = DAG.getNode(ISD::BRCOND, DL,
9411 if (N1.hasOneUse() && N1.getOpcode() == ISD::XOR) {
9427 return DAG.getNode(ISD::BRCOND, SDLoc(N),
9437 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) {
9440 Op0.getOpcode() == ISD::XOR) {
9451 Equal ? ISD::SETEQ : ISD::SETNE);
9456 return DAG.getNode(ISD::BRCOND, SDLoc(N),
9483 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC)
9484 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other,
9514 if (N->getOpcode() == ISD::ADD) {
9522 } else if (N->getOpcode() == ISD::SUB) {
9553 if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) &&
9554 !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT))
9561 if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) &&
9562 !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT))
9572 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) ||
9579 ISD::MemIndexedMode AM = ISD::UNINDEXED;
9633 if (Use.getUser()->getOpcode() != ISD::ADD &&
9634 Use.getUser()->getOpcode() != ISD::SUB) {
9732 X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1;
9733 Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1;
9734 X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1;
9735 Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1;
9737 unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD;
9779 if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) &&
9780 !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT))
9787 if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) &&
9788 !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT))
9801 (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB))
9806 ISD::MemIndexedMode AM = ISD::UNINDEXED;
9830 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){
9886 ISD::MemIndexedMode AM = LD->getAddressingMode();
9887 assert(AM != ISD::UNINDEXED);
9894 assert((Inc.getOpcode() != ISD::TargetConstant ||
9897 if (Inc.getOpcode() == ISD::TargetConstant) {
9904 (AM == ISD::PRE_INC || AM == ISD::POST_INC ? ISD::ADD : ISD::SUB);
9947 bool HasOTCInc = LD->getOperand(2).getOpcode() == ISD::TargetConstant &&
9980 if (ISD::isNormalLoad(N) && !LD->isVolatile()) {
9981 if (ISD::isNON_TRUNCStore(Chain.getNode())) {
10022 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
10033 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N),
10217 if (Origin->getOffset().getOpcode() != ISD::UNDEF)
10228 if (!TLI.isOperationLegal(ISD::LOAD, SliceType))
10242 if (!TLI.isOperationLegal(ISD::ADD, PtrType))
10248 !TLI.isOperationLegal(ISD::ZERO_EXTEND, TruncateType))
10292 BaseAddr = DAG->getNode(ISD::ADD, DL, ArithType, BaseAddr,
10309 DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst);
10321 if (Use->getOpcode() != ISD::BITCAST)
10329 if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT))
10349 if (!TLI.isOperationLegal(ISD::LOAD, ResVT))
10508 if (LD->isVolatile() || !ISD::isNormalLoad(LD) ||
10531 if (User->getOpcode() == ISD::SRL && User->hasOneUse() &&
10539 if (User->getOpcode() != ISD::TRUNCATE)
10585 if (SliceInst.getNode()->getOpcode() != ISD::LOAD)
10587 assert(SliceInst->getOpcode() == ISD::LOAD &&
10592 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other,
10606 if (V->getOpcode() != ISD::AND ||
10608 !ISD::isNormalLoad(V->getOperand(0).getNode()))
10619 else if (Chain->getOpcode() != ISD::TokenFactor)
10701 IVal = DAG.getNode(ISD::SRL, DL, IVal.getValueType(), IVal,
10718 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(),
10724 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal);
10757 if (Opc == ISD::OR) {
10773 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) ||
10774 Value.getOperand(1).getOpcode() != ISD::Constant)
10778 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
10790 if (Opc == ISD::AND)
10818 if (Opc == ISD::AND)
10831 SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LD),
10869 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) &&
10883 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) ||
10884 !TLI.isOperationLegal(ISD::STORE, IntVT) ||
10885 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) ||
10886 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT))
10955 if (Ptr->getOpcode() != ISD::ADD)
10971 if (Ptr->getOperand(1)->getOpcode() == ISD::MUL)
10979 if (IndexOffset->getOpcode() == ISD::SIGN_EXTEND) {
10985 if (IndexOffset->getOpcode() != ISD::ADD)
10996 if (Index->getOpcode() == ISD::SIGN_EXTEND) {
11035 if (Use->getOpcode() == ISD::MUL) { // We have another multiply use.
11071 if (OtherOp->getOpcode() == ISD::ADD &&
11095 return DAG.getNode(ISD::BUILD_VECTOR, SL, Ty, BuildVector);
11151 StoredVal = DAG.getNode(IsVec ? ISD::CONCAT_VECTORS : ISD::BUILD_VECTOR,
11187 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
11231 if (BasePtr.Base.getOpcode() == ISD::UNDEF)
11357 bool IsExtractVecSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
11358 StoredVal.getOpcode() == ISD::EXTRACT_SUBVECTOR);
11371 if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE)
11513 if (StoreValOpcode != ISD::EXTRACT_VECTOR_ELT &&
11514 StoreValOpcode != ISD::EXTRACT_SUBVECTOR)
11559 if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
11639 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
11640 TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy, StoreTy) &&
11641 TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy) &&
11704 DAG.getNode(ISD::TokenFactor, StoreDL, MVT::Other, MergeStoreChains);
11747 SDValue Token = DAG.getNode(ISD::TokenFactor, SL,
11759 if (Value.getOpcode() == ISD::TargetConstantFP)
11785 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
11797 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) {
11806 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) {
11825 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
11832 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
11848 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() &&
11856 TLI.isOperationLegalOrCustom(ISD::STORE, SVT)))
11864 if (Value.getOpcode() == ISD::UNDEF && ST->isUnindexed())
11964 if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE)
11984 } while (ST->getOpcode() != ISD::DELETED_NODE);
12010 if (InVal.getOpcode() == ISD::UNDEF)
12016 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
12031 if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse()
12037 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VT,
12040 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()),
12051 if (InVec.getOpcode() == ISD::BUILD_VECTOR && InVec.hasOneUse()) {
12054 } else if (InVec.getOpcode() == ISD::UNDEF) {
12068 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) :
12069 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal);
12074 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
12085 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT))
12103 ISD::MUL, DL, PtrType, Offset,
12107 NewPtr = DAG.getNode(ISD::ADD, DL, PtrType, NewPtr, Offset);
12120 ISD::LoadExtType ExtType = TLI.isLoadExtLegal(ISD::ZEXTLOAD, ResultVT,
12122 ? ISD::ZEXTLOAD
12123 : ISD::EXTLOAD;
12136 Load = DAG.getNode(ISD::TRUNCATE, SDLoc(EVE), ResultVT, Load);
12138 Load = DAG.getNode(ISD::BITCAST, SDLoc(EVE), ResultVT, Load);
12160 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) {
12177 InVec.getOpcode() == ISD::BUILD_VECTOR &&
12198 if (ConstEltNo && InVec.getOpcode() == ISD::VECTOR_SHUFFLE) {
12217 if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) {
12232 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), NVT, SVInVec,
12246 if (InVec.getOpcode() == ISD::BITCAST) {
12262 ISD::isNormalLoad(InVec.getNode()) &&
12283 if (ISD::isNormalLoad(InVec.getNode())) {
12285 } else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR &&
12287 ISD::isNormalLoad(InVec.getOperand(0).getNode())) {
12312 if (InVec.getOpcode() == ISD::BITCAST) {
12319 if (ISD::isNormalLoad(InVec.getNode())) {
12367 if (In.getOpcode() == ISD::UNDEF) continue;
12369 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND;
12370 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND;
12420 assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
12421 Cast.getOpcode() == ISD::ZERO_EXTEND ||
12422 Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode");
12424 if (Cast.getOpcode() == ISD::UNDEF)
12443 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, Ops);
12448 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
12458 unsigned Opcode = ISD::DELETED_NODE;
12465 if (Opc == ISD::UNDEF)
12469 if (Opcode == ISD::DELETED_NODE &&
12470 (Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP)) {
12493 assert((Opcode == ISD::UINT_TO_FP || Opcode == ISD::SINT_TO_FP)
12511 if (In.getOpcode() == ISD::UNDEF)
12516 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Opnds);
12528 if (ISD::allOperandsUndef(N))
12546 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT))
12554 if (Op.getOpcode() == ISD::UNDEF) continue;
12564 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
12592 if (Opcode == ISD::UNDEF) {
12598 if (Opcode != ISD::EXTRACT_VECTOR_ELT) {
12600 (Opcode == ISD::Constant || Opcode == ISD::ConstantFP) &&
12641 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, VecIn1,
12645 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, VecIn1, VecIn2);
12663 ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
12667 ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1,
12709 SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT);
12715 if (ISD::BITCAST == Op.getOpcode() &&
12718 else if (ISD::UNDEF == Op.getOpcode())
12739 ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT);
12744 if (Op.getOpcode() == ISD::UNDEF)
12747 Op = DAG.getNode(ISD::BITCAST, DL, SVT, Op);
12754 return DAG.getNode(ISD::BITCAST, DL, VT,
12755 DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, Ops));
12773 while (Op.getOpcode() == ISD::BITCAST)
12777 if (Op.getOpcode() == ISD::UNDEF) {
12782 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR)
12793 while (ExtVec.getOpcode() == ISD::BITCAST)
12797 if (ExtVec.getOpcode() == ISD::UNDEF) {
12821 if (SV0.getOpcode() == ISD::UNDEF || SV0 == ExtVec) {
12825 } else if (SV1.getOpcode() == ISD::UNDEF || SV1 == ExtVec) {
12848 if (ISD::allOperandsUndef(N))
12853 return Op.getOpcode() == ISD::UNDEF;
12859 if (In->getOpcode() == ISD::BITCAST &&
12866 if (Scalar->getOpcode() == ISD::TRUNCATE &&
12882 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NVT, Scalar);
12883 return DAG.getNode(ISD::BITCAST, dl, VT, Res);
12892 return ISD::UNDEF == Op.getOpcode() || ISD::BUILD_VECTOR == Op.getOpcode();
12906 if (ISD::BUILD_VECTOR == Op.getOpcode()) {
12918 if (ISD::UNDEF == Op.getOpcode())
12921 if (ISD::BUILD_VECTOR == Op.getOpcode()) {
12928 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i)));
12935 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Opnds);
12957 if (Op.getOpcode() == ISD::UNDEF)
12961 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR)
12999 if (V->getOpcode() == ISD::CONCAT_VECTORS) {
13016 if (V->getOpcode() == ISD::BITCAST)
13019 if (V->getOpcode() == ISD::INSERT_SUBVECTOR) {
13042 return DAG.getNode(ISD::BITCAST, dl, NVT, V->getOperand(1));
13043 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT,
13044 DAG.getNode(ISD::BITCAST, dl,
13062 case ISD::CONCAT_VECTORS: {
13084 V = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, NewOps);
13088 case ISD::INSERT_SUBVECTOR: {
13113 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
13157 if (NumElemsPerConcat * 2 == NumElts && N1.getOpcode() == ISD::UNDEF &&
13163 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1);
13200 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops);
13213 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
13231 if (N0.getOpcode() == ISD::UNDEF) {
13248 if (N1.getOpcode() == ISD::UNDEF) {
13271 if (V->getOpcode() == ISD::BITCAST) {
13278 if (V->getOpcode() == ISD::BUILD_VECTOR) {
13284 if (V->getOperand(i).getOpcode() != ISD::UNDEF) {
13305 SDValue NewBV = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N),
13311 NewBV = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, NewBV);
13322 if (N0.getOpcode() == ISD::CONCAT_VECTORS &&
13324 (N1.getOpcode() == ISD::UNDEF ||
13325 (N1.getOpcode() == ISD::CONCAT_VECTORS &&
13342 if (S.getOpcode() == ISD::BUILD_VECTOR && S.hasOneUse()) {
13344 } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR && S.hasOneUse()) {
13366 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Ops);
13373 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
13374 N1.getOpcode() == ISD::UNDEF && Level < AfterLegalizeVectorOps &&
13379 while (BC0.getOpcode() == ISD::BITCAST) {
13396 if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) {
13435 SV0 = DAG.getNode(ISD::BITCAST, SDLoc(N), ScaleVT, SV0);
13436 SV1 = DAG.getNode(ISD::BITCAST, SDLoc(N), ScaleVT, SV1);
13438 ISD::BITCAST, SDLoc(N), VT,
13449 if (N1.getOpcode() == ISD::VECTOR_SHUFFLE &&
13450 N0.getOpcode() != ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG &&
13460 bool IsSV1Undef = SV1.getOpcode() == ISD::UNDEF;
13473 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && N->isOnlyUserOf(N0.getNode()) &&
13513 if (CurrentVec.getOpcode() == ISD::UNDEF) {
13580 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
13608 if (N0.getOpcode() == ISD::CONCAT_VECTORS &&
13609 N0->getNumOperands() == 2 && N2.getOpcode() == ISD::Constant) {
13616 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
13622 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
13633 if (N0->getOpcode() == ISD::FP16_TO_FP)
13643 if (N0->getOpcode() == ISD::AND) {
13646 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), N->getValueType(0),
13669 if (N->getOpcode() != ISD::AND)
13672 if (RHS.getOpcode() == ISD::BITCAST)
13675 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
13693 if (Elt.getOpcode() == ISD::UNDEF) {
13772 LHS.getOperand(1).getOpcode() == ISD::UNDEF &&
13773 RHS.getOperand(1).getOpcode() == ISD::UNDEF) {
13794 assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!");
13805 if (SCC.getOpcode() == ISD::SELECT_CC) {
13806 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0),
13831 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) {
13834 ISD::CondCode CC;
13838 if (TheSelect->getOpcode() == ISD::SELECT_CC) {
13845 if (Cmp.getOpcode() == ISD::SETCC) {
13852 Sqrt.getOperand(0) == CmpLHS && (CC == ISD::SETOLT ||
13853 CC == ISD::SETULT || CC == ISD::SETLT)) {
13873 if (LHS.getOpcode() == ISD::LOAD) {
13889 LLD->getExtensionType() != ISD::EXTLOAD &&
13890 RLD->getExtensionType() != ISD::EXTLOAD) ||
13906 if (TheSelect->getOpcode() == ISD::SELECT) {
13929 Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect),
13943 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
13951 Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ?
13978 ISD::CondCode CC, bool NotExtCompare) {
14002 if ((CC == ISD::SETGE || CC == ISD::SETGT) &&
14003 N0 == N2 && N3.getOpcode() == ISD::FNEG &&
14005 return DAG.getNode(ISD::FABS, DL, VT, N0);
14008 if ((CC == ISD::SETLT || CC == ISD::SETLE) &&
14009 N0 == N3 && N2.getOpcode() == ISD::FNEG &&
14011 return DAG.getNode(ISD::FABS, DL, VT, N3);
14026 (TLI.getOperationAction(ISD::ConstantFP, N2.getValueType()) !=
14060 CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx,
14072 if (isNullConstant(N3) && CC == ISD::SETLT &&
14085 SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0),
14090 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
14094 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
14097 SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0),
14105 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift);
14109 return DAG.getNode(ISD::AND, DL, AType, Shift, N2);
14119 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND &&
14129 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt);
14136 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt);
14138 return DAG.getNode(ISD::AND, DL, VT, Shr, N3);
14155 TLI.isOperationLegal(ISD::SETCC, N0.getValueType())) {
14165 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2),
14169 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2),
14181 ISD::SHL, DL, N2.getValueType(), Temp,
14195 if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
14196 (N1C->isAllOnesValue() && CC == ISD::SETGT)) &&
14197 N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1))
14199 else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) ||
14200 (N1C->isOne() && CC == ISD::SETLT)) &&
14201 N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1))
14207 SDValue Shift = DAG.getNode(ISD::SRA, DL, XType,
14211 SDValue Add = DAG.getNode(ISD::ADD, DL,
14215 return DAG.getNode(ISD::XOR, DL, XType, Add, Shift);
14224 SDValue N1, ISD::CondCode Cond,
14231 /// Given an ISD::SDIV node expressing a divide by constant, return
14253 /// Given an ISD::SDIV node expressing a divide by constant power of 2, return a
14272 /// Given an ISD::UDIV node expressing a divide by constant, return a DAG
14318 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, Est, Flags);
14321 NewEst = DAG.getNode(ISD::FSUB, DL, VT, FPOne, NewEst, Flags);
14324 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags);
14327 Est = DAG.getNode(ISD::FADD, DL, VT, Est, NewEst, Flags);
14352 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg, Flags);
14355 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg, Flags);
14360 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags);
14363 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst, Flags);
14366 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst, Flags);
14369 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst, Flags);
14390 SDValue HalfEst = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf, Flags);
14393 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Est, Flags);
14396 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg, Flags);
14399 Est = DAG.getNode(ISD::FADD, DL, VT, Est, MinusThree, Flags);
14402 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, HalfEst, Flags);
14437 if (Base.getOpcode() == ISD::ADD) {
14601 case ISD::EntryToken:
14605 case ISD::LOAD:
14606 case ISD::STORE: {
14623 case ISD::TokenFactor:
14719 return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Aliases);
14732 if (BasePtr.Base.getOpcode() == ISD::UNDEF)
