Lines Matching full:x86
1 //===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===//
10 // This file defines a DAG pattern matching instruction selector for X86,
11 // converting from a legalized dag to a X86 dag.
15 #define DEBUG_TYPE "x86-isel"
16 #include "X86.h"
91 return RegNode->getReg() == X86::RIP;
140 /// ISel - X86 specific code to select X86 machine instructions for
145 /// X86-specific SelectionDAG.
164 return "X86 DAG->DAG Instruction Selection";
487 // late" legalization of these inline with the X86 isel pass.
558 Subtarget->is64Bit() ? X86::CALL64pcrel32 : X86::CALLpcrel32;
586 if (!X86::isOffsetSuitableForCodeModel(Val, M,
607 // gs:0 (or fs:0 on X86-64) contains its own address.
614 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
617 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
637 // Handle X86-64 rip-relative addresses. We check this before checking direct
640 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
682 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
687 // X86-32 always and X86-64 when in -mcmodel=small mode. In 64-bit
747 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
1314 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1316 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1425 // For X86-64, we should always use lea to materialize RIP relative
1459 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
1540 X86::LOCK_ADD8mi,
1541 X86::LOCK_ADD8mr,
1542 X86::LOCK_ADD16mi8,
1543 X86::LOCK_ADD16mi,
1544 X86::LOCK_ADD16mr,
1545 X86::LOCK_ADD32mi8,
1546 X86::LOCK_ADD32mi,
1547 X86::LOCK_ADD32mr,
1548 X86::LOCK_ADD64mi8,
1549 X86::LOCK_ADD64mi32,
1550 X86::LOCK_ADD64mr,
1553 X86::LOCK_SUB8mi,
1554 X86::LOCK_SUB8mr,
1555 X86::LOCK_SUB16mi8,
1556 X86::LOCK_SUB16mi,
1557 X86::LOCK_SUB16mr,
1558 X86::LOCK_SUB32mi8,
1559 X86::LOCK_SUB32mi,
1560 X86::LOCK_SUB32mr,
1561 X86::LOCK_SUB64mi8,
1562 X86::LOCK_SUB64mi32,
1563 X86::LOCK_SUB64mr,
1567 X86::LOCK_INC8m,
1570 X86::LOCK_INC16m,
1573 X86::LOCK_INC32m,
1576 X86::LOCK_INC64m,
1580 X86::LOCK_DEC8m,
1583 X86::LOCK_DEC16m,
1586 X86::LOCK_DEC32m,
1589 X86::LOCK_DEC64m,
1592 X86::LOCK_OR8mi,
1593 X86::LOCK_OR8mr,
1594 X86::LOCK_OR16mi8,
1595 X86::LOCK_OR16mi,
1596 X86::LOCK_OR16mr,
1597 X86::LOCK_OR32mi8,
1598 X86::LOCK_OR32mi,
1599 X86::LOCK_OR32mr,
1600 X86::LOCK_OR64mi8,
1601 X86::LOCK_OR64mi32,
1602 X86::LOCK_OR64mr,
1605 X86::LOCK_AND8mi,
1606 X86::LOCK_AND8mr,
1607 X86::LOCK_AND16mi8,
1608 X86::LOCK_AND16mi,
1609 X86::LOCK_AND16mr,
1610 X86::LOCK_AND32mi8,
1611 X86::LOCK_AND32mi,
1612 X86::LOCK_AND32mr,
1613 X86::LOCK_AND64mi8,
1614 X86::LOCK_AND64mi32,
1615 X86::LOCK_AND64mr,
1618 X86::LOCK_XOR8mi,
1619 X86::LOCK_XOR8mr,
1620 X86::LOCK_XOR16mi8,
1621 X86::LOCK_XOR16mi,
1622 X86::LOCK_XOR16mr,
1623 X86::LOCK_XOR32mi8,
1624 X86::LOCK_XOR32mi,
1625 X86::LOCK_XOR32mr,
1626 X86::LOCK_XOR64mi8,
1627 X86::LOCK_XOR64mi32,
1628 X86::LOCK_XOR64mr,
1667 if (Val.getOpcode() == ISD::SUB && X86::isZeroNode(Val.getOperand(0))) {
1676 X86::isZeroNode(Val.getOperand(0).getOperand(0))) {
1679 return CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, NVT,
1796 X86::EFLAGS)
1808 case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr:
1809 case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr:
1810 case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm:
1811 case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm:
1812 case X86::JA_4: case X86::JAE_4: case X86::JB_4: case X86::JBE_4:
1813 case X86::JE_4: case X86::JNE_4: case X86::JP_4: case X86::JNP_4:
1814 case X86::CMOVA16rr: case X86::CMOVA16rm:
1815 case X86::CMOVA32rr: case X86::CMOVA32rm:
1816 case X86::CMOVA64rr: case X86::CMOVA64rm:
1817 case X86::CMOVAE16rr: case X86::CMOVAE16rm:
1818 case X86::CMOVAE32rr: case X86::CMOVAE32rm:
1819 case X86::CMOVAE64rr: case X86::CMOVAE64rm:
1820 case X86::CMOVB16rr: case X86::CMOVB16rm:
1821 case X86::CMOVB32rr: case X86::CMOVB32rm:
1822 case X86::CMOVB64rr: case X86::CMOVB64rm:
1823 case X86::CMOVBE16rr: case X86::CMOVBE16rm:
1824 case X86::CMOVBE32rr: case X86::CMOVBE32rm:
1825 case X86::CMOVBE64rr: case X86::CMOVBE64rm:
1826 case X86::CMOVE16rr: case X86::CMOVE16rm:
1827 case X86::CMOVE32rr: case X86::CMOVE32rm:
1828 case X86::CMOVE64rr: case X86::CMOVE64rm:
1829 case X86::CMOVNE16rr: case X86::CMOVNE16rm:
1830 case X86::CMOVNE32rr: case X86::CMOVNE32rm:
1831 case X86::CMOVNE64rr: case X86::CMOVNE64rm:
1832 case X86::CMOVNP16rr: case X86::CMOVNP16rm:
1833 case X86::CMOVNP32rr: case X86::CMOVNP32rm:
1834 case X86::CMOVNP64rr: case X86::CMOVNP64rm:
1835 case X86::CMOVP16rr: case X86::CMOVP16rm:
1836 case X86::CMOVP32rr: case X86::CMOVP32rm:
1837 case X86::CMOVP64rr: case X86::CMOVP64rm:
1933 /// getFusedLdStOpcode - Get the appropriate X86 opcode for an in memory
1937 if (LdVT == MVT::i64) return X86::DEC64m;
1938 if (LdVT == MVT::i32) return X86::DEC32m;
1939 if (LdVT == MVT::i16) return X86::DEC16m;
1940 if (LdVT == MVT::i8) return X86::DEC8m;
1943 if (LdVT == MVT::i64) return X86::INC64m;
1944 if (LdVT == MVT::i32) return X86::INC32m;
1945 if (LdVT == MVT::i16) return X86::INC16m;
1946 if (LdVT == MVT::i8) return X86::INC8m;
2021 case Intrinsic::x86_avx2_gather_d_pd: Opc = X86::VGATHERDPDrm; break;
2022 case Intrinsic::x86_avx2_gather_d_pd_256: Opc = X86::VGATHERDPDYrm; break;
2023 case Intrinsic::x86_avx2_gather_q_pd: Opc = X86::VGATHERQPDrm; break;
2024 case Intrinsic::x86_avx2_gather_q_pd_256: Opc = X86::VGATHERQPDYrm; break;
2025 case Intrinsic::x86_avx2_gather_d_ps: Opc = X86::VGATHERDPSrm; break;
2026 case Intrinsic::x86_avx2_gather_d_ps_256: Opc = X86::VGATHERDPSYrm; break;
2027 case Intrinsic::x86_avx2_gather_q_ps: Opc = X86::VGATHERQPSrm; break;
2028 case Intrinsic::x86_avx2_gather_q_ps_256: Opc = X86::VGATHERQPSYrm; break;
2029 case Intrinsic::x86_avx2_gather_d_q: Opc = X86::VPGATHERDQrm; break;
2030 case Intrinsic::x86_avx2_gather_d_q_256: Opc = X86::VPGATHERDQYrm; break;
2031 case Intrinsic::x86_avx2_gather_q_q: Opc = X86::VPGATHERQQrm; break;
2032 case Intrinsic::x86_avx2_gather_q_q_256: Opc = X86::VPGATHERQQYrm; break;
2033 case Intrinsic::x86_avx2_gather_d_d: Opc = X86::VPGATHERDDrm; break;
2034 case Intrinsic::x86_avx2_gather_d_d_256: Opc = X86::VPGATHERDDYrm; break;
2035 case Intrinsic::x86_avx2_gather_q_d: Opc = X86::VPGATHERQDrm; break;
2036 case Intrinsic::x86_avx2_gather_q_d_256: Opc = X86::VPGATHERQDYrm; break;
2065 case X86ISD::ATOMOR64_DAG: Opc = X86::ATOMOR6432; break;
2066 case X86ISD::ATOMXOR64_DAG: Opc = X86::ATOMXOR6432; break;
2067 case X86ISD::ATOMADD64_DAG: Opc = X86::ATOMADD6432; break;
2068 case X86ISD::ATOMSUB64_DAG: Opc = X86::ATOMSUB6432; break;
2069 case X86ISD::ATOMNAND64_DAG: Opc = X86::ATOMNAND6432; break;
2070 case X86ISD::ATOMAND64_DAG: Opc = X86::ATOMAND6432; break;
2071 case X86ISD::ATOMMAX64_DAG: Opc = X86::ATOMMAX6432; break;
2072 case X86ISD::ATOMMIN64_DAG: Opc = X86::ATOMMIN6432; break;
2073 case X86ISD::ATOMUMAX64_DAG: Opc = X86::ATOMUMAX6432; break;
2074 case X86ISD::ATOMUMIN64_DAG: Opc = X86::ATOMUMIN6432; break;
2075 case X86ISD::ATOMSWAP64_DAG: Opc = X86::ATOMSWAP6432; break;
2141 ShlOp = X86::SHL32ri;
2145 case ISD::AND: Op = X86::AND32ri8; break;
2146 case ISD::OR: Op = X86::OR32ri8; break;
2147 case ISD::XOR: Op = X86::XOR32ri8; break;
2152 ShlOp = X86::SHL64ri;
2156 case ISD::AND: Op = CstVT==MVT::i8? X86::AND64ri8 : X86::AND64ri32; break;
2157 case ISD::OR: Op = CstVT==MVT::i8? X86::OR64ri8 : X86::OR64ri32; break;
2158 case ISD::XOR: Op = CstVT==MVT::i8? X86::XOR64ri8 : X86::XOR64ri32; break;
2176 case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r; break;
2177 case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r; break;
2178 case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break;
2179 case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break;
2205 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break;
2206 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break;
2207 case MVT::i32: Opc = hasBMI2 ? X86::MULX32rr : X86::MUL32r;
2208 MOpc = hasBMI2 ? X86::MULX32rm : X86::MUL32m; break;
2209 case MVT::i64: Opc = hasBMI2 ? X86::MULX64rr : X86::MUL64r;
2210 MOpc = hasBMI2 ? X86::MULX64rm : X86::MUL64m; break;
2215 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break;
2216 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break;
2217 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break;
2218 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break;
2225 X86::IMUL8r:
2226 case X86::MUL8r:
2227 SrcReg = LoReg = X86::AL; HiReg = X86::AH;
2229 case X86::IMUL16r:
2230 case X86::MUL16r:
2231 SrcReg = LoReg = X86::AX; HiReg = X86::DX;
2233 case X86::IMUL32r:
2234 case X86::MUL32r:
2235 SrcReg = LoReg = X86::EAX; HiReg = X86::EDX;
2237 case X86::IMUL64r:
2238 case X86::MUL64r:
2239 SrcReg = LoReg = X86::RAX; HiReg = X86::RDX;
2241 case X86::MULX32rr:
2242 SrcReg = X86::EDX; LoReg = HiReg = 0;
2244 case X86::MULX64rr:
2245 SrcReg = X86::RDX; LoReg = HiReg = 0;
2266 if (MOpc == X86::MULX32rm || MOpc == X86::MULX64rm) {
2286 if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) {
2302 if (HiReg == X86::AH && Subtarget->is64Bit() &&
2305 X86::AX, MVT::i16, InFlag);
2311 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result));
2314 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16,
2319 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result));
2356 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break;
2357 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break;
2358 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break;
2359 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break;
2364 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break;
2365 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break;
2366 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break;
2367 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break;
2376 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
2378 SExtOpcode = X86::CBW;
2381 LoReg = X86::AX; HiReg = X86::DX;
2382 ClrOpcode = X86::MOV16r0; ClrReg = X86::DX;
2383 SExtOpcode = X86::CWD;
2386 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
2387 ClrOpcode = X86::MOV32r0;
2388 SExtOpcode = X86::CDQ;
2391 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
2392 ClrOpcode = X86::MOV64r0;
2393 SExtOpcode = X86::CQO;
2409 SDValue(CurDAG->getMachineNode(X86::MOVZX32rm8, dl, MVT::i32,
2416 SDValue(CurDAG->getMachineNode(X86::MOVZX32rr8, dl, MVT::i32, N0),0);
2419 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move, SDValue());
2454 if (HiReg == X86::AH && Subtarget->is64Bit() &&
2457 X86::AX, MVT::i16, InFlag);
2465 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result));
2468 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16,
2473 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result));
2513 X86::isZeroNode(N1)) {
2524 // On x86-32, only the ABCD registers have 8-bit subregisters.
2528 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break;
2529 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break;
2533 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl,
2538 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl,
2542 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32,
2563 case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break;
2564 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break;
2565 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break;
2569 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl,
2573 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl,
2579 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl,
2597 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl,
2601 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32,
2619 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl,
2623 SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32,
2721 /// X86-specific DAG, ready for instruction scheduling.
