Lines Matching refs:SystemZ
1 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
28 #define DEBUG_TYPE "systemz-lower"
91 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass);
93 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass);
94 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass);
96 addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass);
97 addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass);
99 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass);
100 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass);
102 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass);
105 addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass);
106 addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass);
107 addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass);
108 addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass);
109 addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass);
110 addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass);
117 setStackPointerRegisterToSaveRestore(SystemZ::R15D);
121 // such as SystemZ has with CC, so set this to the register-pressure
429 // VASTART and VACOPY need to deal with the SystemZ-specific varargs
668 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
670 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
671 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
675 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
677 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
678 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
681 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
685 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
687 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
688 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
698 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass,
701 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass,
703 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass,
708 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass,
711 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass,
713 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass,
896 RC = &SystemZ::GR32BitRegClass;
900 RC = &SystemZ::GR64BitRegClass;
904 RC = &SystemZ::FP32BitRegClass;
908 RC = &SystemZ::FP64BitRegClass;
916 RC = &SystemZ::VR128BitRegClass;
966 if (NumFixedFPRs < SystemZ::NumArgFPRs) {
967 SDValue MemOps[SystemZ::NumArgFPRs];
968 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) {
969 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]);
972 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I],
973 &SystemZ::FP64BitRegClass);
982 SystemZ::NumArgFPRs-NumFixedFPRs));
1000 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D)
1075 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT);
1104 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue);
1106 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType());
1252 CCValid = SystemZ::CCMASK_TBEGIN;
1257 CCValid = SystemZ::CCMASK_TBEGIN;
1262 SystemZ::CCMASK_TEND;
1280 CCValid = SystemZ::CCMASK_VCMP;
1287 CCValid = SystemZ::CCMASK_VCMP;
1295 CCValid = SystemZ::CCMASK_VCMP;
1303 CCValid = SystemZ::CCMASK_VCMP;
1311 CCValid = SystemZ::CCMASK_VCMP;
1316 CCValid = SystemZ::CCMASK_VCMP;
1323 CCValid = SystemZ::CCMASK_ANY;
1330 CCValid = SystemZ::CCMASK_ANY;
1337 CCValid = SystemZ::CCMASK_ANY;
1344 CCValid = SystemZ::CCMASK_ANY;
1351 CCValid = SystemZ::CCMASK_ANY;
1358 CCValid = SystemZ::CCMASK_ANY;
1365 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3;
1372 CCValid = SystemZ::CCMASK_ANY;
1379 CCValid = SystemZ::CCMASK_ANY;
1384 CCValid = SystemZ::CCMASK_VCMP;
1389 CCValid = SystemZ::CCMASK_VCMP;
1394 CCValid = SystemZ::CCMASK_VCMP;
1399 CCValid = SystemZ::CCMASK_VCMP;
1451 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
1452 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
1453 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
1466 case ISD::SETO: return SystemZ::CCMASK_CMP_O;
1467 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO;
1478 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1479 return IPMConversion(0, 0, SystemZ::IPM_CC);
1480 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1481 return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1491 if (CCMask == (CCValid & SystemZ::CCMASK_0))
1492 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1493 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1494 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1495 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1496 | SystemZ::CCMASK_1
1497 | SystemZ::CCMASK_2)))
1498 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1499 if (CCMask == (CCValid & SystemZ::CCMASK_3))
1500 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1501 if (CCMask == (CCValid & (SystemZ::CCMASK_1
1502 | SystemZ::CCMASK_2
1503 | SystemZ::CCMASK_3)))
1504 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1508 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1509 return IPMConversion(-1, 0, SystemZ::IPM_CC);
1513 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1514 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1515 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1516 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1521 if (CCMask == (CCValid & SystemZ::CCMASK_1))
1522 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1523 if (CCMask == (CCValid & SystemZ::CCMASK_2))
1524 return IPMConversion(1 << SystemZ::IPM_CC,
1525 TopBit - (3 << SystemZ::IPM_CC), 31);
1526 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1527 | SystemZ::CCMASK_1
1528 | SystemZ::CCMASK_3)))
1529 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1530 if (CCMask == (CCValid & (SystemZ::CCMASK_0
1531 | SystemZ::CCMASK_2
1532 | SystemZ::CCMASK_3)))
1533 return IPMConversion(1 << SystemZ::IPM_CC,
1534 TopBit - (1 << SystemZ::IPM_CC), 31);
1550 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) ||
1551 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) ||
1552 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) ||
1553 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) {
1554 C.CCMask ^= SystemZ::CCMASK_CMP_EQ;
1592 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT)
1594 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT;
1595 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE)
1597 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT;
1713 SystemZ::CCMASK_CMP_EQ) |
1714 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) |
1715 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) |
1716 (CCMask & SystemZ::CCMASK_CMP_UO));
1723 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
1724 C.CCMask == SystemZ::CCMASK_CMP_NE) {
1832 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) &&
1833 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask))
1847 if (CCMask == SystemZ::CCMASK_CMP_EQ)
1848 return SystemZ::CCMASK_TM_ALL_0;
1849 if (CCMask == SystemZ::CCMASK_CMP_NE)
1850 return SystemZ::CCMASK_TM_SOME_1;
1853 if (CCMask == SystemZ::CCMASK_CMP_LT)
1854 return SystemZ::CCMASK_TM_ALL_0;
1855 if (CCMask == SystemZ::CCMASK_CMP_GE)
1856 return SystemZ::CCMASK_TM_SOME_1;
1859 if (CCMask == SystemZ::CCMASK_CMP_LE)
1860 return SystemZ::CCMASK_TM_ALL_0;
1861 if (CCMask == SystemZ::CCMASK_CMP_GT)
1862 return SystemZ::CCMASK_TM_SOME_1;
1867 if (CCMask == SystemZ::CCMASK_CMP_EQ)
1868 return SystemZ::CCMASK_TM_ALL_1;
1869 if (CCMask == SystemZ::CCMASK_CMP_NE)
1870 return SystemZ::CCMASK_TM_SOME_0;
1873 if (CCMask == SystemZ::CCMASK_CMP_GT)
1874 return SystemZ::CCMASK_TM_ALL_1;
1875 if (CCMask == SystemZ::CCMASK_CMP_LE)
1876 return SystemZ::CCMASK_TM_SOME_0;
1879 if (CCMask == SystemZ::CCMASK_CMP_GE)
1880 return SystemZ::CCMASK_TM_ALL_1;
1881 if (CCMask == SystemZ::CCMASK_CMP_LT)
1882 return SystemZ::CCMASK_TM_SOME_0;
1887 if (CCMask == SystemZ::CCMASK_CMP_LE)
1888 return SystemZ::CCMASK_TM_MSB_0;
1889 if (CCMask == SystemZ::CCMASK_CMP_GT)
1890 return SystemZ::CCMASK_TM_MSB_1;
1893 if (CCMask == SystemZ::CCMASK_CMP_LT)
1894 return SystemZ::CCMASK_TM_MSB_0;
1895 if (CCMask == SystemZ::CCMASK_CMP_GE)
1896 return SystemZ::CCMASK_TM_MSB_1;
1902 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low)
1903 return SystemZ::CCMASK_TM_MIXED_MSB_0;
1904 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low)
1905 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY;
1906 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High)
1907 return SystemZ::CCMASK_TM_MIXED_MSB_1;
1908 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High)
1909 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY;
1941 NewC.CCMask == SystemZ::CCMASK_CMP_EQ ||
1942 NewC.CCMask == SystemZ::CCMASK_CMP_NE ||
1946 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE ||
1947 NewC.CCMask == SystemZ::CCMASK_CMP_GT) {
1951 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ;
1997 C.CCValid = SystemZ::CCMASK_TM;
2055 C.CCValid = SystemZ::CCMASK_FCMP;
2059 C.CCValid = SystemZ::CCMASK_ICMP;
2066 if (C.CCMask == SystemZ::CCMASK_CMP_EQ ||
2067 C.CCMask == SystemZ::CCMASK_CMP_NE ||
2070 else if (C.CCMask & SystemZ::CCMASK_CMP_UO)
2074 C.CCMask &= ~SystemZ::CCMASK_CMP_UO;
2111 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
2112 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
2147 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result);
2148 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result);
2369 C.CCMask != SystemZ::CCMASK_CMP_EQ &&
2370 C.CCMask != SystemZ::CCMASK_CMP_NE &&
2374 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT);
2376 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT);
2462 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue);
2464 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue);
2476 Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT));
2477 Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT));
2495 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue);
2662 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL,
2670 return DAG.getTargetExtractSubreg(SystemZ::subreg_r32,
2675 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_r32, DL,
2679 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL,
2829 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64,
2853 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, SystemZISD::UMUL_LOHI64,
2880 lowerGR128Binary(DAG, DL, VT, SystemZ::AEXT128_64, Opcode,
2895 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_32, SystemZISD::UDIVREM32,
2898 lowerGR128Binary(DAG, DL, VT, SystemZ::ZEXT128_64, SystemZISD::UDIVREM64,
2955 return DAG.getTargetInsertSubreg(SystemZ::subreg_l32, DL,
3051 return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other,
3216 SystemZ::R15D, Op.getValueType());
3224 SystemZ::R15D, Op.getOperand(1));
3236 unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
3255 DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
3342 unsigned char Bytes[SystemZ::VectorBytes];
3421 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I) {
3427 if ((Elt ^ P.Bytes[I]) & (SystemZ::VectorBytes - 1))
3429 int ModelOpNo = P.Bytes[I] / SystemZ::VectorBytes;
3430 int RealOpNo = unsigned(Elt) / SystemZ::VectorBytes;
3459 for (unsigned From = 0; From < SystemZ::VectorBytes; ++From) {
3467 if (To == SystemZ::VectorBytes)
3536 int ExpectedShift = (Index - I) % SystemZ::VectorBytes;
3537 int ModelOpNo = unsigned(ExpectedShift + I) / SystemZ::VectorBytes;
3538 int RealOpNo = unsigned(Index) / SystemZ::VectorBytes;
3565 SystemZ::VectorBytes / InBytes);
3574 SystemZ::VectorBytes / P.Operand);
3597 SDValue IndexNodes[SystemZ::VectorBytes];
3598 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
3616 SmallVector<SDValue, SystemZ::VectorBytes> Ops;
3619 // result comes from byte Bytes[I] % SystemZ::VectorBytes of operand
3620 // Bytes[I] / SystemZ::VectorBytes.
3621 SmallVector<int, SystemZ::VectorBytes> Bytes;
3649 unsigned Byte = ((Elem * FromBytesPerElement) % SystemZ::VectorBytes +
3659 SmallVector<int, SystemZ::VectorBytes> OpBytes;
3668 Op = Op.getOperand(unsigned(NewByte) / SystemZ::VectorBytes);
3669 Byte = unsigned(NewByte) % SystemZ::VectorBytes;
3686 unsigned Base = OpNo * SystemZ::VectorBytes + Byte;
3693 assert(Bytes.size() == SystemZ::VectorBytes && "Incomplete vector");
3717 SmallVector<int, SystemZ::VectorBytes> NewBytes(SystemZ::VectorBytes);
3718 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
3719 unsigned OpNo = unsigned(Bytes[J]) / SystemZ::VectorBytes;
3720 unsigned Byte = unsigned(Bytes[J]) % SystemZ::VectorBytes;
3724 NewBytes[J] = SystemZ::VectorBytes + Byte;
3729 SmallVector<int, SystemZSystemZ::VectorBytes);
3733 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J) {
3735 assert(unsigned(NewBytesMap[J]) < SystemZ::VectorBytes &&
3737 Bytes[J] = I * SystemZ::VectorBytes + NewBytesMap[J];
3744 for (unsigned J = 0; J < SystemZ::VectorBytes; ++J)
3746 Bytes[J] = I * SystemZ::VectorBytes + J;
3754 for (unsigned I = 0; I < SystemZ::VectorBytes; ++I)
3755 if (Bytes[I] >= int(SystemZ::VectorBytes))
3756 Bytes[I] -= (Stride - 1) * SystemZ::VectorBytes;
3871 SystemZ::VectorBits / BitsPerElement);
3888 SystemZ::VectorBits / BitsPerElement);
3911 SmallVector<SDValue, SystemZ::VectorBytes> ResidueOps;
4016 SmallVector<SDValue, SystemZ::VectorBytes> Constants(NumElements, SDValue());
4017 SmallVector<bool, SystemZ::VectorBytes> Done(NumElements, false);
4133 SmallVector<SDValue, SystemZ::VectorBytes> Ops(NumElements);
4249 SystemZ::VectorBits / FromBits);
4557 SmallVector<int, SystemZ::VectorBytes> Bytes;
4729 SystemZ::VectorBytes / ElemBytes / 2);
4862 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
4863 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LA), Reg)
4890 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
4904 BuildMI(*MBB, MI, DL, TII->get(SystemZ::PHI), DestReg)
4959 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5009 &SystemZ::GR32BitRegClass :
5010 &SystemZ::GR64BitRegClass);
5011 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG;
5012 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG;
5050 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
5054 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
5063 BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal)
5069 BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp);
5070 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal)
5080 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal)
5084 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
5088 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5089 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
5128 &SystemZ::GR32BitRegClass :
5129 &SystemZ::GR64BitRegClass);
5130 unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG;
5131 unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG;
5168 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
5172 BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal)
5176 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5177 .addImm(SystemZ::CCMASK_ICMP).addImm(KeepOldMask).addMBB(UpdateMBB);
5186 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal)
5199 BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal)
5203 BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal)
5207 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5208 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
5237 const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
5240 unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
5241 unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
5284 BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
5287 BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
5290 BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
5293 BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
5295 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
5297 BuildMI(MBB, DL, TII->get(SystemZ::CR))
5299 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5300 .addImm(SystemZ::CCMASK_ICMP)
5301 .addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
5315 BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
5317 BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
5321 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5322 .addImm(SystemZ::CCMASK_CS).addImm(SystemZ::CCMASK_CS_NE).addMBB(LoopMBB);
5346 unsigned In128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
5350 unsigned NewIn128 = MRI.createVirtualRegister(&SystemZ::GR128BitRegClass);
5351 unsigned Zero64 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);
5353 BuildMI(*MBB, MI, DL, TII->get(SystemZ::LLILL), Zero64)
5356 .addReg(In128).addReg(Zero64).addImm(SystemZ::subreg_h64);
5384 MachineBasicBlock *EndMBB = (Length > 256 && Opcode == SystemZ::CLC ?
5396 const TargetRegisterClass *RC = &SystemZ::ADDR64BitRegClass;
5404 RC = &SystemZ::GR64BitRegClass;
5431 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisDestReg)
5435 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisSrcReg)
5438 BuildMI(MBB, DL, TII->get(SystemZ::PHI), ThisCountReg)
5441 if (Opcode == SystemZ::MVC)
5442 BuildMI(MBB, DL, TII->get(SystemZ::PFD))
5443 .addImm(SystemZ::PFD_WRITE)
5449 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5450 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
5467 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextDestReg)
5470 BuildMI(MBB, DL, TII->get(SystemZ::LA), NextSrcReg)
5472 BuildMI(MBB, DL, TII->get(SystemZ::AGHI), NextCountReg)
5474 BuildMI(MBB, DL, TII->get(SystemZ::CGHI))
5476 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5477 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
5493 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
5494 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg)
5500 unsigned Reg = MRI.createVirtualRegister(&SystemZ::ADDR64BitRegClass);
5501 BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(SystemZ::LAY), Reg)
5516 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5517 .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_NE)
5527 MBB->addLiveIn(SystemZ::CC);
5551 const TargetRegisterClass *RC = &SystemZ::GR64BitRegClass;
5575 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This1Reg)
5578 BuildMI(MBB, DL, TII->get(SystemZ::PHI), This2Reg)
5581 BuildMI(MBB, DL, TII->get(TargetOpcode::COPY), SystemZ::R0L).addReg(CharReg);
5585 BuildMI(MBB, DL, TII->get(SystemZ::BRC))
5586 .addImm(SystemZ::CCMASK_ANY).addImm(SystemZ::CCMASK_3).addMBB(LoopMBB);
5590 DoneMBB->addLiveIn(SystemZ::CC);
5675 case SystemZ::Select32Mux:
5676 case SystemZ::Select32:
5677 case SystemZ::SelectF32:
5678 case SystemZ::Select64:
5679 case SystemZ::SelectF64:
5680 case SystemZ::SelectF128:
5683 case SystemZ::CondStore8Mux:
5684 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, false);
5685 case SystemZ::CondStore8MuxInv:
5686 return emitCondStore(MI, MBB, SystemZ::STCMux, 0, true);
5687 case SystemZ::CondStore16Mux:
5688 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, false);
5689 case SystemZ::CondStore16MuxInv:
5690 return emitCondStore(MI, MBB, SystemZ::STHMux, 0, true);
5691 case SystemZ::CondStore8:
5692 return emitCondStore(MI, MBB, SystemZ::STC, 0, false);
5693 case SystemZ::CondStore8Inv:
5694 return emitCondStore(MI, MBB, SystemZ::STC, 0, true);
5695 case SystemZ::CondStore16:
5696 return emitCondStore(MI, MBB, SystemZ::STH, 0, false);
5697 case SystemZ::CondStore16Inv:
5698 return emitCondStore(MI, MBB, SystemZ::STH, 0, true);
5699 case SystemZ::CondStore32:
5700 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, false);
5701 case SystemZ::CondStore32Inv:
5702 return emitCondStore(MI, MBB, SystemZ::ST, SystemZ::STOC, true);
5703 case SystemZ::CondStore64:
5704 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, false);
5705 case SystemZ::CondStore64Inv:
5706 return emitCondStore(MI, MBB, SystemZ::STG, SystemZ::STOCG, true);
5707 case SystemZ::CondStoreF32:
5708 return emitCondStore(MI, MBB, SystemZ::STE, 0, false);
5709 case SystemZ::CondStoreF32Inv:
5710 return emitCondStore(MI, MBB, SystemZ::STE, 0, true);
5711 case SystemZ::CondStoreF64:
5712 return emitCondStore(MI, MBB, SystemZ::STD, 0, false);
5713 case SystemZ::CondStoreF64Inv:
5714 return emitCondStore(MI, MBB, SystemZ::STD, 0, true);
5716 case SystemZ::AEXT128_64:
5717 return emitExt128(MI, MBB, false, SystemZ::subreg_l64);
5718 case SystemZ::ZEXT128_32:
5719 return emitExt128(MI, MBB, true, SystemZ::subreg_l32);
5720 case SystemZ::ZEXT128_64:
5721 return emitExt128(MI, MBB, true, SystemZ::subreg_l64);
5723 case SystemZ::ATOMIC_SWAPW:
5725 case SystemZ::ATOMIC_SWAP_32:
5727 case SystemZ::ATOMIC_SWAP_64:
5730 case SystemZ::ATOMIC_LOADW_AR:
5731 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0);
5732 case SystemZ::ATOMIC_LOADW_AFI:
5733 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0);
5734 case SystemZ::ATOMIC_LOAD_AR:
5735 return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32);
5736 case SystemZ::ATOMIC_LOAD_AHI:
5737 return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32);
5738 case SystemZ::ATOMIC_LOAD_AFI:
5739 return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32);
5740 case SystemZ::ATOMIC_LOAD_AGR:
5741 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64);
5742 case SystemZ::ATOMIC_LOAD_AGHI:
5743 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64);
5744 case SystemZ::ATOMIC_LOAD_AGFI:
5745 return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64);
5747 case SystemZ::ATOMIC_LOADW_SR:
5748 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0);
5749 case SystemZ::ATOMIC_LOAD_SR:
5750 return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32);
5751 case SystemZ::ATOMIC_LOAD_SGR:
5752 return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64);
5754 case SystemZ::ATOMIC_LOADW_NR:
5755 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0);
5756 case SystemZ::ATOMIC_LOADW_NILH:
5757 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0);
5758 case SystemZ::ATOMIC_LOAD_NR:
5759 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32);
5760 case SystemZ::ATOMIC_LOAD_NILL:
5761 SystemZ::NILL, 32);
5762 case SystemZ::ATOMIC_LOAD_NILH:
5763 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32);
5764 case SystemZ::ATOMIC_LOAD_NILF:
5765 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32);
5766 case SystemZ::ATOMIC_LOAD_NGR:
5767 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64);
5768 case SystemZ::ATOMIC_LOAD_NILL64:
5769 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64);
5770 case SystemZ::ATOMIC_LOAD_NILH64:
5771 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64);
5772 case SystemZ::ATOMIC_LOAD_NIHL64:
5773 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64);
5774 case SystemZ::ATOMIC_LOAD_NIHH64:
5775 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64);
5776 case SystemZ::ATOMIC_LOAD_NILF64:
5777 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64);
5778 case SystemZ::ATOMIC_LOAD_NIHF64:
5779 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64);
5781 case SystemZ::ATOMIC_LOADW_OR:
5782 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0);
5783 case SystemZ::ATOMIC_LOADW_OILH:
5784 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0);
5785 case SystemZ::ATOMIC_LOAD_OR:
5786 return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32);
5787 case SystemZ::ATOMIC_LOAD_OILL:
5788 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32);
5789 case SystemZ::ATOMIC_LOAD_OILH:
5790 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32);
5791 case SystemZ::ATOMIC_LOAD_OILF:
5792 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32);
5793 case SystemZ::ATOMIC_LOAD_OGR:
5794 return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64);
5795 case SystemZ::ATOMIC_LOAD_OILL64:
5796 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64);
5797 case SystemZ::ATOMIC_LOAD_OILH64:
5798 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64);
5799 case SystemZ::ATOMIC_LOAD_OIHL64:
5800 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64);
5801 case SystemZ::ATOMIC_LOAD_OIHH64:
5802 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64);
5803 case SystemZ::ATOMIC_LOAD_OILF64:
5804 return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64);
5805 case SystemZ::ATOMIC_LOAD_OIHF64:
5806 return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64);
5808 case SystemZ::ATOMIC_LOADW_XR:
5809 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0);
5810 case SystemZ::ATOMIC_LOADW_XILF:
5811 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0);
5812 case SystemZ::ATOMIC_LOAD_XR:
5813 return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32);
5814 case SystemZ::ATOMIC_LOAD_XILF:
5815 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32);
5816 case SystemZ::ATOMIC_LOAD_XGR:
5817 return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64);
5818 case SystemZ::ATOMIC_LOAD_XILF64:
5819 return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64);
5820 case SystemZ::ATOMIC_LOAD_XIHF64:
5821 return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64);
5823 case SystemZ::ATOMIC_LOADW_NRi:
5824 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true);
5825 case SystemZ::ATOMIC_LOADW_NILHi:
5826 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true);
5827 case SystemZ::ATOMIC_LOAD_NRi:
5828 return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true);
5829 case SystemZ::ATOMIC_LOAD_NILLi:
5830 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true);
5831 case SystemZ::ATOMIC_LOAD_NILHi:
5832 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true);
5833 case SystemZ::ATOMIC_LOAD_NILFi:
5834 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true);
5835 case SystemZ::ATOMIC_LOAD_NGRi:
5836 return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true);
5837 case SystemZ::ATOMIC_LOAD_NILL64i:
5838 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true);
5839 case SystemZ::ATOMIC_LOAD_NILH64i:
5840 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true);
5841 case SystemZ::ATOMIC_LOAD_NIHL64i:
5842 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true);
5843 case SystemZ::ATOMIC_LOAD_NIHH64i:
5844 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true);
5845 case SystemZ::ATOMIC_LOAD_NILF64i:
5846 return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true);
5847 case SystemZ::ATOMIC_LOAD_NIHF64i:
5848 return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true);
5850 case SystemZ::ATOMIC_LOADW_MIN:
5851 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
5852 SystemZ::CCMASK_CMP_LE, 0);
5853 case SystemZ::ATOMIC_LOAD_MIN_32:
5854 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
5855 SystemZ::CCMASK_CMP_LE, 32);
5856 case SystemZ::ATOMIC_LOAD_MIN_64:
5857 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR,
5858 SystemZ::CCMASK_CMP_LE, 64);
5860 case SystemZ::ATOMIC_LOADW_MAX:
5861 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
5862 SystemZ::CCMASK_CMP_GE, 0);
5863 case SystemZ::ATOMIC_LOAD_MAX_32:
5864 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR,
5865 SystemZ::CCMASK_CMP_GE, 32);
5866 case SystemZ::ATOMIC_LOAD_MAX_64:
5867 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR,
5868 SystemZ::CCMASK_CMP_GE, 64);
5870 case SystemZ::ATOMIC_LOADW_UMIN:
5871 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
5872 SystemZ::CCMASK_CMP_LE, 0);
5873 case SystemZ::ATOMIC_LOAD_UMIN_32:
5874 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
5875 SystemZ::CCMASK_CMP_LE, 32);
5876 case SystemZ::ATOMIC_LOAD_UMIN_64:
5877 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR,
5878 SystemZ::CCMASK_CMP_LE, 64);
5880 case SystemZ::ATOMIC_LOADW_UMAX:
5881 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
5882 SystemZ::CCMASK_CMP_GE, 0);
5883 case SystemZ::ATOMIC_LOAD_UMAX_32:
5884 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR,
5885 SystemZ::CCMASK_CMP_GE, 32);
5886 case SystemZ::ATOMIC_LOAD_UMAX_64:
5887 return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR,
5888 SystemZ::CCMASK_CMP_GE, 64);
5890 case SystemZ::ATOMIC_CMP_SWAPW:
5892 case SystemZ::MVCSequence:
5893 case SystemZ::MVCLoop:
5894 return emitMemMemWrapper(MI, MBB, SystemZ::MVC);
5895 case SystemZ::NCSequence:
5896 case SystemZ::NCLoop:
5897 return emitMemMemWrapper(MI, MBB, SystemZ::NC);
5898 case SystemZ::OCSequence:
5899 case SystemZ::OCLoop:
5900 return emitMemMemWrapper(MI, MBB, SystemZ::OC);
5901 case SystemZ::XCSequence:
5902 case SystemZ::XCLoop:
5903 return emitMemMemWrapper(MI, MBB, SystemZ::XC);
5904 case SystemZ::CLCSequence:
5905 case SystemZ::CLCLoop:
5906 return emitMemMemWrapper(MI, MBB, SystemZ::CLC);
5907 case SystemZ::CLSTLoop:
5908 return emitStringWrapper(MI, MBB, SystemZ::CLST);
5909 case SystemZ::MVSTLoop:
5910 return emitStringWrapper(MI, MBB, SystemZ::MVST);
5911 case SystemZ::SRSTLoop:
5912 return emitStringWrapper(MI, MBB, SystemZ::SRST);
5913 case SystemZ::TBEGIN:
5914 return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, false);
5915 case SystemZ::TBEGIN_nofloat:
5916 return emitTransactionBegin(MI, MBB, SystemZ::TBEGIN, true);
5917 case SystemZ::TBEGINC:
5918 return emitTransactionBegin(MI, MBB, SystemZ::TBEGINC, true);
5919 case SystemZ::LTEBRCompare_VecPseudo:
5920 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTEBR);
5921 case SystemZ::LTDBRCompare_VecPseudo:
5922 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTDBR);
5923 case SystemZ::LTXBRCompare_VecPseudo:
5924 return emitLoadAndTestCmp0(MI, MBB, SystemZ::LTXBR);
