1 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the SystemZTargetLowering class. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SystemZISelLowering.h" 15 #include "SystemZCallingConv.h" 16 #include "SystemZConstantPoolValue.h" 17 #include "SystemZMachineFunctionInfo.h" 18 #include "SystemZTargetMachine.h" 19 #include "llvm/CodeGen/CallingConvLower.h" 20 #include "llvm/CodeGen/MachineInstrBuilder.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/IntrinsicInst.h" 25 #include "llvm/Support/CommandLine.h" 26 #include "llvm/Support/KnownBits.h" 27 #include <cctype> 28 29 using namespace llvm; 30 31 #define DEBUG_TYPE "systemz-lower" 32 33 namespace { 34 // Represents information about a comparison. 35 struct Comparison { 36 Comparison(SDValue Op0In, SDValue Op1In) 37 : Op0(Op0In), Op1(Op1In), Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {} 38 39 // The operands to the comparison. 40 SDValue Op0, Op1; 41 42 // The opcode that should be used to compare Op0 and Op1. 43 unsigned Opcode; 44 45 // A SystemZICMP value. Only used for integer comparisons. 46 unsigned ICmpType; 47 48 // The mask of CC values that Opcode can produce. 49 unsigned CCValid; 50 51 // The mask of CC values for which the original condition is true. 52 unsigned CCMask; 53 }; 54 } // end anonymous namespace 55 56 // Classify VT as either 32 or 64 bit. 57 static bool is32Bit(EVT VT) { 58 switch (VT.getSimpleVT().SimpleTy) { 59 case MVT::i32: 60 return true; 61 case MVT::i64: 62 return false; 63 default: 64 llvm_unreachable("Unsupported type"); 65 } 66 } 67 68 // Return a version of MachineOperand that can be safely used before the 69 // final use. 70 static MachineOperand earlyUseOperand(MachineOperand Op) { 71 if (Op.isReg()) 72 Op.setIsKill(false); 73 return Op; 74 } 75 76 SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, 77 const SystemZSubtarget &STI) 78 : TargetLowering(TM), Subtarget(STI) { 79 MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize(0)); 80 81 // Set up the register classes. 82 if (Subtarget.hasHighWord()) 83 addRegisterClass(MVT::i32, &SystemZ::GRX32BitRegClass); 84 else 85 addRegisterClass(MVT::i32, &SystemZ::GR32BitRegClass); 86 addRegisterClass(MVT::i64, &SystemZ::GR64BitRegClass); 87 if (Subtarget.hasVector()) { 88 addRegisterClass(MVT::f32, &SystemZ::VR32BitRegClass); 89 addRegisterClass(MVT::f64, &SystemZ::VR64BitRegClass); 90 } else { 91 addRegisterClass(MVT::f32, &SystemZ::FP32BitRegClass); 92 addRegisterClass(MVT::f64, &SystemZ::FP64BitRegClass); 93 } 94 if (Subtarget.hasVectorEnhancements1()) 95 addRegisterClass(MVT::f128, &SystemZ::VR128BitRegClass); 96 else 97 addRegisterClass(MVT::f128, &SystemZ::FP128BitRegClass); 98 99 if (Subtarget.hasVector()) { 100 addRegisterClass(MVT::v16i8, &SystemZ::VR128BitRegClass); 101 addRegisterClass(MVT::v8i16, &SystemZ::VR128BitRegClass); 102 addRegisterClass(MVT::v4i32, &SystemZ::VR128BitRegClass); 103 addRegisterClass(MVT::v2i64, &SystemZ::VR128BitRegClass); 104 addRegisterClass(MVT::v4f32, &SystemZ::VR128BitRegClass); 105 addRegisterClass(MVT::v2f64, &SystemZ::VR128BitRegClass); 106 } 107 108 // Compute derived properties from the register classes 109 computeRegisterProperties(Subtarget.getRegisterInfo()); 110 111 // Set up special registers. 112 setStackPointerRegisterToSaveRestore(SystemZ::R15D); 113 114 // TODO: It may be better to default to latency-oriented scheduling, however 115 // LLVM's current latency-oriented scheduler can't handle physreg definitions 116 // such as SystemZ has with CC, so set this to the register-pressure 117 // scheduler, because it can. 118 setSchedulingPreference(Sched::RegPressure); 119 120 setBooleanContents(ZeroOrOneBooleanContent); 121 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 122 123 // Instructions are strings of 2-byte aligned 2-byte values. 124 setMinFunctionAlignment(2); 125 // For performance reasons we prefer 16-byte alignment. 126 setPrefFunctionAlignment(4); 127 128 // Handle operations that are handled in a similar way for all types. 129 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 130 I <= MVT::LAST_FP_VALUETYPE; 131 ++I) { 132 MVT VT = MVT::SimpleValueType(I); 133 if (isTypeLegal(VT)) { 134 // Lower SET_CC into an IPM-based sequence. 135 setOperationAction(ISD::SETCC, VT, Custom); 136 137 // Expand SELECT(C, A, B) into SELECT_CC(X, 0, A, B, NE). 138 setOperationAction(ISD::SELECT, VT, Expand); 139 140 // Lower SELECT_CC and BR_CC into separate comparisons and branches. 141 setOperationAction(ISD::SELECT_CC, VT, Custom); 142 setOperationAction(ISD::BR_CC, VT, Custom); 143 } 144 } 145 146 // Expand jump table branches as address arithmetic followed by an 147 // indirect jump. 148 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 149 150 // Expand BRCOND into a BR_CC (see above). 151 setOperationAction(ISD::BRCOND, MVT::Other, Expand); 152 153 // Handle integer types. 154 for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE; 155 I <= MVT::LAST_INTEGER_VALUETYPE; 156 ++I) { 157 MVT VT = MVT::SimpleValueType(I); 158 if (isTypeLegal(VT)) { 159 // Expand individual DIV and REMs into DIVREMs. 160 setOperationAction(ISD::SDIV, VT, Expand); 161 setOperationAction(ISD::UDIV, VT, Expand); 162 setOperationAction(ISD::SREM, VT, Expand); 163 setOperationAction(ISD::UREM, VT, Expand); 164 setOperationAction(ISD::SDIVREM, VT, Custom); 165 setOperationAction(ISD::UDIVREM, VT, Custom); 166 167 // Support addition/subtraction with overflow. 168 setOperationAction(ISD::SADDO, VT, Custom); 169 setOperationAction(ISD::SSUBO, VT, Custom); 170 171 // Support addition/subtraction with carry. 172 setOperationAction(ISD::UADDO, VT, Custom); 173 setOperationAction(ISD::USUBO, VT, Custom); 174 175 // Support carry in as value rather than glue. 176 setOperationAction(ISD::ADDCARRY, VT, Custom); 177 setOperationAction(ISD::SUBCARRY, VT, Custom); 178 179 // Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and 180 // stores, putting a serialization instruction after the stores. 181 setOperationAction(ISD::ATOMIC_LOAD, VT, Custom); 182 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); 183 184 // Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are 185 // available, or if the operand is constant. 186 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); 187 188 // Use POPCNT on z196 and above. 189 if (Subtarget.hasPopulationCount()) 190 setOperationAction(ISD::CTPOP, VT, Custom); 191 else 192 setOperationAction(ISD::CTPOP, VT, Expand); 193 194 // No special instructions for these. 195 setOperationAction(ISD::CTTZ, VT, Expand); 196 setOperationAction(ISD::ROTR, VT, Expand); 197 198 // Use *MUL_LOHI where possible instead of MULH*. 199 setOperationAction(ISD::MULHS, VT, Expand); 200 setOperationAction(ISD::MULHU, VT, Expand); 201 setOperationAction(ISD::SMUL_LOHI, VT, Custom); 202 setOperationAction(ISD::UMUL_LOHI, VT, Custom); 203 204 // Only z196 and above have native support for conversions to unsigned. 205 // On z10, promoting to i64 doesn't generate an inexact condition for 206 // values that are outside the i32 range but in the i64 range, so use 207 // the default expansion. 208 if (!Subtarget.hasFPExtension()) 209 setOperationAction(ISD::FP_TO_UINT, VT, Expand); 210 } 211 } 212 213 // Type legalization will convert 8- and 16-bit atomic operations into 214 // forms that operate on i32s (but still keeping the original memory VT). 215 // Lower them into full i32 operations. 216 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Custom); 217 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Custom); 218 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); 219 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); 220 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Custom); 221 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Custom); 222 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Custom); 223 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Custom); 224 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Custom); 225 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Custom); 226 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Custom); 227 228 // Even though i128 is not a legal type, we still need to custom lower 229 // the atomic operations in order to exploit SystemZ instructions. 230 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom); 231 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom); 232 233 // We can use the CC result of compare-and-swap to implement 234 // the "success" result of ATOMIC_CMP_SWAP_WITH_SUCCESS. 235 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Custom); 236 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Custom); 237 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); 238 239 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); 240 241 // Traps are legal, as we will convert them to "j .+2". 242 setOperationAction(ISD::TRAP, MVT::Other, Legal); 243 244 // z10 has instructions for signed but not unsigned FP conversion. 245 // Handle unsigned 32-bit types as signed 64-bit types. 246 if (!Subtarget.hasFPExtension()) { 247 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Promote); 248 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); 249 } 250 251 // We have native support for a 64-bit CTLZ, via FLOGR. 252 setOperationAction(ISD::CTLZ, MVT::i32, Promote); 253 setOperationAction(ISD::CTLZ, MVT::i64, Legal); 254 255 // Give LowerOperation the chance to replace 64-bit ORs with subregs. 256 setOperationAction(ISD::OR, MVT::i64, Custom); 257 258 // FIXME: Can we support these natively? 259 setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand); 260 setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand); 261 setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand); 262 263 // We have native instructions for i8, i16 and i32 extensions, but not i1. 264 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 265 for (MVT VT : MVT::integer_valuetypes()) { 266 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 267 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 268 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote); 269 } 270 271 // Handle the various types of symbolic address. 272 setOperationAction(ISD::ConstantPool, PtrVT, Custom); 273 setOperationAction(ISD::GlobalAddress, PtrVT, Custom); 274 setOperationAction(ISD::GlobalTLSAddress, PtrVT, Custom); 275 setOperationAction(ISD::BlockAddress, PtrVT, Custom); 276 setOperationAction(ISD::JumpTable, PtrVT, Custom); 277 278 // We need to handle dynamic allocations specially because of the 279 // 160-byte area at the bottom of the stack. 280 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom); 281 setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, PtrVT, Custom); 282 283 // Use custom expanders so that we can force the function to use 284 // a frame pointer. 285 setOperationAction(ISD::STACKSAVE, MVT::Other, Custom); 286 setOperationAction(ISD::STACKRESTORE, MVT::Other, Custom); 287 288 // Handle prefetches with PFD or PFDRL. 289 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); 290 291 for (MVT VT : MVT::vector_valuetypes()) { 292 // Assume by default that all vector operations need to be expanded. 293 for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode) 294 if (getOperationAction(Opcode, VT) == Legal) 295 setOperationAction(Opcode, VT, Expand); 296 297 // Likewise all truncating stores and extending loads. 298 for (MVT InnerVT : MVT::vector_valuetypes()) { 299 setTruncStoreAction(VT, InnerVT, Expand); 300 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); 301 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); 302 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); 303 } 304 305 if (isTypeLegal(VT)) { 306 // These operations are legal for anything that can be stored in a 307 // vector register, even if there is no native support for the format 308 // as such. In particular, we can do these for v4f32 even though there 309 // are no specific instructions for that format. 310 setOperationAction(ISD::LOAD, VT, Legal); 311 setOperationAction(ISD::STORE, VT, Legal); 312 setOperationAction(ISD::VSELECT, VT, Legal); 313 setOperationAction(ISD::BITCAST, VT, Legal); 314 setOperationAction(ISD::UNDEF, VT, Legal); 315 316 // Likewise, except that we need to replace the nodes with something 317 // more specific. 318 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); 319 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); 320 } 321 } 322 323 // Handle integer vector types. 324 for (MVT VT : MVT::integer_vector_valuetypes()) { 325 if (isTypeLegal(VT)) { 326 // These operations have direct equivalents. 327 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal); 328 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Legal); 329 setOperationAction(ISD::ADD, VT, Legal); 330 setOperationAction(ISD::SUB, VT, Legal); 331 if (VT != MVT::v2i64) 332 setOperationAction(ISD::MUL, VT, Legal); 333 setOperationAction(ISD::AND, VT, Legal); 334 setOperationAction(ISD::OR, VT, Legal); 335 setOperationAction(ISD::XOR, VT, Legal); 336 if (Subtarget.hasVectorEnhancements1()) 337 setOperationAction(ISD::CTPOP, VT, Legal); 338 else 339 setOperationAction(ISD::CTPOP, VT, Custom); 340 setOperationAction(ISD::CTTZ, VT, Legal); 341 setOperationAction(ISD::CTLZ, VT, Legal); 342 343 // Convert a GPR scalar to a vector by inserting it into element 0. 344 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); 345 346 // Use a series of unpacks for extensions. 347 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom); 348 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom); 349 350 // Detect shifts by a scalar amount and convert them into 351 // V*_BY_SCALAR. 352 setOperationAction(ISD::SHL, VT, Custom); 353 setOperationAction(ISD::SRA, VT, Custom); 354 setOperationAction(ISD::SRL, VT, Custom); 355 356 // At present ROTL isn't matched by DAGCombiner. ROTR should be 357 // converted into ROTL. 358 setOperationAction(ISD::ROTL, VT, Expand); 359 setOperationAction(ISD::ROTR, VT, Expand); 360 361 // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands 362 // and inverting the result as necessary. 363 setOperationAction(ISD::SETCC, VT, Custom); 364 } 365 } 366 367 if (Subtarget.hasVector()) { 368 // There should be no need to check for float types other than v2f64 369 // since <2 x f32> isn't a legal type. 370 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); 371 setOperationAction(ISD::FP_TO_SINT, MVT::v2f64, Legal); 372 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); 373 setOperationAction(ISD::FP_TO_UINT, MVT::v2f64, Legal); 374 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); 375 setOperationAction(ISD::SINT_TO_FP, MVT::v2f64, Legal); 376 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); 377 setOperationAction(ISD::UINT_TO_FP, MVT::v2f64, Legal); 378 } 379 380 // Handle floating-point types. 381 for (unsigned I = MVT::FIRST_FP_VALUETYPE; 382 I <= MVT::LAST_FP_VALUETYPE; 383 ++I) { 384 MVT VT = MVT::SimpleValueType(I); 385 if (isTypeLegal(VT)) { 386 // We can use FI for FRINT. 387 setOperationAction(ISD::FRINT, VT, Legal); 388 389 // We can use the extended form of FI for other rounding operations. 390 if (Subtarget.hasFPExtension()) { 391 setOperationAction(ISD::FNEARBYINT, VT, Legal); 392 setOperationAction(ISD::FFLOOR, VT, Legal); 393 setOperationAction(ISD::FCEIL, VT, Legal); 394 setOperationAction(ISD::FTRUNC, VT, Legal); 395 setOperationAction(ISD::FROUND, VT, Legal); 396 } 397 398 // No special instructions for these. 399 setOperationAction(ISD::FSIN, VT, Expand); 400 setOperationAction(ISD::FCOS, VT, Expand); 401 setOperationAction(ISD::FSINCOS, VT, Expand); 402 setOperationAction(ISD::FREM, VT, Expand); 403 setOperationAction(ISD::FPOW, VT, Expand); 404 } 405 } 406 407 // Handle floating-point vector types. 408 if (Subtarget.hasVector()) { 409 // Scalar-to-vector conversion is just a subreg. 410 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); 411 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); 412 413 // Some insertions and extractions can be done directly but others 414 // need to go via integers. 415 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); 416 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); 417 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); 418 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); 419 420 // These operations have direct equivalents. 421 setOperationAction(ISD::FADD, MVT::v2f64, Legal); 422 setOperationAction(ISD::FNEG, MVT::v2f64, Legal); 423 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); 424 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); 425 setOperationAction(ISD::FMA, MVT::v2f64, Legal); 426 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); 427 setOperationAction(ISD::FABS, MVT::v2f64, Legal); 428 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); 429 setOperationAction(ISD::FRINT, MVT::v2f64, Legal); 430 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); 431 setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); 432 setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); 433 setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); 434 setOperationAction(ISD::FROUND, MVT::v2f64, Legal); 435 } 436 437 // The vector enhancements facility 1 has instructions for these. 438 if (Subtarget.hasVectorEnhancements1()) { 439 setOperationAction(ISD::FADD, MVT::v4f32, Legal); 440 setOperationAction(ISD::FNEG, MVT::v4f32, Legal); 441 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); 442 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); 443 setOperationAction(ISD::FMA, MVT::v4f32, Legal); 444 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); 445 setOperationAction(ISD::FABS, MVT::v4f32, Legal); 446 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); 447 setOperationAction(ISD::FRINT, MVT::v4f32, Legal); 448 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); 449 setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); 450 setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); 451 setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); 452 setOperationAction(ISD::FROUND, MVT::v4f32, Legal); 453 454 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal); 455 setOperationAction(ISD::FMAXNAN, MVT::f64, Legal); 456 setOperationAction(ISD::FMINNUM, MVT::f64, Legal); 457 setOperationAction(ISD::FMINNAN, MVT::f64, Legal); 458 459 setOperationAction(ISD::FMAXNUM, MVT::v2f64, Legal); 460 setOperationAction(ISD::FMAXNAN, MVT::v2f64, Legal); 461 setOperationAction(ISD::FMINNUM, MVT::v2f64, Legal); 462 setOperationAction(ISD::FMINNAN, MVT::v2f64, Legal); 463 464 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); 465 setOperationAction(ISD::FMAXNAN, MVT::f32, Legal); 466 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); 467 setOperationAction(ISD::FMINNAN, MVT::f32, Legal); 468 469 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); 470 setOperationAction(ISD::FMAXNAN, MVT::v4f32, Legal); 471 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); 472 setOperationAction(ISD::FMINNAN, MVT::v4f32, Legal); 473 474 setOperationAction(ISD::FMAXNUM, MVT::f128, Legal); 475 setOperationAction(ISD::FMAXNAN, MVT::f128, Legal); 476 setOperationAction(ISD::FMINNUM, MVT::f128, Legal); 477 setOperationAction(ISD::FMINNAN, MVT::f128, Legal); 478 } 479 480 // We have fused multiply-addition for f32 and f64 but not f128. 481 setOperationAction(ISD::FMA, MVT::f32, Legal); 482 setOperationAction(ISD::FMA, MVT::f64, Legal); 483 if (Subtarget.hasVectorEnhancements1()) 484 setOperationAction(ISD::FMA, MVT::f128, Legal); 485 else 486 setOperationAction(ISD::FMA, MVT::f128, Expand); 487 488 // We don't have a copysign instruction on vector registers. 489 if (Subtarget.hasVectorEnhancements1()) 490 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); 491 492 // Needed so that we don't try to implement f128 constant loads using 493 // a load-and-extend of a f80 constant (in cases where the constant 494 // would fit in an f80). 495 for (MVT VT : MVT::fp_valuetypes()) 496 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); 497 498 // We don't have extending load instruction on vector registers. 499 if (Subtarget.hasVectorEnhancements1()) { 500 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand); 501 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand); 502 } 503 504 // Floating-point truncation and stores need to be done separately. 505 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 506 setTruncStoreAction(MVT::f128, MVT::f32, Expand); 507 setTruncStoreAction(MVT::f128, MVT::f64, Expand); 508 509 // We have 64-bit FPR<->GPR moves, but need special handling for 510 // 32-bit forms. 511 if (!Subtarget.hasVector()) { 512 setOperationAction(ISD::BITCAST, MVT::i32, Custom); 513 setOperationAction(ISD::BITCAST, MVT::f32, Custom); 514 } 515 516 // VASTART and VACOPY need to deal with the SystemZ-specific varargs 517 // structure, but VAEND is a no-op. 518 setOperationAction(ISD::VASTART, MVT::Other, Custom); 519 setOperationAction(ISD::VACOPY, MVT::Other, Custom); 520 setOperationAction(ISD::VAEND, MVT::Other, Expand); 521 522 // Codes for which we want to perform some z-specific combinations. 523 setTargetDAGCombine(ISD::ZERO_EXTEND); 524 setTargetDAGCombine(ISD::SIGN_EXTEND); 525 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); 526 setTargetDAGCombine(ISD::STORE); 527 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); 528 setTargetDAGCombine(ISD::FP_ROUND); 529 setTargetDAGCombine(ISD::BSWAP); 530 531 // Handle intrinsics. 532 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 533 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 534 535 // We want to use MVC in preference to even a single load/store pair. 536 MaxStoresPerMemcpy = 0; 537 MaxStoresPerMemcpyOptSize = 0; 538 539 // The main memset sequence is a byte store followed by an MVC. 540 // Two STC or MV..I stores win over that, but the kind of fused stores 541 // generated by target-independent code don't when the byte value is 542 // variable. E.g. "STC <reg>;MHI <reg>,257;STH <reg>" is not better 543 // than "STC;MVC". Handle the choice in target-specific code instead. 544 MaxStoresPerMemset = 0; 545 MaxStoresPerMemsetOptSize = 0; 546 } 547 548 EVT SystemZTargetLowering::getSetCCResultType(const DataLayout &DL, 549 LLVMContext &, EVT VT) const { 550 if (!VT.isVector()) 551 return MVT::i32; 552 return VT.changeVectorElementTypeToInteger(); 553 } 554 555 bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { 556 VT = VT.getScalarType(); 557 558 if (!VT.isSimple()) 559 return false; 560 561 switch (VT.getSimpleVT().SimpleTy) { 562 case MVT::f32: 563 case MVT::f64: 564 return true; 565 case MVT::f128: 566 return Subtarget.hasVectorEnhancements1(); 567 default: 568 break; 569 } 570 571 return false; 572 } 573 574 bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { 575 // We can load zero using LZ?R and negative zero using LZ?R;LC?BR. 576 return Imm.isZero() || Imm.isNegZero(); 577 } 578 579 bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const { 580 // We can use CGFI or CLGFI. 581 return isInt<32>(Imm) || isUInt<32>(Imm); 582 } 583 584 bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const { 585 // We can use ALGFI or SLGFI. 586 return isUInt<32>(Imm) || isUInt<32>(-Imm); 587 } 588 589 bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, 590 unsigned, 591 unsigned, 592 bool *Fast) const { 593 // Unaligned accesses should never be slower than the expanded version. 594 // We check specifically for aligned accesses in the few cases where 595 // they are required. 596 if (Fast) 597 *Fast = true; 598 return true; 599 } 600 601 // Information about the addressing mode for a memory access. 602 struct AddressingMode { 603 // True if a long displacement is supported. 604 bool LongDisplacement; 605 606 // True if use of index register is supported. 607 bool IndexReg; 608 609 AddressingMode(bool LongDispl, bool IdxReg) : 610 LongDisplacement(LongDispl), IndexReg(IdxReg) {} 611 }; 612 613 // Return the desired addressing mode for a Load which has only one use (in 614 // the same block) which is a Store. 615 static AddressingMode getLoadStoreAddrMode(bool HasVector, 616 Type *Ty) { 617 // With vector support a Load->Store combination may be combined to either 618 // an MVC or vector operations and it seems to work best to allow the 619 // vector addressing mode. 620 if (HasVector) 621 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/); 622 623 // Otherwise only the MVC case is special. 624 bool MVC = Ty->isIntegerTy(8); 625 return AddressingMode(!MVC/*LongDispl*/, !MVC/*IdxReg*/); 626 } 627 628 // Return the addressing mode which seems most desirable given an LLVM 629 // Instruction pointer. 630 static AddressingMode 631 supportedAddressingMode(Instruction *I, bool HasVector) { 632 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { 633 switch (II->getIntrinsicID()) { 634 default: break; 635 case Intrinsic::memset: 636 case Intrinsic::memmove: 637 case Intrinsic::memcpy: 638 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/); 639 } 640 } 641 642 if (isa<LoadInst>(I) && I->hasOneUse()) { 643 auto *SingleUser = dyn_cast<Instruction>(*I->user_begin()); 644 if (SingleUser->getParent() == I->getParent()) { 645 if (isa<ICmpInst>(SingleUser)) { 646 if (auto *C = dyn_cast<ConstantInt>(SingleUser->getOperand(1))) 647 if (C->getBitWidth() <= 64 && 648 (isInt<16>(C->getSExtValue()) || isUInt<16>(C->getZExtValue()))) 649 // Comparison of memory with 16 bit signed / unsigned immediate 650 return AddressingMode(false/*LongDispl*/, false/*IdxReg*/); 651 } else if (isa<StoreInst>(SingleUser)) 652 // Load->Store 653 return getLoadStoreAddrMode(HasVector, I->getType()); 654 } 655 } else if (auto *StoreI = dyn_cast<StoreInst>(I)) { 656 if (auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand())) 657 if (LoadI->hasOneUse() && LoadI->getParent() == I->getParent()) 658 // Load->Store 659 return getLoadStoreAddrMode(HasVector, LoadI->getType()); 660 } 661 662 if (HasVector && (isa<LoadInst>(I) || isa<StoreInst>(I))) { 663 664 // * Use LDE instead of LE/LEY for z13 to avoid partial register 665 // dependencies (LDE only supports small offsets). 666 // * Utilize the vector registers to hold floating point 667 // values (vector load / store instructions only support small 668 // offsets). 669 670 Type *MemAccessTy = (isa<LoadInst>(I) ? I->getType() : 671 I->getOperand(0)->getType()); 672 bool IsFPAccess = MemAccessTy->isFloatingPointTy(); 673 bool IsVectorAccess = MemAccessTy->isVectorTy(); 674 675 // A store of an extracted vector element will be combined into a VSTE type 676 // instruction. 677 if (!IsVectorAccess && isa<StoreInst>(I)) { 678 Value *DataOp = I->getOperand(0); 679 if (isa<ExtractElementInst>(DataOp)) 680 IsVectorAccess = true; 681 } 682 683 // A load which gets inserted into a vector element will be combined into a 684 // VLE type instruction. 685 if (!IsVectorAccess && isa<LoadInst>(I) && I->hasOneUse()) { 686 User *LoadUser = *I->user_begin(); 687 if (isa<InsertElementInst>(LoadUser)) 688 IsVectorAccess = true; 689 } 690 691 if (IsFPAccess || IsVectorAccess) 692 return AddressingMode(false/*LongDispl*/, true/*IdxReg*/); 693 } 694 695 return AddressingMode(true/*LongDispl*/, true/*IdxReg*/); 696 } 697 698 bool SystemZTargetLowering::isLegalAddressingMode(const DataLayout &DL, 699 const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const { 700 // Punt on globals for now, although they can be used in limited 701 // RELATIVE LONG cases. 702 if (AM.BaseGV) 703 return false; 704 705 // Require a 20-bit signed offset. 706 if (!isInt<20>(AM.BaseOffs)) 707 return false; 708 709 AddressingMode SupportedAM(true, true); 710 if (I != nullptr) 711 SupportedAM = supportedAddressingMode(I, Subtarget.hasVector()); 712 713 if (!SupportedAM.LongDisplacement && !isUInt<12>(AM.BaseOffs)) 714 return false; 715 716 if (!SupportedAM.IndexReg) 717 // No indexing allowed. 718 return AM.Scale == 0; 719 else 720 // Indexing is OK but no scale factor can be applied. 721 return AM.Scale == 0 || AM.Scale == 1; 722 } 723 724 bool SystemZTargetLowering::isTruncateFree(Type *FromType, Type *ToType) const { 725 if (!FromType->isIntegerTy() || !ToType->isIntegerTy()) 726 return false; 727 unsigned FromBits = FromType->getPrimitiveSizeInBits(); 728 unsigned ToBits = ToType->getPrimitiveSizeInBits(); 729 return FromBits > ToBits; 730 } 731 732 bool SystemZTargetLowering::isTruncateFree(EVT FromVT, EVT ToVT) const { 733 if (!FromVT.isInteger() || !ToVT.isInteger()) 734 return false; 735 unsigned FromBits = FromVT.getSizeInBits(); 736 unsigned ToBits = ToVT.getSizeInBits(); 737 return FromBits > ToBits; 738 } 739 740 //===----------------------------------------------------------------------===// 741 // Inline asm support 742 //===----------------------------------------------------------------------===// 743 744 TargetLowering::ConstraintType 745 SystemZTargetLowering::getConstraintType(StringRef Constraint) const { 746 if (Constraint.size() == 1) { 747 switch (Constraint[0]) { 748 case 'a': // Address register 749 case 'd': // Data register (equivalent to 'r') 750 case 'f': // Floating-point register 751 case 'h': // High-part register 752 case 'r': // General-purpose register 753 case 'v': // Vector register 754 return C_RegisterClass; 755 756 case 'Q': // Memory with base and unsigned 12-bit displacement 757 case 'R': // Likewise, plus an index 758 case 'S': // Memory with base and signed 20-bit displacement 759 case 'T': // Likewise, plus an index 760 case 'm': // Equivalent to 'T'. 761 return C_Memory; 762 763 case 'I': // Unsigned 8-bit constant 764 case 'J': // Unsigned 12-bit constant 765 case 'K': // Signed 16-bit constant 766 case 'L': // Signed 20-bit displacement (on all targets we support) 767 case 'M': // 0x7fffffff 768 return C_Other; 769 770 default: 771 break; 772 } 773 } 774 return TargetLowering::getConstraintType(Constraint); 775 } 776 777 TargetLowering::ConstraintWeight SystemZTargetLowering:: 778 getSingleConstraintMatchWeight(AsmOperandInfo &info, 779 const char *constraint) const { 780 ConstraintWeight weight = CW_Invalid; 781 Value *CallOperandVal = info.CallOperandVal; 782 // If we don't have a value, we can't do a match, 783 // but allow it at the lowest weight. 784 if (!CallOperandVal) 785 return CW_Default; 786 Type *type = CallOperandVal->getType(); 787 // Look at the constraint type. 788 switch (*constraint) { 789 default: 790 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); 791 break; 792 793 case 'a': // Address register 794 case 'd': // Data register (equivalent to 'r') 795 case 'h': // High-part register 796 case 'r': // General-purpose register 797 if (CallOperandVal->getType()->isIntegerTy()) 798 weight = CW_Register; 799 break; 800 801 case 'f': // Floating-point register 802 if (type->isFloatingPointTy()) 803 weight = CW_Register; 804 break; 805 806 case 'v': // Vector register 807 if ((type->isVectorTy() || type->isFloatingPointTy()) && 808 Subtarget.hasVector()) 809 weight = CW_Register; 810 break; 811 812 case 'I': // Unsigned 8-bit constant 813 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 814 if (isUInt<8>(C->getZExtValue())) 815 weight = CW_Constant; 816 break; 817 818 case 'J': // Unsigned 12-bit constant 819 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 820 if (isUInt<12>(C->getZExtValue())) 821 weight = CW_Constant; 822 break; 823 824 case 'K': // Signed 16-bit constant 825 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 826 if (isInt<16>(C->getSExtValue())) 827 weight = CW_Constant; 828 break; 829 830 case 'L': // Signed 20-bit displacement (on all targets we support) 831 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 832 if (isInt<20>(C->getSExtValue())) 833 weight = CW_Constant; 834 break; 835 836 case 'M': // 0x7fffffff 837 if (auto *C = dyn_cast<ConstantInt>(CallOperandVal)) 838 if (C->getZExtValue() == 0x7fffffff) 839 weight = CW_Constant; 840 break; 841 } 842 return weight; 843 } 844 845 // Parse a "{tNNN}" register constraint for which the register type "t" 846 // has already been verified. MC is the class associated with "t" and 847 // Map maps 0-based register numbers to LLVM register numbers. 848 static std::pair<unsigned, const TargetRegisterClass *> 849 parseRegisterNumber(StringRef Constraint, const TargetRegisterClass *RC, 850 const unsigned *Map, unsigned Size) { 851 assert(*(Constraint.end()-1) == '}' && "Missing '}'"); 852 if (isdigit(Constraint[2])) { 853 unsigned Index; 854 bool Failed = 855 Constraint.slice(2, Constraint.size() - 1).getAsInteger(10, Index); 856 if (!Failed && Index < Size && Map[Index]) 857 return std::make_pair(Map[Index], RC); 858 } 859 return std::make_pair(0U, nullptr); 860 } 861 862 std::pair<unsigned, const TargetRegisterClass *> 863 SystemZTargetLowering::getRegForInlineAsmConstraint( 864 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { 865 if (Constraint.size() == 1) { 866 // GCC Constraint Letters 867 switch (Constraint[0]) { 868 default: break; 869 case 'd': // Data register (equivalent to 'r') 870 case 'r': // General-purpose register 871 if (VT == MVT::i64) 872 return std::make_pair(0U, &SystemZ::GR64BitRegClass); 873 else if (VT == MVT::i128) 874 return std::make_pair(0U, &SystemZ::GR128BitRegClass); 875 return std::make_pair(0U, &SystemZ::GR32BitRegClass); 876 877 case 'a': // Address register 878 if (VT == MVT::i64) 879 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass); 880 else if (VT == MVT::i128) 881 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass); 882 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass); 883 884 case 'h': // High-part register (an LLVM extension) 885 return std::make_pair(0U, &SystemZ::GRH32BitRegClass); 886 887 case 'f': // Floating-point register 888 if (VT == MVT::f64) 889 return std::make_pair(0U, &SystemZ::FP64BitRegClass); 890 else if (VT == MVT::f128) 891 return std::make_pair(0U, &SystemZ::FP128BitRegClass); 892 return std::make_pair(0U, &SystemZ::FP32BitRegClass); 893 894 case 'v': // Vector register 895 if (Subtarget.hasVector()) { 896 if (VT == MVT::f32) 897 return std::make_pair(0U, &SystemZ::VR32BitRegClass); 898 if (VT == MVT::f64) 899 return std::make_pair(0U, &SystemZ::VR64BitRegClass); 900 return std::make_pair(0U, &SystemZ::VR128BitRegClass); 901 } 902 break; 903 } 904 } 905 if (Constraint.size() > 0 && Constraint[0] == '{') { 906 // We need to override the default register parsing for GPRs and FPRs 907 // because the interpretation depends on VT. The internal names of 908 // the registers are also different from the external names 909 // (F0D and F0S instead of F0, etc.). 910 if (Constraint[1] == 'r') { 911 if (VT == MVT::i32) 912 return parseRegisterNumber(Constraint, &SystemZ::GR32BitRegClass, 913 SystemZMC::GR32Regs, 16); 914 if (VT == MVT::i128) 915 return parseRegisterNumber(Constraint, &SystemZ::GR128BitRegClass, 916 SystemZMC::GR128Regs, 16); 917 return parseRegisterNumber(Constraint, &SystemZ::GR64BitRegClass, 918 SystemZMC::GR64Regs, 16); 919 } 920 if (Constraint[1] == 'f') { 921 if (VT == MVT::f32) 922 return parseRegisterNumber(Constraint, &SystemZ::FP32BitRegClass, 923 SystemZMC::FP32Regs, 16); 924 if (VT == MVT::f128) 925 return parseRegisterNumber(Constraint, &SystemZ::FP128BitRegClass, 926 SystemZMC::FP128Regs, 16); 927 return parseRegisterNumber(Constraint, &SystemZ::FP64BitRegClass, 928 SystemZMC::FP64Regs, 16); 929 } 930 if (Constraint[1] == 'v') { 931 if (VT == MVT::f32) 932 return parseRegisterNumber(Constraint, &SystemZ::VR32BitRegClass, 933 SystemZMC::VR32Regs, 32); 934 if (VT == MVT::f64) 935 return parseRegisterNumber(Constraint, &SystemZ::VR64BitRegClass, 936 SystemZMC::VR64Regs, 32); 937 return parseRegisterNumber(Constraint, &SystemZ::VR128BitRegClass, 938 SystemZMC::VR128Regs, 32); 939 } 940 } 941 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 942 } 943 944 void SystemZTargetLowering:: 945 LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, 946 std::vector<SDValue> &Ops, 947 SelectionDAG &DAG) const { 948 // Only support length 1 constraints for now. 949 if (Constraint.length() == 1) { 950 switch (Constraint[0]) { 951 case 'I': // Unsigned 8-bit constant 952 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 953 if (isUInt<8>(C->getZExtValue())) 954 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), 955 Op.getValueType())); 956 return; 957 958 case 'J': // Unsigned 12-bit constant 959 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 960 if (isUInt<12>(C->getZExtValue())) 961 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), 962 Op.getValueType())); 963 return; 964 965 case 'K': // Signed 16-bit constant 966 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 967 if (isInt<16>(C->getSExtValue())) 968 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), 969 Op.getValueType())); 970 return; 971 972 case 'L': // Signed 20-bit displacement (on all targets we support) 973 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 974 if (isInt<20>(C->getSExtValue())) 975 Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), 976 Op.getValueType())); 977 return; 978 979 case 'M': // 0x7fffffff 980 if (auto *C = dyn_cast<ConstantSDNode>(Op)) 981 if (C->getZExtValue() == 0x7fffffff) 982 Ops.push_back(DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op), 983 Op.getValueType())); 984 return; 985 } 986 } 987 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 988 } 989 990 //===----------------------------------------------------------------------===// 991 // Calling conventions 992 //===----------------------------------------------------------------------===// 993 994 #include "SystemZGenCallingConv.inc" 995 996 const MCPhysReg *SystemZTargetLowering::getScratchRegisters( 997 CallingConv::ID) const { 998 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D, 999 SystemZ::R14D, 0 }; 1000 return ScratchRegs; 1001 } 1002 1003 bool SystemZTargetLowering::allowTruncateForTailCall(Type *FromType, 1004 Type *ToType) const { 1005 return isTruncateFree(FromType, ToType); 1006 } 1007 1008 bool SystemZTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { 1009 return CI->isTailCall(); 1010 } 1011 1012 // We do not yet support 128-bit single-element vector types. If the user 1013 // attempts to use such types as function argument or return type, prefer 1014 // to error out instead of emitting code violating the ABI. 1015 static void VerifyVectorType(MVT VT, EVT ArgVT) { 1016 if (ArgVT.isVector() && !VT.isVector()) 1017 report_fatal_error("Unsupported vector argument or return type"); 1018 } 1019 1020 static void VerifyVectorTypes(const SmallVectorImpl<ISD::InputArg> &Ins) { 1021 for (unsigned i = 0; i < Ins.size(); ++i) 1022 VerifyVectorType(Ins[i].VT, Ins[i].ArgVT); 1023 } 1024 1025 static void VerifyVectorTypes(const SmallVectorImpl<ISD::OutputArg> &Outs) { 1026 for (unsigned i = 0; i < Outs.size(); ++i) 1027 VerifyVectorType(Outs[i].VT, Outs[i].ArgVT); 1028 } 1029 1030 // Value is a value that has been passed to us in the location described by VA 1031 // (and so has type VA.getLocVT()). Convert Value to VA.getValVT(), chaining 1032 // any loads onto Chain. 1033 static SDValue convertLocVTToValVT(SelectionDAG &DAG, const SDLoc &DL, 1034 CCValAssign &VA, SDValue Chain, 1035 SDValue Value) { 1036 // If the argument has been promoted from a smaller type, insert an 1037 // assertion to capture this. 1038 if (VA.getLocInfo() == CCValAssign::SExt) 1039 Value = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Value, 1040 DAG.getValueType(VA.getValVT())); 1041 else if (VA.getLocInfo() == CCValAssign::ZExt) 1042 Value = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Value, 1043 DAG.getValueType(VA.getValVT())); 1044 1045 if (VA.isExtInLoc()) 1046 Value = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Value); 1047 else if (VA.getLocInfo() == CCValAssign::BCvt) { 1048 // If this is a short vector argument loaded from the stack, 1049 // extend from i64 to full vector size and then bitcast. 1050 assert(VA.getLocVT() == MVT::i64); 1051 assert(VA.getValVT().isVector()); 1052 Value = DAG.getBuildVector(MVT::v2i64, DL, {Value, DAG.getUNDEF(MVT::i64)}); 1053 Value = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Value); 1054 } else 1055 assert(VA.getLocInfo() == CCValAssign::Full && "Unsupported getLocInfo"); 1056 return Value; 1057 } 1058 1059 // Value is a value of type VA.getValVT() that we need to copy into 1060 // the location described by VA. Return a copy of Value converted to 1061 // VA.getValVT(). The caller is responsible for handling indirect values. 1062 static SDValue convertValVTToLocVT(SelectionDAG &DAG, const SDLoc &DL, 1063 CCValAssign &VA, SDValue Value) { 1064 switch (VA.getLocInfo()) { 1065 case CCValAssign::SExt: 1066 return DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Value); 1067 case CCValAssign::ZExt: 1068 return DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Value); 1069 case CCValAssign::AExt: 1070 return DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Value); 1071 case CCValAssign::BCvt: 1072 // If this is a short vector argument to be stored to the stack, 1073 // bitcast to v2i64 and then extract first element. 1074 assert(VA.getLocVT() == MVT::i64); 1075 assert(VA.getValVT().isVector()); 1076 Value = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Value); 1077 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VA.getLocVT(), Value, 1078 DAG.getConstant(0, DL, MVT::i32)); 1079 case CCValAssign::Full: 1080 return Value; 1081 default: 1082 llvm_unreachable("Unhandled getLocInfo()"); 1083 } 1084 } 1085 1086 SDValue SystemZTargetLowering::LowerFormalArguments( 1087 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, 1088 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, 1089 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 1090 MachineFunction &MF = DAG.getMachineFunction(); 1091 MachineFrameInfo &MFI = MF.getFrameInfo(); 1092 MachineRegisterInfo &MRI = MF.getRegInfo(); 1093 SystemZMachineFunctionInfo *FuncInfo = 1094 MF.getInfo<SystemZMachineFunctionInfo>(); 1095 auto *TFL = 1096 static_cast<const SystemZFrameLowering *>(Subtarget.getFrameLowering()); 1097 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 1098 1099 // Detect unsupported vector argument types. 1100 if (Subtarget.hasVector()) 1101 VerifyVectorTypes(Ins); 1102 1103 // Assign locations to all of the incoming arguments. 1104 SmallVector<CCValAssign, 16> ArgLocs; 1105 SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 1106 CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ); 1107 1108 unsigned NumFixedGPRs = 0; 1109 unsigned NumFixedFPRs = 0; 1110 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 1111 SDValue ArgValue; 1112 CCValAssign &VA = ArgLocs[I]; 1113 EVT LocVT = VA.getLocVT(); 1114 if (VA.isRegLoc()) { 1115 // Arguments passed in registers 1116 const TargetRegisterClass *RC; 1117 switch (LocVT.getSimpleVT().SimpleTy) { 1118 default: 1119 // Integers smaller than i64 should be promoted to i64. 1120 llvm_unreachable("Unexpected argument type"); 1121 case MVT::i32: 1122 NumFixedGPRs += 1; 1123 RC = &SystemZ::GR32BitRegClass; 1124 break; 1125 case MVT::i64: 1126 NumFixedGPRs += 1; 1127 RC = &SystemZ::GR64BitRegClass; 1128 break; 1129 case MVT::f32: 1130 NumFixedFPRs += 1; 1131 RC = &SystemZ::FP32BitRegClass; 1132 break; 1133 case MVT::f64: 1134 NumFixedFPRs += 1; 1135 RC = &SystemZ::FP64BitRegClass; 1136 break; 1137 case MVT::v16i8: 1138 case MVT::v8i16: 1139 case MVT::v4i32: 1140 case MVT::v2i64: 1141 case MVT::v4f32: 1142 case MVT::v2f64: 1143 RC = &SystemZ::VR128BitRegClass; 1144 break; 1145 } 1146 1147 unsigned VReg = MRI.createVirtualRegister(RC); 1148 MRI.addLiveIn(VA.getLocReg(), VReg); 1149 ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, LocVT); 1150 } else { 1151 assert(VA.isMemLoc() && "Argument not register or memory"); 1152 1153 // Create the frame index object for this incoming parameter. 1154 int FI = MFI.CreateFixedObject(LocVT.getSizeInBits() / 8, 1155 VA.getLocMemOffset(), true); 1156 1157 // Create the SelectionDAG nodes corresponding to a load 1158 // from this parameter. Unpromoted ints and floats are 1159 // passed as right-justified 8-byte values. 1160 SDValue FIN = DAG.getFrameIndex(FI, PtrVT); 1161 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 1162 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, 1163 DAG.getIntPtrConstant(4, DL)); 1164 ArgValue = DAG.getLoad(LocVT, DL, Chain, FIN, 1165 MachinePointerInfo::getFixedStack(MF, FI)); 1166 } 1167 1168 // Convert the value of the argument register into the value that's 1169 // being passed. 1170 if (VA.getLocInfo() == CCValAssign::Indirect) { 1171 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue, 1172 MachinePointerInfo())); 1173 // If the original argument was split (e.g. i128), we need 1174 // to load all parts of it here (using the same address). 1175 unsigned ArgIndex = Ins[I].OrigArgIndex; 1176 assert (Ins[I].PartOffset == 0); 1177 while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) { 1178 CCValAssign &PartVA = ArgLocs[I + 1]; 1179 unsigned PartOffset = Ins[I + 1].PartOffset; 1180 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, 1181 DAG.getIntPtrConstant(PartOffset, DL)); 1182 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, 1183 MachinePointerInfo())); 1184 ++I; 1185 } 1186 } else 1187 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); 1188 } 1189 1190 if (IsVarArg) { 1191 // Save the number of non-varargs registers for later use by va_start, etc. 1192 FuncInfo->setVarArgsFirstGPR(NumFixedGPRs); 1193 FuncInfo->setVarArgsFirstFPR(NumFixedFPRs); 1194 1195 // Likewise the address (in the form of a frame index) of where the 1196 // first stack vararg would be. The 1-byte size here is arbitrary. 1197 int64_t StackSize = CCInfo.getNextStackOffset(); 1198 FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true)); 1199 1200 // ...and a similar frame index for the caller-allocated save area 1201 // that will be used to store the incoming registers. 1202 int64_t RegSaveOffset = TFL->getOffsetOfLocalArea(); 1203 unsigned RegSaveIndex = MFI.CreateFixedObject(1, RegSaveOffset, true); 1204 FuncInfo->setRegSaveFrameIndex(RegSaveIndex); 1205 1206 // Store the FPR varargs in the reserved frame slots. (We store the 1207 // GPRs as part of the prologue.) 1208 if (NumFixedFPRs < SystemZ::NumArgFPRs) { 1209 SDValue MemOps[SystemZ::NumArgFPRs]; 1210 for (unsigned I = NumFixedFPRs; I < SystemZ::NumArgFPRs; ++I) { 1211 unsigned Offset = TFL->getRegSpillOffset(SystemZ::ArgFPRs[I]); 1212 int FI = MFI.CreateFixedObject(8, RegSaveOffset + Offset, true); 1213 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); 1214 unsigned VReg = MF.addLiveIn(SystemZ::ArgFPRs[I], 1215 &SystemZ::FP64BitRegClass); 1216 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f64); 1217 MemOps[I] = DAG.getStore(ArgValue.getValue(1), DL, ArgValue, FIN, 1218 MachinePointerInfo::getFixedStack(MF, FI)); 1219 } 1220 // Join the stores, which are independent of one another. 1221 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 1222 makeArrayRef(&MemOps[NumFixedFPRs], 1223 SystemZ::NumArgFPRs-NumFixedFPRs)); 1224 } 1225 } 1226 1227 return Chain; 1228 } 1229 1230 static bool canUseSiblingCall(const CCState &ArgCCInfo, 1231 SmallVectorImpl<CCValAssign> &ArgLocs, 1232 SmallVectorImpl<ISD::OutputArg> &Outs) { 1233 // Punt if there are any indirect or stack arguments, or if the call 1234 // needs the callee-saved argument register R6, or if the call uses 1235 // the callee-saved register arguments SwiftSelf and SwiftError. 1236 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 1237 CCValAssign &VA = ArgLocs[I]; 1238 if (VA.getLocInfo() == CCValAssign::Indirect) 1239 return false; 1240 if (!VA.isRegLoc()) 1241 return false; 1242 unsigned Reg = VA.getLocReg(); 1243 if (Reg == SystemZ::R6H || Reg == SystemZ::R6L || Reg == SystemZ::R6D) 1244 return false; 1245 if (Outs[I].Flags.isSwiftSelf() || Outs[I].Flags.isSwiftError()) 1246 return false; 1247 } 1248 return true; 1249 } 1250 1251 SDValue 1252 SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, 1253 SmallVectorImpl<SDValue> &InVals) const { 1254 SelectionDAG &DAG = CLI.DAG; 1255 SDLoc &DL = CLI.DL; 1256 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 1257 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 1258 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 1259 SDValue Chain = CLI.Chain; 1260 SDValue Callee = CLI.Callee; 1261 bool &IsTailCall = CLI.IsTailCall; 1262 CallingConv::ID CallConv = CLI.CallConv; 1263 bool IsVarArg = CLI.IsVarArg; 1264 MachineFunction &MF = DAG.getMachineFunction(); 1265 EVT PtrVT = getPointerTy(MF.getDataLayout()); 1266 1267 // Detect unsupported vector argument and return types. 1268 if (Subtarget.hasVector()) { 1269 VerifyVectorTypes(Outs); 1270 VerifyVectorTypes(Ins); 1271 } 1272 1273 // Analyze the operands of the call, assigning locations to each operand. 1274 SmallVector<CCValAssign, 16> ArgLocs; 1275 SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); 1276 ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ); 1277 1278 // We don't support GuaranteedTailCallOpt, only automatically-detected 1279 // sibling calls. 1280 if (IsTailCall && !canUseSiblingCall(ArgCCInfo, ArgLocs, Outs)) 1281 IsTailCall = false; 1282 1283 // Get a count of how many bytes are to be pushed on the stack. 1284 unsigned NumBytes = ArgCCInfo.getNextStackOffset(); 1285 1286 // Mark the start of the call. 1287 if (!IsTailCall) 1288 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL); 1289 1290 // Copy argument values to their designated locations. 1291 SmallVector<std::pair<unsigned, SDValue>, 9> RegsToPass; 1292 SmallVector<SDValue, 8> MemOpChains; 1293 SDValue StackPtr; 1294 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) { 1295 CCValAssign &VA = ArgLocs[I]; 1296 SDValue ArgValue = OutVals[I]; 1297 1298 if (VA.getLocInfo() == CCValAssign::Indirect) { 1299 // Store the argument in a stack slot and pass its address. 1300 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[I].ArgVT); 1301 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); 1302 MemOpChains.push_back( 1303 DAG.getStore(Chain, DL, ArgValue, SpillSlot, 1304 MachinePointerInfo::getFixedStack(MF, FI))); 1305 // If the original argument was split (e.g. i128), we need 1306 // to store all parts of it here (and pass just one address). 1307 unsigned ArgIndex = Outs[I].OrigArgIndex; 1308 assert (Outs[I].PartOffset == 0); 1309 while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) { 1310 SDValue PartValue = OutVals[I + 1]; 1311 unsigned PartOffset = Outs[I + 1].PartOffset; 1312 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, 1313 DAG.getIntPtrConstant(PartOffset, DL)); 1314 MemOpChains.push_back( 1315 DAG.getStore(Chain, DL, PartValue, Address, 1316 MachinePointerInfo::getFixedStack(MF, FI))); 1317 ++I; 1318 } 1319 ArgValue = SpillSlot; 1320 } else 1321 ArgValue = convertValVTToLocVT(DAG, DL, VA, ArgValue); 1322 1323 if (VA.isRegLoc()) 1324 // Queue up the argument copies and emit them at the end. 1325 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue)); 1326 else { 1327 assert(VA.isMemLoc() && "Argument not register or memory"); 1328 1329 // Work out the address of the stack slot. Unpromoted ints and 1330 // floats are passed as right-justified 8-byte values. 1331 if (!StackPtr.getNode()) 1332 StackPtr = DAG.getCopyFromReg(Chain, DL, SystemZ::R15D, PtrVT); 1333 unsigned Offset = SystemZMC::CallFrameSize + VA.getLocMemOffset(); 1334 if (VA.getLocVT() == MVT::i32 || VA.getLocVT() == MVT::f32) 1335 Offset += 4; 1336 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, 1337 DAG.getIntPtrConstant(Offset, DL)); 1338 1339 // Emit the store. 1340 MemOpChains.push_back( 1341 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo())); 1342 } 1343 } 1344 1345 // Join the stores, which are independent of one another. 1346 if (!MemOpChains.empty()) 1347 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); 1348 1349 // Accept direct calls by converting symbolic call addresses to the 1350 // associated Target* opcodes. Force %r1 to be used for indirect 1351 // tail calls. 1352 SDValue Glue; 1353 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) { 1354 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), DL, PtrVT); 1355 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 1356 } else if (auto *E = dyn_cast<ExternalSymbolSDNode>(Callee)) { 1357 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT); 1358 Callee = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Callee); 1359 } else if (IsTailCall) { 1360 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R1D, Callee, Glue); 1361 Glue = Chain.getValue(1); 1362 Callee = DAG.getRegister(SystemZ::R1D, Callee.getValueType()); 1363 } 1364 1365 // Build a sequence of copy-to-reg nodes, chained and glued together. 1366 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) { 1367 Chain = DAG.getCopyToReg(Chain, DL, RegsToPass[I].first, 1368 RegsToPass[I].second, Glue); 1369 Glue = Chain.getValue(1); 1370 } 1371 1372 // The first call operand is the chain and the second is the target address. 1373 SmallVector<SDValue, 8> Ops; 1374 Ops.push_back(Chain); 1375 Ops.push_back(Callee); 1376 1377 // Add argument registers to the end of the list so that they are 1378 // known live into the call. 1379 for (unsigned I = 0, E = RegsToPass.size(); I != E; ++I) 1380 Ops.push_back(DAG.getRegister(RegsToPass[I].first, 1381 RegsToPass[I].second.getValueType())); 1382 1383 // Add a register mask operand representing the call-preserved registers. 1384 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 1385 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv); 1386 assert(Mask && "Missing call preserved mask for calling convention"); 1387 Ops.push_back(DAG.getRegisterMask(Mask)); 1388 1389 // Glue the call to the argument copies, if any. 1390 if (Glue.getNode()) 1391 Ops.push_back(Glue); 1392 1393 // Emit the call. 1394 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 1395 if (IsTailCall) 1396 return DAG.getNode(SystemZISD::SIBCALL, DL, NodeTys, Ops); 1397 Chain = DAG.getNode(SystemZISD::CALL, DL, NodeTys, Ops); 1398 Glue = Chain.getValue(1); 1399 1400 // Mark the end of the call, which is glued to the call itself. 1401 Chain = DAG.getCALLSEQ_END(Chain, 1402 DAG.getConstant(NumBytes, DL, PtrVT, true), 1403 DAG.getConstant(0, DL, PtrVT, true), 1404 Glue, DL); 1405 Glue = Chain.getValue(1); 1406 1407 // Assign locations to each value returned by this call. 1408 SmallVector<CCValAssign, 16> RetLocs; 1409 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); 1410 RetCCInfo.AnalyzeCallResult(Ins, RetCC_SystemZ); 1411 1412 // Copy all of the result registers out of their specified physreg. 1413 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 1414 CCValAssign &VA = RetLocs[I]; 1415 1416 // Copy the value out, gluing the copy to the end of the call sequence. 1417 SDValue RetValue = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), 1418 VA.getLocVT(), Glue); 1419 Chain = RetValue.getValue(1); 1420 Glue = RetValue.getValue(2); 1421 1422 // Convert the value of the return register into the value that's 1423 // being returned. 1424 InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, RetValue)); 1425 } 1426 1427 return Chain; 1428 } 1429 1430 bool SystemZTargetLowering:: 1431 CanLowerReturn(CallingConv::ID CallConv, 1432 MachineFunction &MF, bool isVarArg, 1433 const SmallVectorImpl<ISD::OutputArg> &Outs, 1434 LLVMContext &Context) const { 1435 // Detect unsupported vector return types. 1436 if (Subtarget.hasVector()) 1437 VerifyVectorTypes(Outs); 1438 1439 // Special case that we cannot easily detect in RetCC_SystemZ since 1440 // i128 is not a legal type. 1441 for (auto &Out : Outs) 1442 if (Out.ArgVT == MVT::i128) 1443 return false; 1444 1445 SmallVector<CCValAssign, 16> RetLocs; 1446 CCState RetCCInfo(CallConv, isVarArg, MF, RetLocs, Context); 1447 return RetCCInfo.CheckReturn(Outs, RetCC_SystemZ); 1448 } 1449 1450 SDValue 1451 SystemZTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 1452 bool IsVarArg, 1453 const SmallVectorImpl<ISD::OutputArg> &Outs, 1454 const SmallVectorImpl<SDValue> &OutVals, 1455 const SDLoc &DL, SelectionDAG &DAG) const { 1456 MachineFunction &MF = DAG.getMachineFunction(); 1457 1458 // Detect unsupported vector return types. 1459 if (Subtarget.hasVector()) 1460 VerifyVectorTypes(Outs); 1461 1462 // Assign locations to each returned value. 1463 SmallVector<CCValAssign, 16> RetLocs; 1464 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, *DAG.getContext()); 1465 RetCCInfo.AnalyzeReturn(Outs, RetCC_SystemZ); 1466 1467 // Quick exit for void returns 1468 if (RetLocs.empty()) 1469 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, Chain); 1470 1471 // Copy the result values into the output registers. 1472 SDValue Glue; 1473 SmallVector<SDValue, 4> RetOps; 1474 RetOps.push_back(Chain); 1475 for (unsigned I = 0, E = RetLocs.size(); I != E; ++I) { 1476 CCValAssign &VA = RetLocs[I]; 1477 SDValue RetValue = OutVals[I]; 1478 1479 // Make the return register live on exit. 1480 assert(VA.isRegLoc() && "Can only return in registers!"); 1481 1482 // Promote the value as required. 1483 RetValue = convertValVTToLocVT(DAG, DL, VA, RetValue); 1484 1485 // Chain and glue the copies together. 1486 unsigned Reg = VA.getLocReg(); 1487 Chain = DAG.getCopyToReg(Chain, DL, Reg, RetValue, Glue); 1488 Glue = Chain.getValue(1); 1489 RetOps.push_back(DAG.getRegister(Reg, VA.getLocVT())); 1490 } 1491 1492 // Update chain and glue. 1493 RetOps[0] = Chain; 1494 if (Glue.getNode()) 1495 RetOps.push_back(Glue); 1496 1497 return DAG.getNode(SystemZISD::RET_FLAG, DL, MVT::Other, RetOps); 1498 } 1499 1500 // Return true if Op is an intrinsic node with chain that returns the CC value 1501 // as its only (other) argument. Provide the associated SystemZISD opcode and 1502 // the mask of valid CC values if so. 1503 static bool isIntrinsicWithCCAndChain(SDValue Op, unsigned &Opcode, 1504 unsigned &CCValid) { 1505 unsigned Id = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 1506 switch (Id) { 1507 case Intrinsic::s390_tbegin: 1508 Opcode = SystemZISD::TBEGIN; 1509 CCValid = SystemZ::CCMASK_TBEGIN; 1510 return true; 1511 1512 case Intrinsic::s390_tbegin_nofloat: 1513 Opcode = SystemZISD::TBEGIN_NOFLOAT; 1514 CCValid = SystemZ::CCMASK_TBEGIN; 1515 return true; 1516 1517 case Intrinsic::s390_tend: 1518 Opcode = SystemZISD::TEND; 1519 CCValid = SystemZ::CCMASK_TEND; 1520 return true; 1521 1522 default: 1523 return false; 1524 } 1525 } 1526 1527 // Return true if Op is an intrinsic node without chain that returns the 1528 // CC value as its final argument. Provide the associated SystemZISD 1529 // opcode and the mask of valid CC values if so. 1530 static bool isIntrinsicWithCC(SDValue Op, unsigned &Opcode, unsigned &CCValid) { 1531 unsigned Id = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 1532 switch (Id) { 1533 case Intrinsic::s390_vpkshs: 1534 case Intrinsic::s390_vpksfs: 1535 case Intrinsic::s390_vpksgs: 1536 Opcode = SystemZISD::PACKS_CC; 1537 CCValid = SystemZ::CCMASK_VCMP; 1538 return true; 1539 1540 case Intrinsic::s390_vpklshs: 1541 case Intrinsic::s390_vpklsfs: 1542 case Intrinsic::s390_vpklsgs: 1543 Opcode = SystemZISD::PACKLS_CC; 1544 CCValid = SystemZ::CCMASK_VCMP; 1545 return true; 1546 1547 case Intrinsic::s390_vceqbs: 1548 case Intrinsic::s390_vceqhs: 1549 case Intrinsic::s390_vceqfs: 1550 case Intrinsic::s390_vceqgs: 1551 Opcode = SystemZISD::VICMPES; 1552 CCValid = SystemZ::CCMASK_VCMP; 1553 return true; 1554 1555 case Intrinsic::s390_vchbs: 1556 case Intrinsic::s390_vchhs: 1557 case Intrinsic::s390_vchfs: 1558 case Intrinsic::s390_vchgs: 1559 Opcode = SystemZISD::VICMPHS; 1560 CCValid = SystemZ::CCMASK_VCMP; 1561 return true; 1562 1563 case Intrinsic::s390_vchlbs: 1564 case Intrinsic::s390_vchlhs: 1565 case Intrinsic::s390_vchlfs: 1566 case Intrinsic::s390_vchlgs: 1567 Opcode = SystemZISD::VICMPHLS; 1568 CCValid = SystemZ::CCMASK_VCMP; 1569 return true; 1570 1571 case Intrinsic::s390_vtm: 1572 Opcode = SystemZISD::VTM; 1573 CCValid = SystemZ::CCMASK_VCMP; 1574 return true; 1575 1576 case Intrinsic::s390_vfaebs: 1577 case Intrinsic::s390_vfaehs: 1578 case Intrinsic::s390_vfaefs: 1579 Opcode = SystemZISD::VFAE_CC; 1580 CCValid = SystemZ::CCMASK_ANY; 1581 return true; 1582 1583 case Intrinsic::s390_vfaezbs: 1584 case Intrinsic::s390_vfaezhs: 1585 case Intrinsic::s390_vfaezfs: 1586 Opcode = SystemZISD::VFAEZ_CC; 1587 CCValid = SystemZ::CCMASK_ANY; 1588 return true; 1589 1590 case Intrinsic::s390_vfeebs: 1591 case Intrinsic::s390_vfeehs: 1592 case Intrinsic::s390_vfeefs: 1593 Opcode = SystemZISD::VFEE_CC; 1594 CCValid = SystemZ::CCMASK_ANY; 1595 return true; 1596 1597 case Intrinsic::s390_vfeezbs: 1598 case Intrinsic::s390_vfeezhs: 1599 case Intrinsic::s390_vfeezfs: 1600 Opcode = SystemZISD::VFEEZ_CC; 1601 CCValid = SystemZ::CCMASK_ANY; 1602 return true; 1603 1604 case Intrinsic::s390_vfenebs: 1605 case Intrinsic::s390_vfenehs: 1606 case Intrinsic::s390_vfenefs: 1607 Opcode = SystemZISD::VFENE_CC; 1608 CCValid = SystemZ::CCMASK_ANY; 1609 return true; 1610 1611 case Intrinsic::s390_vfenezbs: 1612 case Intrinsic::s390_vfenezhs: 1613 case Intrinsic::s390_vfenezfs: 1614 Opcode = SystemZISD::VFENEZ_CC; 1615 CCValid = SystemZ::CCMASK_ANY; 1616 return true; 1617 1618 case Intrinsic::s390_vistrbs: 1619 case Intrinsic::s390_vistrhs: 1620 case Intrinsic::s390_vistrfs: 1621 Opcode = SystemZISD::VISTR_CC; 1622 CCValid = SystemZ::CCMASK_0 | SystemZ::CCMASK_3; 1623 return true; 1624 1625 case Intrinsic::s390_vstrcbs: 1626 case Intrinsic::s390_vstrchs: 1627 case Intrinsic::s390_vstrcfs: 1628 Opcode = SystemZISD::VSTRC_CC; 1629 CCValid = SystemZ::CCMASK_ANY; 1630 return true; 1631 1632 case Intrinsic::s390_vstrczbs: 1633 case Intrinsic::s390_vstrczhs: 1634 case Intrinsic::s390_vstrczfs: 1635 Opcode = SystemZISD::VSTRCZ_CC; 1636 CCValid = SystemZ::CCMASK_ANY; 1637 return true; 1638 1639 case Intrinsic::s390_vfcedbs: 1640 case Intrinsic::s390_vfcesbs: 1641 Opcode = SystemZISD::VFCMPES; 1642 CCValid = SystemZ::CCMASK_VCMP; 1643 return true; 1644 1645 case Intrinsic::s390_vfchdbs: 1646 case Intrinsic::s390_vfchsbs: 1647 Opcode = SystemZISD::VFCMPHS; 1648 CCValid = SystemZ::CCMASK_VCMP; 1649 return true; 1650 1651 case Intrinsic::s390_vfchedbs: 1652 case Intrinsic::s390_vfchesbs: 1653 Opcode = SystemZISD::VFCMPHES; 1654 CCValid = SystemZ::CCMASK_VCMP; 1655 return true; 1656 1657 case Intrinsic::s390_vftcidb: 1658 case Intrinsic::s390_vftcisb: 1659 Opcode = SystemZISD::VFTCI; 1660 CCValid = SystemZ::CCMASK_VCMP; 1661 return true; 1662 1663 case Intrinsic::s390_tdc: 1664 Opcode = SystemZISD::TDC; 1665 CCValid = SystemZ::CCMASK_TDC; 1666 return true; 1667 1668 default: 1669 return false; 1670 } 1671 } 1672 1673 // Emit an intrinsic with chain and an explicit CC register result. 1674 static SDNode *emitIntrinsicWithCCAndChain(SelectionDAG &DAG, SDValue Op, 1675 unsigned Opcode) { 1676 // Copy all operands except the intrinsic ID. 1677 unsigned NumOps = Op.getNumOperands(); 1678 SmallVector<SDValue, 6> Ops; 1679 Ops.reserve(NumOps - 1); 1680 Ops.push_back(Op.getOperand(0)); 1681 for (unsigned I = 2; I < NumOps; ++I) 1682 Ops.push_back(Op.getOperand(I)); 1683 1684 assert(Op->getNumValues() == 2 && "Expected only CC result and chain"); 1685 SDVTList RawVTs = DAG.getVTList(MVT::i32, MVT::Other); 1686 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), RawVTs, Ops); 1687 SDValue OldChain = SDValue(Op.getNode(), 1); 1688 SDValue NewChain = SDValue(Intr.getNode(), 1); 1689 DAG.ReplaceAllUsesOfValueWith(OldChain, NewChain); 1690 return Intr.getNode(); 1691 } 1692 1693 // Emit an intrinsic with an explicit CC register result. 1694 static SDNode *emitIntrinsicWithCC(SelectionDAG &DAG, SDValue Op, 1695 unsigned Opcode) { 1696 // Copy all operands except the intrinsic ID. 1697 unsigned NumOps = Op.getNumOperands(); 1698 SmallVector<SDValue, 6> Ops; 1699 Ops.reserve(NumOps - 1); 1700 for (unsigned I = 1; I < NumOps; ++I) 1701 Ops.push_back(Op.getOperand(I)); 1702 1703 SDValue Intr = DAG.getNode(Opcode, SDLoc(Op), Op->getVTList(), Ops); 1704 return Intr.getNode(); 1705 } 1706 1707 // CC is a comparison that will be implemented using an integer or 1708 // floating-point comparison. Return the condition code mask for 1709 // a branch on true. In the integer case, CCMASK_CMP_UO is set for 1710 // unsigned comparisons and clear for signed ones. In the floating-point 1711 // case, CCMASK_CMP_UO has its normal mask meaning (unordered). 1712 static unsigned CCMaskForCondCode(ISD::CondCode CC) { 1713 #define CONV(X) \ 1714 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \ 1715 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \ 1716 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X 1717 1718 switch (CC) { 1719 default: 1720 llvm_unreachable("Invalid integer condition!"); 1721 1722 CONV(EQ); 1723 CONV(NE); 1724 CONV(GT); 1725 CONV(GE); 1726 CONV(LT); 1727 CONV(LE); 1728 1729 case ISD::SETO: return SystemZ::CCMASK_CMP_O; 1730 case ISD::SETUO: return SystemZ::CCMASK_CMP_UO; 1731 } 1732 #undef CONV 1733 } 1734 1735 // If C can be converted to a comparison against zero, adjust the operands 1736 // as necessary. 1737 static void adjustZeroCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) { 1738 if (C.ICmpType == SystemZICMP::UnsignedOnly) 1739 return; 1740 1741 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1.getNode()); 1742 if (!ConstOp1) 1743 return; 1744 1745 int64_t Value = ConstOp1->getSExtValue(); 1746 if ((Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_GT) || 1747 (Value == -1 && C.CCMask == SystemZ::CCMASK_CMP_LE) || 1748 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_LT) || 1749 (Value == 1 && C.CCMask == SystemZ::CCMASK_CMP_GE)) { 1750 C.CCMask ^= SystemZ::CCMASK_CMP_EQ; 1751 C.Op1 = DAG.getConstant(0, DL, C.Op1.getValueType()); 1752 } 1753 } 1754 1755 // If a comparison described by C is suitable for CLI(Y), CHHSI or CLHHSI, 1756 // adjust the operands as necessary. 1757 static void adjustSubwordCmp(SelectionDAG &DAG, const SDLoc &DL, 1758 Comparison &C) { 1759 // For us to make any changes, it must a comparison between a single-use 1760 // load and a constant. 1761 if (!C.Op0.hasOneUse() || 1762 C.Op0.getOpcode() != ISD::LOAD || 1763 C.Op1.getOpcode() != ISD::Constant) 1764 return; 1765 1766 // We must have an 8- or 16-bit load. 1767 auto *Load = cast<LoadSDNode>(C.Op0); 1768 unsigned NumBits = Load->getMemoryVT().getStoreSizeInBits(); 1769 if (NumBits != 8 && NumBits != 16) 1770 return; 1771 1772 // The load must be an extending one and the constant must be within the 1773 // range of the unextended value. 1774 auto *ConstOp1 = cast<ConstantSDNode>(C.Op1); 1775 uint64_t Value = ConstOp1->getZExtValue(); 1776 uint64_t Mask = (1 << NumBits) - 1; 1777 if (Load->getExtensionType() == ISD::SEXTLOAD) { 1778 // Make sure that ConstOp1 is in range of C.Op0. 1779 int64_t SignedValue = ConstOp1->getSExtValue(); 1780 if (uint64_t(SignedValue) + (uint64_t(1) << (NumBits - 1)) > Mask) 1781 return; 1782 if (C.ICmpType != SystemZICMP::SignedOnly) { 1783 // Unsigned comparison between two sign-extended values is equivalent 1784 // to unsigned comparison between two zero-extended values. 1785 Value &= Mask; 1786 } else if (NumBits == 8) { 1787 // Try to treat the comparison as unsigned, so that we can use CLI. 1788 // Adjust CCMask and Value as necessary. 1789 if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_LT) 1790 // Test whether the high bit of the byte is set. 1791 Value = 127, C.CCMask = SystemZ::CCMASK_CMP_GT; 1792 else if (Value == 0 && C.CCMask == SystemZ::CCMASK_CMP_GE) 1793 // Test whether the high bit of the byte is clear. 1794 Value = 128, C.CCMask = SystemZ::CCMASK_CMP_LT; 1795 else 1796 // No instruction exists for this combination. 1797 return; 1798 C.ICmpType = SystemZICMP::UnsignedOnly; 1799 } 1800 } else if (Load->getExtensionType() == ISD::ZEXTLOAD) { 1801 if (Value > Mask) 1802 return; 1803 // If the constant is in range, we can use any comparison. 1804 C.ICmpType = SystemZICMP::Any; 1805 } else 1806 return; 1807 1808 // Make sure that the first operand is an i32 of the right extension type. 1809 ISD::LoadExtType ExtType = (C.ICmpType == SystemZICMP::SignedOnly ? 1810 ISD::SEXTLOAD : 1811 ISD::ZEXTLOAD); 1812 if (C.Op0.getValueType() != MVT::i32 || 1813 Load->getExtensionType() != ExtType) { 1814 C.Op0 = DAG.getExtLoad(ExtType, SDLoc(Load), MVT::i32, Load->getChain(), 1815 Load->getBasePtr(), Load->getPointerInfo(), 1816 Load->getMemoryVT(), Load->getAlignment(), 1817 Load->getMemOperand()->getFlags()); 1818 // Update the chain uses. 1819 DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), C.Op0.getValue(1)); 1820 } 1821 1822 // Make sure that the second operand is an i32 with the right value. 1823 if (C.Op1.getValueType() != MVT::i32 || 1824 Value != ConstOp1->getZExtValue()) 1825 C.Op1 = DAG.getConstant(Value, DL, MVT::i32); 1826 } 1827 1828 // Return true if Op is either an unextended load, or a load suitable 1829 // for integer register-memory comparisons of type ICmpType. 1830 static bool isNaturalMemoryOperand(SDValue Op, unsigned ICmpType) { 1831 auto *Load = dyn_cast<LoadSDNode>(Op.getNode()); 1832 if (Load) { 1833 // There are no instructions to compare a register with a memory byte. 1834 if (Load->getMemoryVT() == MVT::i8) 1835 return false; 1836 // Otherwise decide on extension type. 1837 switch (Load->getExtensionType()) { 1838 case ISD::NON_EXTLOAD: 1839 return true; 1840 case ISD::SEXTLOAD: 1841 return ICmpType != SystemZICMP::UnsignedOnly; 1842 case ISD::ZEXTLOAD: 1843 return ICmpType != SystemZICMP::SignedOnly; 1844 default: 1845 break; 1846 } 1847 } 1848 return false; 1849 } 1850 1851 // Return true if it is better to swap the operands of C. 1852 static bool shouldSwapCmpOperands(const Comparison &C) { 1853 // Leave f128 comparisons alone, since they have no memory forms. 1854 if (C.Op0.getValueType() == MVT::f128) 1855 return false; 1856 1857 // Always keep a floating-point constant second, since comparisons with 1858 // zero can use LOAD TEST and comparisons with other constants make a 1859 // natural memory operand. 1860 if (isa<ConstantFPSDNode>(C.Op1)) 1861 return false; 1862 1863 // Never swap comparisons with zero since there are many ways to optimize 1864 // those later. 1865 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 1866 if (ConstOp1 && ConstOp1->getZExtValue() == 0) 1867 return false; 1868 1869 // Also keep natural memory operands second if the loaded value is 1870 // only used here. Several comparisons have memory forms. 1871 if (isNaturalMemoryOperand(C.Op1, C.ICmpType) && C.Op1.hasOneUse()) 1872 return false; 1873 1874 // Look for cases where Cmp0 is a single-use load and Cmp1 isn't. 1875 // In that case we generally prefer the memory to be second. 1876 if (isNaturalMemoryOperand(C.Op0, C.ICmpType) && C.Op0.hasOneUse()) { 1877 // The only exceptions are when the second operand is a constant and 1878 // we can use things like CHHSI. 1879 if (!ConstOp1) 1880 return true; 1881 // The unsigned memory-immediate instructions can handle 16-bit 1882 // unsigned integers. 1883 if (C.ICmpType != SystemZICMP::SignedOnly && 1884 isUInt<16>(ConstOp1->getZExtValue())) 1885 return false; 1886 // The signed memory-immediate instructions can handle 16-bit 1887 // signed integers. 1888 if (C.ICmpType != SystemZICMP::UnsignedOnly && 1889 isInt<16>(ConstOp1->getSExtValue())) 1890 return false; 1891 return true; 1892 } 1893 1894 // Try to promote the use of CGFR and CLGFR. 1895 unsigned Opcode0 = C.Op0.getOpcode(); 1896 if (C.ICmpType != SystemZICMP::UnsignedOnly && Opcode0 == ISD::SIGN_EXTEND) 1897 return true; 1898 if (C.ICmpType != SystemZICMP::SignedOnly && Opcode0 == ISD::ZERO_EXTEND) 1899 return true; 1900 if (C.ICmpType != SystemZICMP::SignedOnly && 1901 Opcode0 == ISD::AND && 1902 C.Op0.getOperand(1).getOpcode() == ISD::Constant && 1903 cast<ConstantSDNode>(C.Op0.getOperand(1))->getZExtValue() == 0xffffffff) 1904 return true; 1905 1906 return false; 1907 } 1908 1909 // Return a version of comparison CC mask CCMask in which the LT and GT 1910 // actions are swapped. 1911 static unsigned reverseCCMask(unsigned CCMask) { 1912 return ((CCMask & SystemZ::CCMASK_CMP_EQ) | 1913 (CCMask & SystemZ::CCMASK_CMP_GT ? SystemZ::CCMASK_CMP_LT : 0) | 1914 (CCMask & SystemZ::CCMASK_CMP_LT ? SystemZ::CCMASK_CMP_GT : 0) | 1915 (CCMask & SystemZ::CCMASK_CMP_UO)); 1916 } 1917 1918 // Check whether C tests for equality between X and Y and whether X - Y 1919 // or Y - X is also computed. In that case it's better to compare the 1920 // result of the subtraction against zero. 1921 static void adjustForSubtraction(SelectionDAG &DAG, const SDLoc &DL, 1922 Comparison &C) { 1923 if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 1924 C.CCMask == SystemZ::CCMASK_CMP_NE) { 1925 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 1926 SDNode *N = *I; 1927 if (N->getOpcode() == ISD::SUB && 1928 ((N->getOperand(0) == C.Op0 && N->getOperand(1) == C.Op1) || 1929 (N->getOperand(0) == C.Op1 && N->getOperand(1) == C.Op0))) { 1930 C.Op0 = SDValue(N, 0); 1931 C.Op1 = DAG.getConstant(0, DL, N->getValueType(0)); 1932 return; 1933 } 1934 } 1935 } 1936 } 1937 1938 // Check whether C compares a floating-point value with zero and if that 1939 // floating-point value is also negated. In this case we can use the 1940 // negation to set CC, so avoiding separate LOAD AND TEST and 1941 // LOAD (NEGATIVE/COMPLEMENT) instructions. 1942 static void adjustForFNeg(Comparison &C) { 1943 auto *C1 = dyn_cast<ConstantFPSDNode>(C.Op1); 1944 if (C1 && C1->isZero()) { 1945 for (auto I = C.Op0->use_begin(), E = C.Op0->use_end(); I != E; ++I) { 1946 SDNode *N = *I; 1947 if (N->getOpcode() == ISD::FNEG) { 1948 C.Op0 = SDValue(N, 0); 1949 C.CCMask = reverseCCMask(C.CCMask); 1950 return; 1951 } 1952 } 1953 } 1954 } 1955 1956 // Check whether C compares (shl X, 32) with 0 and whether X is 1957 // also sign-extended. In that case it is better to test the result 1958 // of the sign extension using LTGFR. 1959 // 1960 // This case is important because InstCombine transforms a comparison 1961 // with (sext (trunc X)) into a comparison with (shl X, 32). 1962 static void adjustForLTGFR(Comparison &C) { 1963 // Check for a comparison between (shl X, 32) and 0. 1964 if (C.Op0.getOpcode() == ISD::SHL && 1965 C.Op0.getValueType() == MVT::i64 && 1966 C.Op1.getOpcode() == ISD::Constant && 1967 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1968 auto *C1 = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1)); 1969 if (C1 && C1->getZExtValue() == 32) { 1970 SDValue ShlOp0 = C.Op0.getOperand(0); 1971 // See whether X has any SIGN_EXTEND_INREG uses. 1972 for (auto I = ShlOp0->use_begin(), E = ShlOp0->use_end(); I != E; ++I) { 1973 SDNode *N = *I; 1974 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG && 1975 cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32) { 1976 C.Op0 = SDValue(N, 0); 1977 return; 1978 } 1979 } 1980 } 1981 } 1982 } 1983 1984 // If C compares the truncation of an extending load, try to compare 1985 // the untruncated value instead. This exposes more opportunities to 1986 // reuse CC. 1987 static void adjustICmpTruncate(SelectionDAG &DAG, const SDLoc &DL, 1988 Comparison &C) { 1989 if (C.Op0.getOpcode() == ISD::TRUNCATE && 1990 C.Op0.getOperand(0).getOpcode() == ISD::LOAD && 1991 C.Op1.getOpcode() == ISD::Constant && 1992 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 1993 auto *L = cast<LoadSDNode>(C.Op0.getOperand(0)); 1994 if (L->getMemoryVT().getStoreSizeInBits() <= C.Op0.getValueSizeInBits()) { 1995 unsigned Type = L->getExtensionType(); 1996 if ((Type == ISD::ZEXTLOAD && C.ICmpType != SystemZICMP::SignedOnly) || 1997 (Type == ISD::SEXTLOAD && C.ICmpType != SystemZICMP::UnsignedOnly)) { 1998 C.Op0 = C.Op0.getOperand(0); 1999 C.Op1 = DAG.getConstant(0, DL, C.Op0.getValueType()); 2000 } 2001 } 2002 } 2003 } 2004 2005 // Return true if shift operation N has an in-range constant shift value. 2006 // Store it in ShiftVal if so. 2007 static bool isSimpleShift(SDValue N, unsigned &ShiftVal) { 2008 auto *Shift = dyn_cast<ConstantSDNode>(N.getOperand(1)); 2009 if (!Shift) 2010 return false; 2011 2012 uint64_t Amount = Shift->getZExtValue(); 2013 if (Amount >= N.getValueSizeInBits()) 2014 return false; 2015 2016 ShiftVal = Amount; 2017 return true; 2018 } 2019 2020 // Check whether an AND with Mask is suitable for a TEST UNDER MASK 2021 // instruction and whether the CC value is descriptive enough to handle 2022 // a comparison of type Opcode between the AND result and CmpVal. 2023 // CCMask says which comparison result is being tested and BitSize is 2024 // the number of bits in the operands. If TEST UNDER MASK can be used, 2025 // return the corresponding CC mask, otherwise return 0. 2026 static unsigned getTestUnderMaskCond(unsigned BitSize, unsigned CCMask, 2027 uint64_t Mask, uint64_t CmpVal, 2028 unsigned ICmpType) { 2029 assert(Mask != 0 && "ANDs with zero should have been removed by now"); 2030 2031 // Check whether the mask is suitable for TMHH, TMHL, TMLH or TMLL. 2032 if (!SystemZ::isImmLL(Mask) && !SystemZ::isImmLH(Mask) && 2033 !SystemZ::isImmHL(Mask) && !SystemZ::isImmHH(Mask)) 2034 return 0; 2035 2036 // Work out the masks for the lowest and highest bits. 2037 unsigned HighShift = 63 - countLeadingZeros(Mask); 2038 uint64_t High = uint64_t(1) << HighShift; 2039 uint64_t Low = uint64_t(1) << countTrailingZeros(Mask); 2040 2041 // Signed ordered comparisons are effectively unsigned if the sign 2042 // bit is dropped. 2043 bool EffectivelyUnsigned = (ICmpType != SystemZICMP::SignedOnly); 2044 2045 // Check for equality comparisons with 0, or the equivalent. 2046 if (CmpVal == 0) { 2047 if (CCMask == SystemZ::CCMASK_CMP_EQ) 2048 return SystemZ::CCMASK_TM_ALL_0; 2049 if (CCMask == SystemZ::CCMASK_CMP_NE) 2050 return SystemZ::CCMASK_TM_SOME_1; 2051 } 2052 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <= Low) { 2053 if (CCMask == SystemZ::CCMASK_CMP_LT) 2054 return SystemZ::CCMASK_TM_ALL_0; 2055 if (CCMask == SystemZ::CCMASK_CMP_GE) 2056 return SystemZ::CCMASK_TM_SOME_1; 2057 } 2058 if (EffectivelyUnsigned && CmpVal < Low) { 2059 if (CCMask == SystemZ::CCMASK_CMP_LE) 2060 return SystemZ::CCMASK_TM_ALL_0; 2061 if (CCMask == SystemZ::CCMASK_CMP_GT) 2062 return SystemZ::CCMASK_TM_SOME_1; 2063 } 2064 2065 // Check for equality comparisons with the mask, or the equivalent. 2066 if (CmpVal == Mask) { 2067 if (CCMask == SystemZ::CCMASK_CMP_EQ) 2068 return SystemZ::CCMASK_TM_ALL_1; 2069 if (CCMask == SystemZ::CCMASK_CMP_NE) 2070 return SystemZ::CCMASK_TM_SOME_0; 2071 } 2072 if (EffectivelyUnsigned && CmpVal >= Mask - Low && CmpVal < Mask) { 2073 if (CCMask == SystemZ::CCMASK_CMP_GT) 2074 return SystemZ::CCMASK_TM_ALL_1; 2075 if (CCMask == SystemZ::CCMASK_CMP_LE) 2076 return SystemZ::CCMASK_TM_SOME_0; 2077 } 2078 if (EffectivelyUnsigned && CmpVal > Mask - Low && CmpVal <= Mask) { 2079 if (CCMask == SystemZ::CCMASK_CMP_GE) 2080 return SystemZ::CCMASK_TM_ALL_1; 2081 if (CCMask == SystemZ::CCMASK_CMP_LT) 2082 return SystemZ::CCMASK_TM_SOME_0; 2083 } 2084 2085 // Check for ordered comparisons with the top bit. 2086 if (EffectivelyUnsigned && CmpVal >= Mask - High && CmpVal < High) { 2087 if (CCMask == SystemZ::CCMASK_CMP_LE) 2088 return SystemZ::CCMASK_TM_MSB_0; 2089 if (CCMask == SystemZ::CCMASK_CMP_GT) 2090 return SystemZ::CCMASK_TM_MSB_1; 2091 } 2092 if (EffectivelyUnsigned && CmpVal > Mask - High && CmpVal <= High) { 2093 if (CCMask == SystemZ::CCMASK_CMP_LT) 2094 return SystemZ::CCMASK_TM_MSB_0; 2095 if (CCMask == SystemZ::CCMASK_CMP_GE) 2096 return SystemZ::CCMASK_TM_MSB_1; 2097 } 2098 2099 // If there are just two bits, we can do equality checks for Low and High 2100 // as well. 2101 if (Mask == Low + High) { 2102 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == Low) 2103 return SystemZ::CCMASK_TM_MIXED_MSB_0; 2104 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == Low) 2105 return SystemZ::CCMASK_TM_MIXED_MSB_0 ^ SystemZ::CCMASK_ANY; 2106 if (CCMask == SystemZ::CCMASK_CMP_EQ && CmpVal == High) 2107 return SystemZ::CCMASK_TM_MIXED_MSB_1; 2108 if (CCMask == SystemZ::CCMASK_CMP_NE && CmpVal == High) 2109 return SystemZ::CCMASK_TM_MIXED_MSB_1 ^ SystemZ::CCMASK_ANY; 2110 } 2111 2112 // Looks like we've exhausted our options. 2113 return 0; 2114 } 2115 2116 // See whether C can be implemented as a TEST UNDER MASK instruction. 2117 // Update the arguments with the TM version if so. 2118 static void adjustForTestUnderMask(SelectionDAG &DAG, const SDLoc &DL, 2119 Comparison &C) { 2120 // Check that we have a comparison with a constant. 2121 auto *ConstOp1 = dyn_cast<ConstantSDNode>(C.Op1); 2122 if (!ConstOp1) 2123 return; 2124 uint64_t CmpVal = ConstOp1->getZExtValue(); 2125 2126 // Check whether the nonconstant input is an AND with a constant mask. 2127 Comparison NewC(C); 2128 uint64_t MaskVal; 2129 ConstantSDNode *Mask = nullptr; 2130 if (C.Op0.getOpcode() == ISD::AND) { 2131 NewC.Op0 = C.Op0.getOperand(0); 2132 NewC.Op1 = C.Op0.getOperand(1); 2133 Mask = dyn_cast<ConstantSDNode>(NewC.Op1); 2134 if (!Mask) 2135 return; 2136 MaskVal = Mask->getZExtValue(); 2137 } else { 2138 // There is no instruction to compare with a 64-bit immediate 2139 // so use TMHH instead if possible. We need an unsigned ordered 2140 // comparison with an i64 immediate. 2141 if (NewC.Op0.getValueType() != MVT::i64 || 2142 NewC.CCMask == SystemZ::CCMASK_CMP_EQ || 2143 NewC.CCMask == SystemZ::CCMASK_CMP_NE || 2144 NewC.ICmpType == SystemZICMP::SignedOnly) 2145 return; 2146 // Convert LE and GT comparisons into LT and GE. 2147 if (NewC.CCMask == SystemZ::CCMASK_CMP_LE || 2148 NewC.CCMask == SystemZ::CCMASK_CMP_GT) { 2149 if (CmpVal == uint64_t(-1)) 2150 return; 2151 CmpVal += 1; 2152 NewC.CCMask ^= SystemZ::CCMASK_CMP_EQ; 2153 } 2154 // If the low N bits of Op1 are zero than the low N bits of Op0 can 2155 // be masked off without changing the result. 2156 MaskVal = -(CmpVal & -CmpVal); 2157 NewC.ICmpType = SystemZICMP::UnsignedOnly; 2158 } 2159 if (!MaskVal) 2160 return; 2161 2162 // Check whether the combination of mask, comparison value and comparison 2163 // type are suitable. 2164 unsigned BitSize = NewC.Op0.getValueSizeInBits(); 2165 unsigned NewCCMask, ShiftVal; 2166 if (NewC.ICmpType != SystemZICMP::SignedOnly && 2167 NewC.Op0.getOpcode() == ISD::SHL && 2168 isSimpleShift(NewC.Op0, ShiftVal) && 2169 (MaskVal >> ShiftVal != 0) && 2170 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal && 2171 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 2172 MaskVal >> ShiftVal, 2173 CmpVal >> ShiftVal, 2174 SystemZICMP::Any))) { 2175 NewC.Op0 = NewC.Op0.getOperand(0); 2176 MaskVal >>= ShiftVal; 2177 } else if (NewC.ICmpType != SystemZICMP::SignedOnly && 2178 NewC.Op0.getOpcode() == ISD::SRL && 2179 isSimpleShift(NewC.Op0, ShiftVal) && 2180 (MaskVal << ShiftVal != 0) && 2181 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal && 2182 (NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, 2183 MaskVal << ShiftVal, 2184 CmpVal << ShiftVal, 2185 SystemZICMP::UnsignedOnly))) { 2186 NewC.Op0 = NewC.Op0.getOperand(0); 2187 MaskVal <<= ShiftVal; 2188 } else { 2189 NewCCMask = getTestUnderMaskCond(BitSize, NewC.CCMask, MaskVal, CmpVal, 2190 NewC.ICmpType); 2191 if (!NewCCMask) 2192 return; 2193 } 2194 2195 // Go ahead and make the change. 2196 C.Opcode = SystemZISD::TM; 2197 C.Op0 = NewC.Op0; 2198 if (Mask && Mask->getZExtValue() == MaskVal) 2199 C.Op1 = SDValue(Mask, 0); 2200 else 2201 C.Op1 = DAG.getConstant(MaskVal, DL, C.Op0.getValueType()); 2202 C.CCValid = SystemZ::CCMASK_TM; 2203 C.CCMask = NewCCMask; 2204 } 2205 2206 // See whether the comparison argument contains a redundant AND 2207 // and remove it if so. This sometimes happens due to the generic 2208 // BRCOND expansion. 2209 static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL, 2210 Comparison &C) { 2211 if (C.Op0.getOpcode() != ISD::AND) 2212 return; 2213 auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1)); 2214 if (!Mask) 2215 return; 2216 KnownBits Known; 2217 DAG.computeKnownBits(C.Op0.getOperand(0), Known); 2218 if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue()) 2219 return; 2220 2221 C.Op0 = C.Op0.getOperand(0); 2222 } 2223 2224 // Return a Comparison that tests the condition-code result of intrinsic 2225 // node Call against constant integer CC using comparison code Cond. 2226 // Opcode is the opcode of the SystemZISD operation for the intrinsic 2227 // and CCValid is the set of possible condition-code results. 2228 static Comparison getIntrinsicCmp(SelectionDAG &DAG, unsigned Opcode, 2229 SDValue Call, unsigned CCValid, uint64_t CC, 2230 ISD::CondCode Cond) { 2231 Comparison C(Call, SDValue()); 2232 C.Opcode = Opcode; 2233 C.CCValid = CCValid; 2234 if (Cond == ISD::SETEQ) 2235 // bit 3 for CC==0, bit 0 for CC==3, always false for CC>3. 2236 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0; 2237 else if (Cond == ISD::SETNE) 2238 // ...and the inverse of that. 2239 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1; 2240 else if (Cond == ISD::SETLT || Cond == ISD::SETULT) 2241 // bits above bit 3 for CC==0 (always false), bits above bit 0 for CC==3, 2242 // always true for CC>3. 2243 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1; 2244 else if (Cond == ISD::SETGE || Cond == ISD::SETUGE) 2245 // ...and the inverse of that. 2246 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0; 2247 else if (Cond == ISD::SETLE || Cond == ISD::SETULE) 2248 // bit 3 and above for CC==0, bit 0 and above for CC==3 (always true), 2249 // always true for CC>3. 2250 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1; 2251 else if (Cond == ISD::SETGT || Cond == ISD::SETUGT) 2252 // ...and the inverse of that. 2253 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0; 2254 else 2255 llvm_unreachable("Unexpected integer comparison type"); 2256 C.CCMask &= CCValid; 2257 return C; 2258 } 2259 2260 // Decide how to implement a comparison of type Cond between CmpOp0 with CmpOp1. 2261 static Comparison getCmp(SelectionDAG &DAG, SDValue CmpOp0, SDValue CmpOp1, 2262 ISD::CondCode Cond, const SDLoc &DL) { 2263 if (CmpOp1.getOpcode() == ISD::Constant) { 2264 uint64_t Constant = cast<ConstantSDNode>(CmpOp1)->getZExtValue(); 2265 unsigned Opcode, CCValid; 2266 if (CmpOp0.getOpcode() == ISD::INTRINSIC_W_CHAIN && 2267 CmpOp0.getResNo() == 0 && CmpOp0->hasNUsesOfValue(1, 0) && 2268 isIntrinsicWithCCAndChain(CmpOp0, Opcode, CCValid)) 2269 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond); 2270 if (CmpOp0.getOpcode() == ISD::INTRINSIC_WO_CHAIN && 2271 CmpOp0.getResNo() == CmpOp0->getNumValues() - 1 && 2272 isIntrinsicWithCC(CmpOp0, Opcode, CCValid)) 2273 return getIntrinsicCmp(DAG, Opcode, CmpOp0, CCValid, Constant, Cond); 2274 } 2275 Comparison C(CmpOp0, CmpOp1); 2276 C.CCMask = CCMaskForCondCode(Cond); 2277 if (C.Op0.getValueType().isFloatingPoint()) { 2278 C.CCValid = SystemZ::CCMASK_FCMP; 2279 C.Opcode = SystemZISD::FCMP; 2280 adjustForFNeg(C); 2281 } else { 2282 C.CCValid = SystemZ::CCMASK_ICMP; 2283 C.Opcode = SystemZISD::ICMP; 2284 // Choose the type of comparison. Equality and inequality tests can 2285 // use either signed or unsigned comparisons. The choice also doesn't 2286 // matter if both sign bits are known to be clear. In those cases we 2287 // want to give the main isel code the freedom to choose whichever 2288 // form fits best. 2289 if (C.CCMask == SystemZ::CCMASK_CMP_EQ || 2290 C.CCMask == SystemZ::CCMASK_CMP_NE || 2291 (DAG.SignBitIsZero(C.Op0) && DAG.SignBitIsZero(C.Op1))) 2292 C.ICmpType = SystemZICMP::Any; 2293 else if (C.CCMask & SystemZ::CCMASK_CMP_UO) 2294 C.ICmpType = SystemZICMP::UnsignedOnly; 2295 else 2296 C.ICmpType = SystemZICMP::SignedOnly; 2297 C.CCMask &= ~SystemZ::CCMASK_CMP_UO; 2298 adjustForRedundantAnd(DAG, DL, C); 2299 adjustZeroCmp(DAG, DL, C); 2300 adjustSubwordCmp(DAG, DL, C); 2301 adjustForSubtraction(DAG, DL, C); 2302 adjustForLTGFR(C); 2303 adjustICmpTruncate(DAG, DL, C); 2304 } 2305 2306 if (shouldSwapCmpOperands(C)) { 2307 std::swap(C.Op0, C.Op1); 2308 C.CCMask = reverseCCMask(C.CCMask); 2309 } 2310 2311 adjustForTestUnderMask(DAG, DL, C); 2312 return C; 2313 } 2314 2315 // Emit the comparison instruction described by C. 2316 static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) { 2317 if (!C.Op1.getNode()) { 2318 SDNode *Node; 2319 switch (C.Op0.getOpcode()) { 2320 case ISD::INTRINSIC_W_CHAIN: 2321 Node = emitIntrinsicWithCCAndChain(DAG, C.Op0, C.Opcode); 2322 return SDValue(Node, 0); 2323 case ISD::INTRINSIC_WO_CHAIN: 2324 Node = emitIntrinsicWithCC(DAG, C.Op0, C.Opcode); 2325 return SDValue(Node, Node->getNumValues() - 1); 2326 default: 2327 llvm_unreachable("Invalid comparison operands"); 2328 } 2329 } 2330 if (C.Opcode == SystemZISD::ICMP) 2331 return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1, 2332 DAG.getConstant(C.ICmpType, DL, MVT::i32)); 2333 if (C.Opcode == SystemZISD::TM) { 2334 bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) != 2335 bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1)); 2336 return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1, 2337 DAG.getConstant(RegisterOnly, DL, MVT::i32)); 2338 } 2339 return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1); 2340 } 2341 2342 // Implement a 32-bit *MUL_LOHI operation by extending both operands to 2343 // 64 bits. Extend is the extension type to use. Store the high part 2344 // in Hi and the low part in Lo. 2345 static void lowerMUL_LOHI32(SelectionDAG &DAG, const SDLoc &DL, unsigned Extend, 2346 SDValue Op0, SDValue Op1, SDValue &Hi, 2347 SDValue &Lo) { 2348 Op0 = DAG.getNode(Extend, DL, MVT::i64, Op0); 2349 Op1 = DAG.getNode(Extend, DL, MVT::i64, Op1); 2350 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, Op0, Op1); 2351 Hi = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, 2352 DAG.getConstant(32, DL, MVT::i64)); 2353 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Hi); 2354 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul); 2355 } 2356 2357 // Lower a binary operation that produces two VT results, one in each 2358 // half of a GR128 pair. Op0 and Op1 are the VT operands to the operation, 2359 // and Opcode performs the GR128 operation. Store the even register result 2360 // in Even and the odd register result in Odd. 2361 static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT, 2362 unsigned Opcode, SDValue Op0, SDValue Op1, 2363 SDValue &Even, SDValue &Odd) { 2364 SDValue Result = DAG.getNode(Opcode, DL, MVT::Untyped, Op0, Op1); 2365 bool Is32Bit = is32Bit(VT); 2366 Even = DAG.getTargetExtractSubreg(SystemZ::even128(Is32Bit), DL, VT, Result); 2367 Odd = DAG.getTargetExtractSubreg(SystemZ::odd128(Is32Bit), DL, VT, Result); 2368 } 2369 2370 // Return an i32 value that is 1 if the CC value produced by CCReg is 2371 // in the mask CCMask and 0 otherwise. CC is known to have a value 2372 // in CCValid, so other values can be ignored. 2373 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg, 2374 unsigned CCValid, unsigned CCMask) { 2375 SDValue Ops[] = { DAG.getConstant(1, DL, MVT::i32), 2376 DAG.getConstant(0, DL, MVT::i32), 2377 DAG.getConstant(CCValid, DL, MVT::i32), 2378 DAG.getConstant(CCMask, DL, MVT::i32), CCReg }; 2379 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops); 2380 } 2381 2382 // Return the SystemISD vector comparison operation for CC, or 0 if it cannot 2383 // be done directly. IsFP is true if CC is for a floating-point rather than 2384 // integer comparison. 2385 static unsigned getVectorComparison(ISD::CondCode CC, bool IsFP) { 2386 switch (CC) { 2387 case ISD::SETOEQ: 2388 case ISD::SETEQ: 2389 return IsFP ? SystemZISD::VFCMPE : SystemZISD::VICMPE; 2390 2391 case ISD::SETOGE: 2392 case ISD::SETGE: 2393 return IsFP ? SystemZISD::VFCMPHE : static_cast<SystemZISD::NodeType>(0); 2394 2395 case ISD::SETOGT: 2396 case ISD::SETGT: 2397 return IsFP ? SystemZISD::VFCMPH : SystemZISD::VICMPH; 2398 2399 case ISD::SETUGT: 2400 return IsFP ? static_cast<SystemZISD::NodeType>(0) : SystemZISD::VICMPHL; 2401 2402 default: 2403 return 0; 2404 } 2405 } 2406 2407 // Return the SystemZISD vector comparison operation for CC or its inverse, 2408 // or 0 if neither can be done directly. Indicate in Invert whether the 2409 // result is for the inverse of CC. IsFP is true if CC is for a 2410 // floating-point rather than integer comparison. 2411 static unsigned getVectorComparisonOrInvert(ISD::CondCode CC, bool IsFP, 2412 bool &Invert) { 2413 if (unsigned Opcode = getVectorComparison(CC, IsFP)) { 2414 Invert = false; 2415 return Opcode; 2416 } 2417 2418 CC = ISD::getSetCCInverse(CC, !IsFP); 2419 if (unsigned Opcode = getVectorComparison(CC, IsFP)) { 2420 Invert = true; 2421 return Opcode; 2422 } 2423 2424 return 0; 2425 } 2426 2427 // Return a v2f64 that contains the extended form of elements Start and Start+1 2428 // of v4f32 value Op. 2429 static SDValue expandV4F32ToV2F64(SelectionDAG &DAG, int Start, const SDLoc &DL, 2430 SDValue Op) { 2431 int Mask[] = { Start, -1, Start + 1, -1 }; 2432 Op = DAG.getVectorShuffle(MVT::v4f32, DL, Op, DAG.getUNDEF(MVT::v4f32), Mask); 2433 return DAG.getNode(SystemZISD::VEXTEND, DL, MVT::v2f64, Op); 2434 } 2435 2436 // Build a comparison of vectors CmpOp0 and CmpOp1 using opcode Opcode, 2437 // producing a result of type VT. 2438 SDValue SystemZTargetLowering::getVectorCmp(SelectionDAG &DAG, unsigned Opcode, 2439 const SDLoc &DL, EVT VT, 2440 SDValue CmpOp0, 2441 SDValue CmpOp1) const { 2442 // There is no hardware support for v4f32 (unless we have the vector 2443 // enhancements facility 1), so extend the vector into two v2f64s 2444 // and compare those. 2445 if (CmpOp0.getValueType() == MVT::v4f32 && 2446 !Subtarget.hasVectorEnhancements1()) { 2447 SDValue H0 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp0); 2448 SDValue L0 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp0); 2449 SDValue H1 = expandV4F32ToV2F64(DAG, 0, DL, CmpOp1); 2450 SDValue L1 = expandV4F32ToV2F64(DAG, 2, DL, CmpOp1); 2451 SDValue HRes = DAG.getNode(Opcode, DL, MVT::v2i64, H0, H1); 2452 SDValue LRes = DAG.getNode(Opcode, DL, MVT::v2i64, L0, L1); 2453 return DAG.getNode(SystemZISD::PACK, DL, VT, HRes, LRes); 2454 } 2455 return DAG.getNode(Opcode, DL, VT, CmpOp0, CmpOp1); 2456 } 2457 2458 // Lower a vector comparison of type CC between CmpOp0 and CmpOp1, producing 2459 // an integer mask of type VT. 2460 SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG, 2461 const SDLoc &DL, EVT VT, 2462 ISD::CondCode CC, 2463 SDValue CmpOp0, 2464 SDValue CmpOp1) const { 2465 bool IsFP = CmpOp0.getValueType().isFloatingPoint(); 2466 bool Invert = false; 2467 SDValue Cmp; 2468 switch (CC) { 2469 // Handle tests for order using (or (ogt y x) (oge x y)). 2470 case ISD::SETUO: 2471 Invert = true; 2472 LLVM_FALLTHROUGH; 2473 case ISD::SETO: { 2474 assert(IsFP && "Unexpected integer comparison"); 2475 SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0); 2476 SDValue GE = getVectorCmp(DAG, SystemZISD::VFCMPHE, DL, VT, CmpOp0, CmpOp1); 2477 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GE); 2478 break; 2479 } 2480 2481 // Handle <> tests using (or (ogt y x) (ogt x y)). 2482 case ISD::SETUEQ: 2483 Invert = true; 2484 LLVM_FALLTHROUGH; 2485 case ISD::SETONE: { 2486 assert(IsFP && "Unexpected integer comparison"); 2487 SDValue LT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp1, CmpOp0); 2488 SDValue GT = getVectorCmp(DAG, SystemZISD::VFCMPH, DL, VT, CmpOp0, CmpOp1); 2489 Cmp = DAG.getNode(ISD::OR, DL, VT, LT, GT); 2490 break; 2491 } 2492 2493 // Otherwise a single comparison is enough. It doesn't really 2494 // matter whether we try the inversion or the swap first, since 2495 // there are no cases where both work. 2496 default: 2497 if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert)) 2498 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp0, CmpOp1); 2499 else { 2500 CC = ISD::getSetCCSwappedOperands(CC); 2501 if (unsigned Opcode = getVectorComparisonOrInvert(CC, IsFP, Invert)) 2502 Cmp = getVectorCmp(DAG, Opcode, DL, VT, CmpOp1, CmpOp0); 2503 else 2504 llvm_unreachable("Unhandled comparison"); 2505 } 2506 break; 2507 } 2508 if (Invert) { 2509 SDValue Mask = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8, 2510 DAG.getConstant(65535, DL, MVT::i32)); 2511 Mask = DAG.getNode(ISD::BITCAST, DL, VT, Mask); 2512 Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask); 2513 } 2514 return Cmp; 2515 } 2516 2517 SDValue SystemZTargetLowering::lowerSETCC(SDValue Op, 2518 SelectionDAG &DAG) const { 2519 SDValue CmpOp0 = Op.getOperand(0); 2520 SDValue CmpOp1 = Op.getOperand(1); 2521 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); 2522 SDLoc DL(Op); 2523 EVT VT = Op.getValueType(); 2524 if (VT.isVector()) 2525 return lowerVectorSETCC(DAG, DL, VT, CC, CmpOp0, CmpOp1); 2526 2527 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); 2528 SDValue CCReg = emitCmp(DAG, DL, C); 2529 return emitSETCC(DAG, DL, CCReg, C.CCValid, C.CCMask); 2530 } 2531 2532 SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const { 2533 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); 2534 SDValue CmpOp0 = Op.getOperand(2); 2535 SDValue CmpOp1 = Op.getOperand(3); 2536 SDValue Dest = Op.getOperand(4); 2537 SDLoc DL(Op); 2538 2539 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); 2540 SDValue CCReg = emitCmp(DAG, DL, C); 2541 return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(), 2542 Op.getOperand(0), DAG.getConstant(C.CCValid, DL, MVT::i32), 2543 DAG.getConstant(C.CCMask, DL, MVT::i32), Dest, CCReg); 2544 } 2545 2546 // Return true if Pos is CmpOp and Neg is the negative of CmpOp, 2547 // allowing Pos and Neg to be wider than CmpOp. 2548 static bool isAbsolute(SDValue CmpOp, SDValue Pos, SDValue Neg) { 2549 return (Neg.getOpcode() == ISD::SUB && 2550 Neg.getOperand(0).getOpcode() == ISD::Constant && 2551 cast<ConstantSDNode>(Neg.getOperand(0))->getZExtValue() == 0 && 2552 Neg.getOperand(1) == Pos && 2553 (Pos == CmpOp || 2554 (Pos.getOpcode() == ISD::SIGN_EXTEND && 2555 Pos.getOperand(0) == CmpOp))); 2556 } 2557 2558 // Return the absolute or negative absolute of Op; IsNegative decides which. 2559 static SDValue getAbsolute(SelectionDAG &DAG, const SDLoc &DL, SDValue Op, 2560 bool IsNegative) { 2561 Op = DAG.getNode(SystemZISD::IABS, DL, Op.getValueType(), Op); 2562 if (IsNegative) 2563 Op = DAG.getNode(ISD::SUB, DL, Op.getValueType(), 2564 DAG.getConstant(0, DL, Op.getValueType()), Op); 2565 return Op; 2566 } 2567 2568 SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op, 2569 SelectionDAG &DAG) const { 2570 SDValue CmpOp0 = Op.getOperand(0); 2571 SDValue CmpOp1 = Op.getOperand(1); 2572 SDValue TrueOp = Op.getOperand(2); 2573 SDValue FalseOp = Op.getOperand(3); 2574 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); 2575 SDLoc DL(Op); 2576 2577 Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL)); 2578 2579 // Check for absolute and negative-absolute selections, including those 2580 // where the comparison value is sign-extended (for LPGFR and LNGFR). 2581 // This check supplements the one in DAGCombiner. 2582 if (C.Opcode == SystemZISD::ICMP && 2583 C.CCMask != SystemZ::CCMASK_CMP_EQ && 2584 C.CCMask != SystemZ::CCMASK_CMP_NE && 2585 C.Op1.getOpcode() == ISD::Constant && 2586 cast<ConstantSDNode>(C.Op1)->getZExtValue() == 0) { 2587 if (isAbsolute(C.Op0, TrueOp, FalseOp)) 2588 return getAbsolute(DAG, DL, TrueOp, C.CCMask & SystemZ::CCMASK_CMP_LT); 2589 if (isAbsolute(C.Op0, FalseOp, TrueOp)) 2590 return getAbsolute(DAG, DL, FalseOp, C.CCMask & SystemZ::CCMASK_CMP_GT); 2591 } 2592 2593 SDValue CCReg = emitCmp(DAG, DL, C); 2594 SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, DL, MVT::i32), 2595 DAG.getConstant(C.CCMask, DL, MVT::i32), CCReg}; 2596 2597 return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops); 2598 } 2599 2600 SDValue SystemZTargetLowering::lowerGlobalAddress(GlobalAddressSDNode *Node, 2601 SelectionDAG &DAG) const { 2602 SDLoc DL(Node); 2603 const GlobalValue *GV = Node->getGlobal(); 2604 int64_t Offset = Node->getOffset(); 2605 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2606 CodeModel::Model CM = DAG.getTarget().getCodeModel(); 2607 2608 SDValue Result; 2609 if (Subtarget.isPC32DBLSymbol(GV, CM)) { 2610 // Assign anchors at 1<<12 byte boundaries. 2611 uint64_t Anchor = Offset & ~uint64_t(0xfff); 2612 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor); 2613 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 2614 2615 // The offset can be folded into the address if it is aligned to a halfword. 2616 Offset -= Anchor; 2617 if (Offset != 0 && (Offset & 1) == 0) { 2618 SDValue Full = DAG.getTargetGlobalAddress(GV, DL, PtrVT, Anchor + Offset); 2619 Result = DAG.getNode(SystemZISD::PCREL_OFFSET, DL, PtrVT, Full, Result); 2620 Offset = 0; 2621 } 2622 } else { 2623 Result = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, SystemZII::MO_GOT); 2624 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 2625 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, 2626 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 2627 } 2628 2629 // If there was a non-zero offset that we didn't fold, create an explicit 2630 // addition for it. 2631 if (Offset != 0) 2632 Result = DAG.getNode(ISD::ADD, DL, PtrVT, Result, 2633 DAG.getConstant(Offset, DL, PtrVT)); 2634 2635 return Result; 2636 } 2637 2638 SDValue SystemZTargetLowering::lowerTLSGetOffset(GlobalAddressSDNode *Node, 2639 SelectionDAG &DAG, 2640 unsigned Opcode, 2641 SDValue GOTOffset) const { 2642 SDLoc DL(Node); 2643 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2644 SDValue Chain = DAG.getEntryNode(); 2645 SDValue Glue; 2646 2647 // __tls_get_offset takes the GOT offset in %r2 and the GOT in %r12. 2648 SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(PtrVT); 2649 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R12D, GOT, Glue); 2650 Glue = Chain.getValue(1); 2651 Chain = DAG.getCopyToReg(Chain, DL, SystemZ::R2D, GOTOffset, Glue); 2652 Glue = Chain.getValue(1); 2653 2654 // The first call operand is the chain and the second is the TLS symbol. 2655 SmallVector<SDValue, 8> Ops; 2656 Ops.push_back(Chain); 2657 Ops.push_back(DAG.getTargetGlobalAddress(Node->getGlobal(), DL, 2658 Node->getValueType(0), 2659 0, 0)); 2660 2661 // Add argument registers to the end of the list so that they are 2662 // known live into the call. 2663 Ops.push_back(DAG.getRegister(SystemZ::R2D, PtrVT)); 2664 Ops.push_back(DAG.getRegister(SystemZ::R12D, PtrVT)); 2665 2666 // Add a register mask operand representing the call-preserved registers. 2667 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); 2668 const uint32_t *Mask = 2669 TRI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C); 2670 assert(Mask && "Missing call preserved mask for calling convention"); 2671 Ops.push_back(DAG.getRegisterMask(Mask)); 2672 2673 // Glue the call to the argument copies. 2674 Ops.push_back(Glue); 2675 2676 // Emit the call. 2677 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 2678 Chain = DAG.getNode(Opcode, DL, NodeTys, Ops); 2679 Glue = Chain.getValue(1); 2680 2681 // Copy the return value from %r2. 2682 return DAG.getCopyFromReg(Chain, DL, SystemZ::R2D, PtrVT, Glue); 2683 } 2684 2685 SDValue SystemZTargetLowering::lowerThreadPointer(const SDLoc &DL, 2686 SelectionDAG &DAG) const { 2687 SDValue Chain = DAG.getEntryNode(); 2688 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2689 2690 // The high part of the thread pointer is in access register 0. 2691 SDValue TPHi = DAG.getCopyFromReg(Chain, DL, SystemZ::A0, MVT::i32); 2692 TPHi = DAG.getNode(ISD::ANY_EXTEND, DL, PtrVT, TPHi); 2693 2694 // The low part of the thread pointer is in access register 1. 2695 SDValue TPLo = DAG.getCopyFromReg(Chain, DL, SystemZ::A1, MVT::i32); 2696 TPLo = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TPLo); 2697 2698 // Merge them into a single 64-bit address. 2699 SDValue TPHiShifted = DAG.getNode(ISD::SHL, DL, PtrVT, TPHi, 2700 DAG.getConstant(32, DL, PtrVT)); 2701 return DAG.getNode(ISD::OR, DL, PtrVT, TPHiShifted, TPLo); 2702 } 2703 2704 SDValue SystemZTargetLowering::lowerGlobalTLSAddress(GlobalAddressSDNode *Node, 2705 SelectionDAG &DAG) const { 2706 if (DAG.getTarget().useEmulatedTLS()) 2707 return LowerToTLSEmulatedModel(Node, DAG); 2708 SDLoc DL(Node); 2709 const GlobalValue *GV = Node->getGlobal(); 2710 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2711 TLSModel::Model model = DAG.getTarget().getTLSModel(GV); 2712 2713 SDValue TP = lowerThreadPointer(DL, DAG); 2714 2715 // Get the offset of GA from the thread pointer, based on the TLS model. 2716 SDValue Offset; 2717 switch (model) { 2718 case TLSModel::GeneralDynamic: { 2719 // Load the GOT offset of the tls_index (module ID / per-symbol offset). 2720 SystemZConstantPoolValue *CPV = 2721 SystemZConstantPoolValue::Create(GV, SystemZCP::TLSGD); 2722 2723 Offset = DAG.getConstantPool(CPV, PtrVT, 8); 2724 Offset = DAG.getLoad( 2725 PtrVT, DL, DAG.getEntryNode(), Offset, 2726 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2727 2728 // Call __tls_get_offset to retrieve the offset. 2729 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_GDCALL, Offset); 2730 break; 2731 } 2732 2733 case TLSModel::LocalDynamic: { 2734 // Load the GOT offset of the module ID. 2735 SystemZConstantPoolValue *CPV = 2736 SystemZConstantPoolValue::Create(GV, SystemZCP::TLSLDM); 2737 2738 Offset = DAG.getConstantPool(CPV, PtrVT, 8); 2739 Offset = DAG.getLoad( 2740 PtrVT, DL, DAG.getEntryNode(), Offset, 2741 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2742 2743 // Call __tls_get_offset to retrieve the module base offset. 2744 Offset = lowerTLSGetOffset(Node, DAG, SystemZISD::TLS_LDCALL, Offset); 2745 2746 // Note: The SystemZLDCleanupPass will remove redundant computations 2747 // of the module base offset. Count total number of local-dynamic 2748 // accesses to trigger execution of that pass. 2749 SystemZMachineFunctionInfo* MFI = 2750 DAG.getMachineFunction().getInfo<SystemZMachineFunctionInfo>(); 2751 MFI->incNumLocalDynamicTLSAccesses(); 2752 2753 // Add the per-symbol offset. 2754 CPV = SystemZConstantPoolValue::Create(GV, SystemZCP::DTPOFF); 2755 2756 SDValue DTPOffset = DAG.getConstantPool(CPV, PtrVT, 8); 2757 DTPOffset = DAG.getLoad( 2758 PtrVT, DL, DAG.getEntryNode(), DTPOffset, 2759 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2760 2761 Offset = DAG.getNode(ISD::ADD, DL, PtrVT, Offset, DTPOffset); 2762 break; 2763 } 2764 2765 case TLSModel::InitialExec: { 2766 // Load the offset from the GOT. 2767 Offset = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 2768 SystemZII::MO_INDNTPOFF); 2769 Offset = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Offset); 2770 Offset = 2771 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Offset, 2772 MachinePointerInfo::getGOT(DAG.getMachineFunction())); 2773 break; 2774 } 2775 2776 case TLSModel::LocalExec: { 2777 // Force the offset into the constant pool and load it from there. 2778 SystemZConstantPoolValue *CPV = 2779 SystemZConstantPoolValue::Create(GV, SystemZCP::NTPOFF); 2780 2781 Offset = DAG.getConstantPool(CPV, PtrVT, 8); 2782 Offset = DAG.getLoad( 2783 PtrVT, DL, DAG.getEntryNode(), Offset, 2784 MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); 2785 break; 2786 } 2787 } 2788 2789 // Add the base and offset together. 2790 return DAG.getNode(ISD::ADD, DL, PtrVT, TP, Offset); 2791 } 2792 2793 SDValue SystemZTargetLowering::lowerBlockAddress(BlockAddressSDNode *Node, 2794 SelectionDAG &DAG) const { 2795 SDLoc DL(Node); 2796 const BlockAddress *BA = Node->getBlockAddress(); 2797 int64_t Offset = Node->getOffset(); 2798 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2799 2800 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset); 2801 Result = DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 2802 return Result; 2803 } 2804 2805 SDValue SystemZTargetLowering::lowerJumpTable(JumpTableSDNode *JT, 2806 SelectionDAG &DAG) const { 2807 SDLoc DL(JT); 2808 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2809 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); 2810 2811 // Use LARL to load the address of the table. 2812 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 2813 } 2814 2815 SDValue SystemZTargetLowering::lowerConstantPool(ConstantPoolSDNode *CP, 2816 SelectionDAG &DAG) const { 2817 SDLoc DL(CP); 2818 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2819 2820 SDValue Result; 2821 if (CP->isMachineConstantPoolEntry()) 2822 Result = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, 2823 CP->getAlignment()); 2824 else 2825 Result = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, 2826 CP->getAlignment(), CP->getOffset()); 2827 2828 // Use LARL to load the address of the constant pool entry. 2829 return DAG.getNode(SystemZISD::PCREL_WRAPPER, DL, PtrVT, Result); 2830 } 2831 2832 SDValue SystemZTargetLowering::lowerFRAMEADDR(SDValue Op, 2833 SelectionDAG &DAG) const { 2834 MachineFunction &MF = DAG.getMachineFunction(); 2835 MachineFrameInfo &MFI = MF.getFrameInfo(); 2836 MFI.setFrameAddressIsTaken(true); 2837 2838 SDLoc DL(Op); 2839 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 2840 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2841 2842 // If the back chain frame index has not been allocated yet, do so. 2843 SystemZMachineFunctionInfo *FI = MF.getInfo<SystemZMachineFunctionInfo>(); 2844 int BackChainIdx = FI->getFramePointerSaveIndex(); 2845 if (!BackChainIdx) { 2846 // By definition, the frame address is the address of the back chain. 2847 BackChainIdx = MFI.CreateFixedObject(8, -SystemZMC::CallFrameSize, false); 2848 FI->setFramePointerSaveIndex(BackChainIdx); 2849 } 2850 SDValue BackChain = DAG.getFrameIndex(BackChainIdx, PtrVT); 2851 2852 // FIXME The frontend should detect this case. 2853 if (Depth > 0) { 2854 report_fatal_error("Unsupported stack frame traversal count"); 2855 } 2856 2857 return BackChain; 2858 } 2859 2860 SDValue SystemZTargetLowering::lowerRETURNADDR(SDValue Op, 2861 SelectionDAG &DAG) const { 2862 MachineFunction &MF = DAG.getMachineFunction(); 2863 MachineFrameInfo &MFI = MF.getFrameInfo(); 2864 MFI.setReturnAddressIsTaken(true); 2865 2866 if (verifyReturnAddressArgumentIsConstant(Op, DAG)) 2867 return SDValue(); 2868 2869 SDLoc DL(Op); 2870 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 2871 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2872 2873 // FIXME The frontend should detect this case. 2874 if (Depth > 0) { 2875 report_fatal_error("Unsupported stack frame traversal count"); 2876 } 2877 2878 // Return R14D, which has the return address. Mark it an implicit live-in. 2879 unsigned LinkReg = MF.addLiveIn(SystemZ::R14D, &SystemZ::GR64BitRegClass); 2880 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, LinkReg, PtrVT); 2881 } 2882 2883 SDValue SystemZTargetLowering::lowerBITCAST(SDValue Op, 2884 SelectionDAG &DAG) const { 2885 SDLoc DL(Op); 2886 SDValue In = Op.getOperand(0); 2887 EVT InVT = In.getValueType(); 2888 EVT ResVT = Op.getValueType(); 2889 2890 // Convert loads directly. This is normally done by DAGCombiner, 2891 // but we need this case for bitcasts that are created during lowering 2892 // and which are then lowered themselves. 2893 if (auto *LoadN = dyn_cast<LoadSDNode>(In)) 2894 if (ISD::isNormalLoad(LoadN)) { 2895 SDValue NewLoad = DAG.getLoad(ResVT, DL, LoadN->getChain(), 2896 LoadN->getBasePtr(), LoadN->getMemOperand()); 2897 // Update the chain uses. 2898 DAG.ReplaceAllUsesOfValueWith(SDValue(LoadN, 1), NewLoad.getValue(1)); 2899 return NewLoad; 2900 } 2901 2902 if (InVT == MVT::i32 && ResVT == MVT::f32) { 2903 SDValue In64; 2904 if (Subtarget.hasHighWord()) { 2905 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, 2906 MVT::i64); 2907 In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_h32, DL, 2908 MVT::i64, SDValue(U64, 0), In); 2909 } else { 2910 In64 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, In); 2911 In64 = DAG.getNode(ISD::SHL, DL, MVT::i64, In64, 2912 DAG.getConstant(32, DL, MVT::i64)); 2913 } 2914 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::f64, In64); 2915 return DAG.getTargetExtractSubreg(SystemZ::subreg_r32, 2916 DL, MVT::f32, Out64); 2917 } 2918 if (InVT == MVT::f32 && ResVT == MVT::i32) { 2919 SDNode *U64 = DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::f64); 2920 SDValue In64 = DAG.getTargetInsertSubreg(SystemZ::subreg_r32, DL, 2921 MVT::f64, SDValue(U64, 0), In); 2922 SDValue Out64 = DAG.getNode(ISD::BITCAST, DL, MVT::i64, In64); 2923 if (Subtarget.hasHighWord()) 2924 return DAG.getTargetExtractSubreg(SystemZ::subreg_h32, DL, 2925 MVT::i32, Out64); 2926 SDValue Shift = DAG.getNode(ISD::SRL, DL, MVT::i64, Out64, 2927 DAG.getConstant(32, DL, MVT::i64)); 2928 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Shift); 2929 } 2930 llvm_unreachable("Unexpected bitcast combination"); 2931 } 2932 2933 SDValue SystemZTargetLowering::lowerVASTART(SDValue Op, 2934 SelectionDAG &DAG) const { 2935 MachineFunction &MF = DAG.getMachineFunction(); 2936 SystemZMachineFunctionInfo *FuncInfo = 2937 MF.getInfo<SystemZMachineFunctionInfo>(); 2938 EVT PtrVT = getPointerTy(DAG.getDataLayout()); 2939 2940 SDValue Chain = Op.getOperand(0); 2941 SDValue Addr = Op.getOperand(1); 2942 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); 2943 SDLoc DL(Op); 2944 2945 // The initial values of each field. 2946 const unsigned NumFields = 4; 2947 SDValue Fields[NumFields] = { 2948 DAG.getConstant(FuncInfo->getVarArgsFirstGPR(), DL, PtrVT), 2949 DAG.getConstant(FuncInfo->getVarArgsFirstFPR(), DL, PtrVT), 2950 DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT), 2951 DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT) 2952 }; 2953 2954 // Store each field into its respective slot. 2955 SDValue MemOps[NumFields]; 2956 unsigned Offset = 0; 2957 for (unsigned I = 0; I < NumFields; ++I) { 2958 SDValue FieldAddr = Addr; 2959 if (Offset != 0) 2960 FieldAddr = DAG.getNode(ISD::ADD, DL, PtrVT, FieldAddr, 2961 DAG.getIntPtrConstant(Offset, DL)); 2962 MemOps[I] = DAG.getStore(Chain, DL, Fields[I], FieldAddr, 2963 MachinePointerInfo(SV, Offset)); 2964 Offset += 8; 2965 } 2966 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); 2967 } 2968 2969 SDValue SystemZTargetLowering::lowerVACOPY(SDValue Op, 2970 SelectionDAG &DAG) const { 2971 SDValue Chain = Op.getOperand(0); 2972 SDValue DstPtr = Op.getOperand(1); 2973 SDValue SrcPtr = Op.getOperand(2); 2974 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue(); 2975 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue(); 2976 SDLoc DL(Op); 2977 2978 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr, DAG.getIntPtrConstant(32, DL), 2979 /*Align*/8, /*isVolatile*/false, /*AlwaysInline*/false, 2980 /*isTailCall*/false, 2981 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV)); 2982 } 2983 2984 SDValue SystemZTargetLowering:: 2985 lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { 2986 const TargetFrameLowering *TFI = Subtarget.getFrameLowering(); 2987 MachineFunction &MF = DAG.getMachineFunction(); 2988 bool RealignOpt = !MF.getFunction().hasFnAttribute("no-realign-stack"); 2989 bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain"); 2990 2991 SDValue Chain = Op.getOperand(0); 2992 SDValue Size = Op.getOperand(1); 2993 SDValue Align = Op.getOperand(2); 2994 SDLoc DL(Op); 2995 2996 // If user has set the no alignment function attribute, ignore 2997 // alloca alignments. 2998 uint64_t AlignVal = (RealignOpt ? 2999 dyn_cast<ConstantSDNode>(Align)->getZExtValue() : 0); 3000 3001 uint64_t StackAlign = TFI->getStackAlignment(); 3002 uint64_t RequiredAlign = std::max(AlignVal, StackAlign); 3003 uint64_t ExtraAlignSpace = RequiredAlign - StackAlign; 3004 3005 unsigned SPReg = getStackPointerRegisterToSaveRestore(); 3006 SDValue NeededSpace = Size; 3007 3008 // Get a reference to the stack pointer. 3009 SDValue OldSP = DAG.getCopyFromReg(Chain, DL, SPReg, MVT::i64); 3010 3011 // If we need a backchain, save it now. 3012 SDValue Backchain; 3013 if (StoreBackchain) 3014 Backchain = DAG.getLoad(MVT::i64, DL, Chain, OldSP, MachinePointerInfo()); 3015 3016 // Add extra space for alignment if needed. 3017 if (ExtraAlignSpace) 3018 NeededSpace = DAG.getNode(ISD::ADD, DL, MVT::i64, NeededSpace, 3019 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64)); 3020 3021 // Get the new stack pointer value. 3022 SDValue NewSP = DAG.getNode(ISD::SUB, DL, MVT::i64, OldSP, NeededSpace); 3023 3024 // Copy the new stack pointer back. 3025 Chain = DAG.getCopyToReg(Chain, DL, SPReg, NewSP); 3026 3027 // The allocated data lives above the 160 bytes allocated for the standard 3028 // frame, plus any outgoing stack arguments. We don't know how much that 3029 // amounts to yet, so emit a special ADJDYNALLOC placeholder. 3030 SDValue ArgAdjust = DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 3031 SDValue Result = DAG.getNode(ISD::ADD, DL, MVT::i64, NewSP, ArgAdjust); 3032 3033 // Dynamically realign if needed. 3034 if (RequiredAlign > StackAlign) { 3035 Result = 3036 DAG.getNode(ISD::ADD, DL, MVT::i64, Result, 3037 DAG.getConstant(ExtraAlignSpace, DL, MVT::i64)); 3038 Result = 3039 DAG.getNode(ISD::AND, DL, MVT::i64, Result, 3040 DAG.getConstant(~(RequiredAlign - 1), DL, MVT::i64)); 3041 } 3042 3043 if (StoreBackchain) 3044 Chain = DAG.getStore(Chain, DL, Backchain, NewSP, MachinePointerInfo()); 3045 3046 SDValue Ops[2] = { Result, Chain }; 3047 return DAG.getMergeValues(Ops, DL); 3048 } 3049 3050 SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET( 3051 SDValue Op, SelectionDAG &DAG) const { 3052 SDLoc DL(Op); 3053 3054 return DAG.getNode(SystemZISD::ADJDYNALLOC, DL, MVT::i64); 3055 } 3056 3057 SDValue SystemZTargetLowering::lowerSMUL_LOHI(SDValue Op, 3058 SelectionDAG &DAG) const { 3059 EVT VT = Op.getValueType(); 3060 SDLoc DL(Op); 3061 SDValue Ops[2]; 3062 if (is32Bit(VT)) 3063 // Just do a normal 64-bit multiplication and extract the results.