1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines an instruction selector for the SystemZ target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "SystemZTargetMachine.h" 15 #include "llvm/Analysis/AliasAnalysis.h" 16 #include "llvm/CodeGen/SelectionDAGISel.h" 17 #include "llvm/Support/Debug.h" 18 #include "llvm/Support/raw_ostream.h" 19 20 using namespace llvm; 21 22 #define DEBUG_TYPE "systemz-isel" 23 24 namespace { 25 // Used to build addressing modes. 26 struct SystemZAddressingMode { 27 // The shape of the address. 28 enum AddrForm { 29 // base+displacement 30 FormBD, 31 32 // base+displacement+index for load and store operands 33 FormBDXNormal, 34 35 // base+displacement+index for load address operands 36 FormBDXLA, 37 38 // base+displacement+index+ADJDYNALLOC 39 FormBDXDynAlloc 40 }; 41 AddrForm Form; 42 43 // The type of displacement. The enum names here correspond directly 44 // to the definitions in SystemZOperand.td. We could split them into 45 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it. 46 enum DispRange { 47 Disp12Only, 48 Disp12Pair, 49 Disp20Only, 50 Disp20Only128, 51 Disp20Pair 52 }; 53 DispRange DR; 54 55 // The parts of the address. The address is equivalent to: 56 // 57 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0) 58 SDValue Base; 59 int64_t Disp; 60 SDValue Index; 61 bool IncludesDynAlloc; 62 63 SystemZAddressingMode(AddrForm form, DispRange dr) 64 : Form(form), DR(dr), Base(), Disp(0), Index(), 65 IncludesDynAlloc(false) {} 66 67 // True if the address can have an index register. 68 bool hasIndexField() { return Form != FormBD; } 69 70 // True if the address can (and must) include ADJDYNALLOC. 71 bool isDynAlloc() { return Form == FormBDXDynAlloc; } 72 73 void dump() { 74 errs() << "SystemZAddressingMode " << this << '\n'; 75 76 errs() << " Base "; 77 if (Base.getNode()) 78 Base.getNode()->dump(); 79 else 80 errs() << "null\n"; 81 82 if (hasIndexField()) { 83 errs() << " Index "; 84 if (Index.getNode()) 85 Index.getNode()->dump(); 86 else 87 errs() << "null\n"; 88 } 89 90 errs() << " Disp " << Disp; 91 if (IncludesDynAlloc) 92 errs() << " + ADJDYNALLOC"; 93 errs() << '\n'; 94 } 95 }; 96 97 // Return a mask with Count low bits set. 98 static uint64_t allOnes(unsigned int Count) { 99 assert(Count <= 64); 100 if (Count > 63) 101 return UINT64_MAX; 102 return (uint64_t(1) << Count) - 1; 103 } 104 105 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation 106 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and 107 // Rotate (I5). The combined operand value is effectively: 108 // 109 // (or (rotl Input, Rotate), ~Mask) 110 // 111 // for RNSBG and: 112 // 113 // (and (rotl Input, Rotate), Mask) 114 // 115 // otherwise. The output value has BitSize bits, although Input may be 116 // narrower (in which case the upper bits are don't care). 117 struct RxSBGOperands { 118 RxSBGOperands(unsigned Op, SDValue N) 119 : Opcode(Op), BitSize(N.getValueType().getSizeInBits()), 120 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63), 121 Rotate(0) {} 122 123 unsigned Opcode; 124 unsigned BitSize; 125 uint64_t Mask; 126 SDValue Input; 127 unsigned Start; 128 unsigned End; 129 unsigned Rotate; 130 }; 131 132 class SystemZDAGToDAGISel : public SelectionDAGISel { 133 const SystemZSubtarget *Subtarget; 134 135 // Used by SystemZOperands.td to create integer constants. 136 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const { 137 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0)); 138 } 139 140 const SystemZTargetMachine &getTargetMachine() const { 141 return static_cast<const SystemZTargetMachine &>(TM); 142 } 143 144 const SystemZInstrInfo *getInstrInfo() const { 145 return Subtarget->getInstrInfo(); 146 } 147 148 // Try to fold more of the base or index of AM into AM, where IsBase 149 // selects between the base and index. 150 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const; 151 152 // Try to describe N in AM, returning true on success. 153 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const; 154 155 // Extract individual target operands from matched address AM. 156 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 157 SDValue &Base, SDValue &Disp) const; 158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 159 SDValue &Base, SDValue &Disp, SDValue &Index) const; 160 161 // Try to match Addr as a FormBD address with displacement type DR. 162 // Return true on success, storing the base and displacement in 163 // Base and Disp respectively. 164 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 165 SDValue &Base, SDValue &Disp) const; 166 167 // Try to match Addr as a FormBDX address with displacement type DR. 168 // Return true on success and if the result had no index. Store the 169 // base and displacement in Base and Disp respectively. 170 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 171 SDValue &Base, SDValue &Disp) const; 172 173 // Try to match Addr as a FormBDX* address of form Form with 174 // displacement type DR. Return true on success, storing the base, 175 // displacement and index in Base, Disp and Index respectively. 176 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form, 177 SystemZAddressingMode::DispRange DR, SDValue Addr, 178 SDValue &Base, SDValue &Disp, SDValue &Index) const; 179 180 // PC-relative address matching routines used by SystemZOperands.td. 181 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const { 182 if (SystemZISD::isPCREL(Addr.getOpcode())) { 183 Target = Addr.getOperand(0); 184 return true; 185 } 186 return false; 187 } 188 189 // BD matching routines used by SystemZOperands.td. 190 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 191 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp); 192 } 193 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 194 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 195 } 196 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 197 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp); 198 } 199 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 200 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 201 } 202 203 // MVI matching routines used by SystemZOperands.td. 204 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 205 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 206 } 207 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 208 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 209 } 210 211 // BDX matching routines used by SystemZOperands.td. 212 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 213 SDValue &Index) const { 214 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 215 SystemZAddressingMode::Disp12Only, 216 Addr, Base, Disp, Index); 217 } 218 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 219 SDValue &Index) const { 220 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 221 SystemZAddressingMode::Disp12Pair, 222 Addr, Base, Disp, Index); 223 } 224 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 225 SDValue &Index) const { 226 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc, 227 SystemZAddressingMode::Disp12Only, 228 Addr, Base, Disp, Index); 229 } 230 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp, 231 SDValue &Index) const { 232 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 233 SystemZAddressingMode::Disp20Only, 234 Addr, Base, Disp, Index); 235 } 236 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp, 237 SDValue &Index) const { 238 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 239 SystemZAddressingMode::Disp20Only128, 240 Addr, Base, Disp, Index); 241 } 242 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 243 SDValue &Index) const { 244 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 245 SystemZAddressingMode::Disp20Pair, 246 Addr, Base, Disp, Index); 247 } 248 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 249 SDValue &Index) const { 250 return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 251 SystemZAddressingMode::Disp12Pair, 252 Addr, Base, Disp, Index); 253 } 254 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 255 SDValue &Index) const { 256 return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 257 SystemZAddressingMode::Disp20Pair, 258 Addr, Base, Disp, Index); 259 } 260 261 // Try to match Addr as an address with a base, 12-bit displacement 262 // and index, where the index is element Elem of a vector. 263 // Return true on success, storing the base, displacement and vector 264 // in Base, Disp and Index respectively. 265 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base, 266 SDValue &Disp, SDValue &Index) const; 267 268 // Check whether (or Op (and X InsertMask)) is effectively an insertion 269 // of X into bits InsertMask of some Y != Op. Return true if so and 270 // set Op to that Y. 271 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const; 272 273 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used. 274 // Return true on success. 275 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const; 276 277 // Try to fold some of RxSBG.Input into other fields of RxSBG. 278 // Return true on success. 279 bool expandRxSBG(RxSBGOperands &RxSBG) const; 280 281 // Return an undefined value of type VT. 282 SDValue getUNDEF(SDLoc DL, EVT VT) const; 283 284 // Convert N to VT, if it isn't already. 285 SDValue convertTo(SDLoc DL, EVT VT, SDValue N) const; 286 287 // Try to implement AND or shift node N using RISBG with the zero flag set. 288 // Return the selected node on success, otherwise return null. 289 SDNode *tryRISBGZero(SDNode *N); 290 291 // Try to use RISBG or Opcode to implement OR or XOR node N. 292 // Return the selected node on success, otherwise return null. 293 SDNode *tryRxSBG(SDNode *N, unsigned Opcode); 294 295 // If Op0 is null, then Node is a constant that can be loaded using: 296 // 297 // (Opcode UpperVal LowerVal) 298 // 299 // If Op0 is nonnull, then Node can be implemented using: 300 // 301 // (Opcode (Opcode Op0 UpperVal) LowerVal) 302 SDNode *splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0, 303 uint64_t UpperVal, uint64_t LowerVal); 304 305 // Try to use gather instruction Opcode to implement vector insertion N. 306 SDNode *tryGather(SDNode *N, unsigned Opcode); 307 308 // Try to use scatter instruction Opcode to implement store Store. 309 SDNode *tryScatter(StoreSDNode *Store, unsigned Opcode); 310 311 // Return true if Load and Store are loads and stores of the same size 312 // and are guaranteed not to overlap. Such operations can be implemented 313 // using block (SS-format) instructions. 314 // 315 // Partial overlap would lead to incorrect code, since the block operations 316 // are logically bytewise, even though they have a fast path for the 317 // non-overlapping case. We also need to avoid full overlap (i.e. two 318 // addresses that might be equal at run time) because although that case 319 // would be handled correctly, it might be implemented by millicode. 320 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const; 321 322 // N is a (store (load Y), X) pattern. Return true if it can use an MVC 323 // from Y to X. 324 bool storeLoadCanUseMVC(SDNode *N) const; 325 326 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true 327 // if A[1 - I] == X and if N can use a block operation like NC from A[I] 328 // to X. 329 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const; 330 331 public: 332 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel) 333 : SelectionDAGISel(TM, OptLevel) {} 334 335 bool runOnMachineFunction(MachineFunction &MF) override { 336 Subtarget = &MF.getSubtarget<SystemZSubtarget>(); 337 return SelectionDAGISel::runOnMachineFunction(MF); 338 } 339 340 // Override MachineFunctionPass. 341 const char *getPassName() const override { 342 return "SystemZ DAG->DAG Pattern Instruction Selection"; 343 } 344 345 // Override SelectionDAGISel. 346 SDNode *Select(SDNode *Node) override; 347 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 348 std::vector<SDValue> &OutOps) override; 349 350 // Include the pieces autogenerated from the target description. 351 #include "SystemZGenDAGISel.inc" 352 }; 353 } // end anonymous namespace 354 355 FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM, 356 CodeGenOpt::Level OptLevel) { 357 return new SystemZDAGToDAGISel(TM, OptLevel); 358 } 359 360 // Return true if Val should be selected as a displacement for an address 361 // with range DR. Here we're interested in the range of both the instruction 362 // described by DR and of any pairing instruction. 363 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 364 switch (DR) { 365 case SystemZAddressingMode::Disp12Only: 366 return isUInt<12>(Val); 367 368 case SystemZAddressingMode::Disp12Pair: 369 case SystemZAddressingMode::Disp20Only: 370 case SystemZAddressingMode::Disp20Pair: 371 return isInt<20>(Val); 372 373 case SystemZAddressingMode::Disp20Only128: 374 return isInt<20>(Val) && isInt<20>(Val + 8); 375 } 376 llvm_unreachable("Unhandled displacement range"); 377 } 378 379 // Change the base or index in AM to Value, where IsBase selects 380 // between the base and index. 381 static void changeComponent(SystemZAddressingMode &AM, bool IsBase, 382 SDValue Value) { 383 if (IsBase) 384 AM.Base = Value; 385 else 386 AM.Index = Value; 387 } 388 389 // The base or index of AM is equivalent to Value + ADJDYNALLOC, 390 // where IsBase selects between the base and index. Try to fold the 391 // ADJDYNALLOC into AM. 392 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, 393 SDValue Value) { 394 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) { 395 changeComponent(AM, IsBase, Value); 396 AM.IncludesDynAlloc = true; 397 return true; 398 } 399 return false; 400 } 401 402 // The base of AM is equivalent to Base + Index. Try to use Index as 403 // the index register. 404 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, 405 SDValue Index) { 406 if (AM.hasIndexField() && !AM.Index.getNode()) { 407 AM.Base = Base; 408 AM.Index = Index; 409 return true; 410 } 411 return false; 412 } 413 414 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects 415 // between the base and index. Try to fold Op1 into AM's displacement. 416 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, 417 SDValue Op0, uint64_t Op1) { 418 // First try adjusting the displacement. 419 int64_t TestDisp = AM.Disp + Op1; 420 if (selectDisp(AM.DR, TestDisp)) { 421 changeComponent(AM, IsBase, Op0); 422 AM.Disp = TestDisp; 423 return true; 424 } 425 426 // We could consider forcing the displacement into a register and 427 // using it as an index, but it would need to be carefully tuned. 428 return false; 429 } 430 431 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM, 432 bool IsBase) const { 433 SDValue N = IsBase ? AM.Base : AM.Index; 434 unsigned Opcode = N.getOpcode(); 435 if (Opcode == ISD::TRUNCATE) { 436 N = N.getOperand(0); 437 Opcode = N.getOpcode(); 438 } 439 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) { 440 SDValue Op0 = N.getOperand(0); 441 SDValue Op1 = N.getOperand(1); 442 443 unsigned Op0Code = Op0->getOpcode(); 444 unsigned Op1Code = Op1->getOpcode(); 445 446 if (Op0Code == SystemZISD::ADJDYNALLOC) 447 return expandAdjDynAlloc(AM, IsBase, Op1); 448 if (Op1Code == SystemZISD::ADJDYNALLOC) 449 return expandAdjDynAlloc(AM, IsBase, Op0); 450 451 if (Op0Code == ISD::Constant) 452 return expandDisp(AM, IsBase, Op1, 453 cast<ConstantSDNode>(Op0)->getSExtValue()); 454 if (Op1Code == ISD::Constant) 455 return expandDisp(AM, IsBase, Op0, 456 cast<ConstantSDNode>(Op1)->getSExtValue()); 457 458 if (IsBase && expandIndex(AM, Op0, Op1)) 459 return true; 460 } 461 if (Opcode == SystemZISD::PCREL_OFFSET) { 462 SDValue Full = N.getOperand(0); 463 SDValue Base = N.getOperand(1); 464 SDValue Anchor = Base.getOperand(0); 465 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() - 466 cast<GlobalAddressSDNode>(Anchor)->getOffset()); 467 return expandDisp(AM, IsBase, Base, Offset); 468 } 469 return false; 470 } 471 472 // Return true if an instruction with displacement range DR should be 473 // used for displacement value Val. selectDisp(DR, Val) must already hold. 474 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 475 assert(selectDisp(DR, Val) && "Invalid displacement"); 476 switch (DR) { 477 case SystemZAddressingMode::Disp12Only: 478 case SystemZAddressingMode::Disp20Only: 479 case SystemZAddressingMode::Disp20Only128: 480 return true; 481 482 case SystemZAddressingMode::Disp12Pair: 483 // Use the other instruction if the displacement is too large. 484 return isUInt<12>(Val); 485 486 case SystemZAddressingMode::Disp20Pair: 487 // Use the other instruction if the displacement is small enough. 488 return !isUInt<12>(Val); 489 } 490 llvm_unreachable("Unhandled displacement range"); 491 } 492 493 // Return true if Base + Disp + Index should be performed by LA(Y). 494 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) { 495 // Don't use LA(Y) for constants. 496 if (!Base) 497 return false; 498 499 // Always use LA(Y) for frame addresses, since we know that the destination 500 // register is almost always (perhaps always) going to be different from 501 // the frame register. 502 if (Base->getOpcode() == ISD::FrameIndex) 503 return true; 504 505 if (Disp) { 506 // Always use LA(Y) if there is a base, displacement and index. 507 if (Index) 508 return true; 509 510 // Always use LA if the displacement is small enough. It should always 511 // be no worse than AGHI (and better if it avoids a move). 512 if (isUInt<12>(Disp)) 513 return true; 514 515 // For similar reasons, always use LAY if the constant is too big for AGHI. 516 // LAY should be no worse than AGFI. 517 if (!isInt<16>(Disp)) 518 return true; 519 } else { 520 // Don't use LA for plain registers. 521 if (!Index) 522 return false; 523 524 // Don't use LA for plain addition if the index operand is only used 525 // once. It should be a natural two-operand addition in that case. 526 if (Index->hasOneUse()) 527 return false; 528 529 // Prefer addition if the second operation is sign-extended, in the 530 // hope of using AGF. 531 unsigned IndexOpcode = Index->getOpcode(); 532 if (IndexOpcode == ISD::SIGN_EXTEND || 533 IndexOpcode == ISD::SIGN_EXTEND_INREG) 534 return false; 535 } 536 537 // Don't use LA for two-operand addition if either operand is only 538 // used once. The addition instructions are better in that case. 539 if (Base->hasOneUse()) 540 return false; 541 542 return true; 543 } 544 545 // Return true if Addr is suitable for AM, updating AM if so. 546 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr, 547 SystemZAddressingMode &AM) const { 548 // Start out assuming that the address will need to be loaded separately, 549 // then try to extend it as much as we can. 550 AM.Base = Addr; 551 552 // First try treating the address as a constant. 553 if (Addr.getOpcode() == ISD::Constant && 554 expandDisp(AM, true, SDValue(), 555 cast<ConstantSDNode>(Addr)->getSExtValue())) 556 ; 557 else 558 // Otherwise try expanding each component. 559 while (expandAddress(AM, true) || 560 (AM.Index.getNode() && expandAddress(AM, false))) 561 continue; 562 563 // Reject cases where it isn't profitable to use LA(Y). 564 if (AM.Form == SystemZAddressingMode::FormBDXLA && 565 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode())) 566 return false; 567 568 // Reject cases where the other instruction in a pair should be used. 569 if (!isValidDisp(AM.DR, AM.Disp)) 570 return false; 571 572 // Make sure that ADJDYNALLOC is included where necessary. 573 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) 574 return false; 575 576 DEBUG(AM.dump()); 577 return true; 578 } 579 580 // Insert a node into the DAG at least before Pos. This will reposition 581 // the node as needed, and will assign it a node ID that is <= Pos's ID. 582 // Note that this does *not* preserve the uniqueness of node IDs! 583 // The selection DAG must no longer depend on their uniqueness when this 584 // function is used. 585 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) { 586 if (N.getNode()->getNodeId() == -1 || 587 N.getNode()->getNodeId() > Pos->getNodeId()) { 588 DAG->RepositionNode(Pos->getIterator(), N.getNode()); 589 N.getNode()->setNodeId(Pos->getNodeId()); 590 } 591 } 592 593 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 594 EVT VT, SDValue &Base, 595 SDValue &Disp) const { 596 Base = AM.Base; 597 if (!Base.getNode()) 598 // Register 0 means "no base". This is mostly useful for shifts. 599 Base = CurDAG->getRegister(0, VT); 600 else if (Base.getOpcode() == ISD::FrameIndex) { 601 // Lower a FrameIndex to a TargetFrameIndex. 602 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex(); 603 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT); 604 } else if (Base.getValueType() != VT) { 605 // Truncate values from i64 to i32, for shifts. 606 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 && 607 "Unexpected truncation"); 608 SDLoc DL(Base); 609 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base); 610 insertDAGNode(CurDAG, Base.getNode(), Trunc); 611 Base = Trunc; 612 } 613 614 // Lower the displacement to a TargetConstant. 615 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT); 616 } 617 618 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 619 EVT VT, SDValue &Base, 620 SDValue &Disp, 621 SDValue &Index) const { 622 getAddressOperands(AM, VT, Base, Disp); 623 624 Index = AM.Index; 625 if (!Index.getNode()) 626 // Register 0 means "no index". 627 Index = CurDAG->getRegister(0, VT); 628 } 629 630 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR, 631 SDValue Addr, SDValue &Base, 632 SDValue &Disp) const { 633 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR); 634 if (!selectAddress(Addr, AM)) 635 return false; 636 637 getAddressOperands(AM, Addr.getValueType(), Base, Disp); 638 return true; 639 } 640 641 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR, 642 SDValue Addr, SDValue &Base, 643 SDValue &Disp) const { 644 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR); 645 if (!selectAddress(Addr, AM) || AM.Index.getNode()) 646 return false; 647 648 getAddressOperands(AM, Addr.getValueType(), Base, Disp); 649 return true; 650 } 651 652 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form, 653 SystemZAddressingMode::DispRange DR, 654 SDValue Addr, SDValue &Base, 655 SDValue &Disp, SDValue &Index) const { 656 SystemZAddressingMode AM(Form, DR); 657 if (!selectAddress(Addr, AM)) 658 return false; 659 660 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index); 661 return true; 662 } 663 664 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem, 665 SDValue &Base, 666 SDValue &Disp, 667 SDValue &Index) const { 668 SDValue Regs[2]; 669 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) && 670 Regs[0].getNode() && Regs[1].getNode()) { 671 for (unsigned int I = 0; I < 2; ++I) { 672 Base = Regs[I]; 673 Index = Regs[1 - I]; 674 // We can't tell here whether the index vector has the right type 675 // for the access; the caller needs to do that instead. 676 if (Index.getOpcode() == ISD::ZERO_EXTEND) 677 Index = Index.getOperand(0); 678 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 679 Index.getOperand(1) == Elem) { 680 Index = Index.getOperand(0); 681 return true; 682 } 683 } 684 } 685 return false; 686 } 687 688 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op, 689 uint64_t InsertMask) const { 690 // We're only interested in cases where the insertion is into some operand 691 // of Op, rather than into Op itself. The only useful case is an AND. 692 if (Op.getOpcode() != ISD::AND) 693 return false; 694 695 // We need a constant mask. 696 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode()); 697 if (!MaskNode) 698 return false; 699 700 // It's not an insertion of Op.getOperand(0) if the two masks overlap. 701 uint64_t AndMask = MaskNode->getZExtValue(); 702 if (InsertMask & AndMask) 703 return false; 704 705 // It's only an insertion if all bits are covered or are known to be zero. 706 // The inner check covers all cases but is more expensive. 707 uint64_t Used = allOnes(Op.getValueType().getSizeInBits()); 708 if (Used != (AndMask | InsertMask)) { 709 APInt KnownZero, KnownOne; 710 CurDAG->computeKnownBits(Op.getOperand(0), KnownZero, KnownOne); 711 if (Used != (AndMask | InsertMask | KnownZero.getZExtValue())) 712 return false; 713 } 714 715 Op = Op.getOperand(0); 716 return true; 717 } 718 719 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG, 720 uint64_t Mask) const { 721 const SystemZInstrInfo *TII = getInstrInfo(); 722 if (RxSBG.Rotate != 0) 723 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)); 724 Mask &= RxSBG.Mask; 725 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) { 726 RxSBG.Mask = Mask; 727 return true; 728 } 729 return false; 730 } 731 732 // Return true if any bits of (RxSBG.Input & Mask) are significant. 733 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) { 734 // Rotate the mask in the same way as RxSBG.Input is rotated. 735 if (RxSBG.Rotate != 0) 736 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate))); 737 return (Mask & RxSBG.Mask) != 0; 738 } 739 740 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const { 741 SDValue N = RxSBG.Input; 742 unsigned Opcode = N.getOpcode(); 743 switch (Opcode) { 744 case ISD::AND: { 745 if (RxSBG.Opcode == SystemZ::RNSBG) 746 return false; 747 748 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 749 if (!MaskNode) 750 return false; 751 752 SDValue Input = N.getOperand(0); 753 uint64_t Mask = MaskNode->getZExtValue(); 754 if (!refineRxSBGMask(RxSBG, Mask)) { 755 // If some bits of Input are already known zeros, those bits will have 756 // been removed from the mask. See if adding them back in makes the 757 // mask suitable. 758 APInt KnownZero, KnownOne; 759 CurDAG->computeKnownBits(Input, KnownZero, KnownOne); 760 Mask |= KnownZero.getZExtValue(); 761 if (!refineRxSBGMask(RxSBG, Mask)) 762 return false; 763 } 764 RxSBG.Input = Input; 765 return true; 766 } 767 768 case ISD::OR: { 769 if (RxSBG.Opcode != SystemZ::RNSBG) 770 return false; 771 772 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 773 if (!MaskNode) 774 return false; 775 776 SDValue Input = N.getOperand(0); 777 uint64_t Mask = ~MaskNode->getZExtValue(); 778 if (!refineRxSBGMask(RxSBG, Mask)) { 779 // If some bits of Input are already known ones, those bits will have 780 // been removed from the mask. See if adding them back in makes the 781 // mask suitable. 782 APInt KnownZero, KnownOne; 783 CurDAG->computeKnownBits(Input, KnownZero, KnownOne); 784 Mask &= ~KnownOne.getZExtValue(); 785 if (!refineRxSBGMask(RxSBG, Mask)) 786 return false; 787 } 788 RxSBG.Input = Input; 789 return true; 790 } 791 792 case ISD::ROTL: { 793 // Any 64-bit rotate left can be merged into the RxSBG. 794 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64) 795 return false; 796 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 797 if (!CountNode) 798 return false; 799 800 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63; 801 RxSBG.Input = N.getOperand(0); 802 return true; 803 } 804 805 case ISD::ANY_EXTEND: 806 // Bits above the extended operand are don't-care. 807 RxSBG.Input = N.getOperand(0); 808 return true; 809 810 case ISD::ZERO_EXTEND: 811 if (RxSBG.Opcode != SystemZ::RNSBG) { 812 // Restrict the mask to the extended operand. 813 unsigned InnerBitSize = N.getOperand(0).getValueType().getSizeInBits(); 814 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize))) 815 return false; 816 817 RxSBG.Input = N.getOperand(0); 818 return true; 819 } 820 // Fall through. 821 822 case ISD::SIGN_EXTEND: { 823 // Check that the extension bits are don't-care (i.e. are masked out 824 // by the final mask). 825 unsigned InnerBitSize = N.getOperand(0).getValueType().getSizeInBits(); 826 if (maskMatters(RxSBG, allOnes(RxSBG.BitSize) - allOnes(InnerBitSize))) 827 return false; 828 829 RxSBG.Input = N.getOperand(0); 830 return true; 831 } 832 833 case ISD::SHL: { 834 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 835 if (!CountNode) 836 return false; 837 838 uint64_t Count = CountNode->getZExtValue(); 839 unsigned BitSize = N.getValueType().getSizeInBits(); 840 if (Count < 1 || Count >= BitSize) 841 return false; 842 843 if (RxSBG.Opcode == SystemZ::RNSBG) { 844 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom 845 // count bits from RxSBG.Input are ignored. 846 if (maskMatters(RxSBG, allOnes(Count))) 847 return false; 848 } else { 849 // Treat (shl X, count) as (and (rotl X, count), ~0<<count). 850 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count)) 851 return false; 852 } 853 854 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63; 855 RxSBG.Input = N.getOperand(0); 856 return true; 857 } 858 859 case ISD::SRL: 860 case ISD::SRA: { 861 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 862 if (!CountNode) 863 return false; 864 865 uint64_t Count = CountNode->getZExtValue(); 866 unsigned BitSize = N.getValueType().getSizeInBits(); 867 if (Count < 1 || Count >= BitSize) 868 return false; 869 870 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) { 871 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top 872 // count bits from RxSBG.Input are ignored. 873 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count))) 874 return false; 875 } else { 876 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count), 877 // which is similar to SLL above. 878 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count))) 879 return false; 880 } 881 882 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63; 883 RxSBG.Input = N.getOperand(0); 884 return true; 885 } 886 default: 887 return false; 888 } 889 } 890 891 SDValue SystemZDAGToDAGISel::getUNDEF(SDLoc DL, EVT VT) const { 892 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT); 893 return SDValue(N, 0); 894 } 895 896 SDValue SystemZDAGToDAGISel::convertTo(SDLoc DL, EVT VT, SDValue N) const { 897 if (N.getValueType() == MVT::i32 && VT == MVT::i64) 898 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32, 899 DL, VT, getUNDEF(DL, MVT::i64), N); 900 if (N.getValueType() == MVT::i64 && VT == MVT::i32) 901 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N); 902 assert(N.getValueType() == VT && "Unexpected value types"); 903 return N; 904 } 905 906 SDNode *SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) { 907 SDLoc DL(N); 908 EVT VT = N->getValueType(0); 909 if (!VT.isInteger() || VT.getSizeInBits() > 64) 910 return nullptr; 911 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0)); 912 unsigned Count = 0; 913 while (expandRxSBG(RISBG)) 914 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND) 915 Count += 1; 916 if (Count == 0) 917 return nullptr; 918 if (Count == 1) { 919 // Prefer to use normal shift instructions over RISBG, since they can handle 920 // all cases and are sometimes shorter. 921 if (N->getOpcode() != ISD::AND) 922 return nullptr; 923 924 // Prefer register extensions like LLC over RISBG. Also prefer to start 925 // out with normal ANDs if one instruction would be enough. We can convert 926 // these ANDs into an RISBG later if a three-address instruction is useful. 927 if (VT == MVT::i32 || 928 RISBG.Mask == 0xff || 929 RISBG.Mask == 0xffff || 930 SystemZ::isImmLF(~RISBG.Mask) || 931 SystemZ::isImmHF(~RISBG.Mask)) { 932 // Force the new mask into the DAG, since it may include known-one bits. 933 auto *MaskN = cast<ConstantSDNode>(N->getOperand(1).getNode()); 934 if (MaskN->getZExtValue() != RISBG.Mask) { 935 SDValue NewMask = CurDAG->getConstant(RISBG.Mask, DL, VT); 936 N = CurDAG->UpdateNodeOperands(N, N->getOperand(0), NewMask); 937 return SelectCode(N); 938 } 939 return nullptr; 940 } 941 } 942 943 // If the RISBG operands require no rotation and just masks the bottom 944 // 8/16 bits, attempt to convert this to a LLC zero extension. 945 if (RISBG.Rotate == 0 && (RISBG.Mask == 0xff || RISBG.Mask == 0xffff)) { 946 unsigned OpCode = (RISBG.Mask == 0xff ? SystemZ::LLGCR : SystemZ::LLGHR); 947 if (VT == MVT::i32) { 948 if (Subtarget->hasHighWord()) 949 OpCode = (RISBG.Mask == 0xff ? SystemZ::LLCRMux : SystemZ::LLHRMux); 950 else 951 OpCode = (RISBG.Mask == 0xff ? SystemZ::LLCR : SystemZ::LLHR); 952 } 953 954 SDValue In = convertTo(DL, VT, RISBG.Input); 955 N = CurDAG->getMachineNode(OpCode, DL, VT, In); 956 return convertTo(DL, VT, SDValue(N, 0)).getNode(); 957 } 958 959 unsigned Opcode = SystemZ::RISBG; 960 // Prefer RISBGN if available, since it does not clobber CC. 961 if (Subtarget->hasMiscellaneousExtensions()) 962 Opcode = SystemZ::RISBGN; 963 EVT OpcodeVT = MVT::i64; 964 if (VT == MVT::i32 && Subtarget->hasHighWord()) { 965 Opcode = SystemZ::RISBMux; 966 OpcodeVT = MVT::i32; 967 RISBG.Start &= 31; 968 RISBG.End &= 31; 969 } 970 SDValue Ops[5] = { 971 getUNDEF(DL, OpcodeVT), 972 convertTo(DL, OpcodeVT, RISBG.Input), 973 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32), 974 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32), 975 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32) 976 }; 977 N = CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops); 978 return convertTo(DL, VT, SDValue(N, 0)).getNode(); 979 } 980 981 SDNode *SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) { 982 SDLoc DL(N); 983 EVT VT = N->getValueType(0); 984 if (!VT.isInteger() || VT.getSizeInBits() > 64) 985 return nullptr; 986 // Try treating each operand of N as the second operand of the RxSBG 987 // and see which goes deepest. 988 RxSBGOperands RxSBG[] = { 989 RxSBGOperands(Opcode, N->getOperand(0)), 990 RxSBGOperands(Opcode, N->getOperand(1)) 991 }; 992 unsigned Count[] = { 0, 0 }; 993 for (unsigned I = 0; I < 2; ++I) 994 while (expandRxSBG(RxSBG[I])) 995 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND) 996 Count[I] += 1; 997 998 // Do nothing if neither operand is suitable. 999 if (Count[0] == 0 && Count[1] == 0) 1000 return nullptr; 1001 1002 // Pick the deepest second operand. 1003 unsigned I = Count[0] > Count[1] ? 0 : 1; 1004 SDValue Op0 = N->getOperand(I ^ 1); 1005 1006 // Prefer IC for character insertions from memory. 1007 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0) 1008 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode())) 1009 if (Load->getMemoryVT() == MVT::i8) 1010 return nullptr; 1011 1012 // See whether we can avoid an AND in the first operand by converting 1013 // ROSBG to RISBG. 1014 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) { 1015 Opcode = SystemZ::RISBG; 1016 // Prefer RISBGN if available, since it does not clobber CC. 1017 if (Subtarget->hasMiscellaneousExtensions()) 1018 Opcode = SystemZ::RISBGN; 1019 } 1020 1021 SDValue Ops[5] = { 1022 convertTo(DL, MVT::i64, Op0), 1023 convertTo(DL, MVT::i64, RxSBG[I].Input), 1024 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32), 1025 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32), 1026 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32) 1027 }; 1028 N = CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops); 1029 return convertTo(DL, VT, SDValue(N, 0)).getNode(); 1030 } 1031 1032 SDNode *SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node, 1033 SDValue Op0, uint64_t UpperVal, 1034 uint64_t LowerVal) { 1035 EVT VT = Node->getValueType(0); 1036 SDLoc DL(Node); 1037 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT); 1038 if (Op0.getNode()) 1039 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper); 1040 Upper = SDValue(Select(Upper.getNode()), 0); 1041 1042 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT); 1043 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower); 1044 return Or.getNode(); 1045 } 1046 1047 SDNode *SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) { 1048 SDValue ElemV = N->getOperand(2); 1049 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1050 if (!ElemN) 1051 return 0; 1052 1053 unsigned Elem = ElemN->getZExtValue(); 1054 EVT VT = N->getValueType(0); 1055 if (Elem >= VT.getVectorNumElements()) 1056 return 0; 1057 1058 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1)); 1059 if (!Load || !Load->hasOneUse()) 1060 return 0; 1061 if (Load->getMemoryVT().getSizeInBits() != 1062 Load->getValueType(0).getSizeInBits()) 1063 return 0; 1064 1065 SDValue Base, Disp, Index; 1066 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) || 1067 Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1068 return 0; 1069 1070 SDLoc DL(Load); 1071 SDValue Ops[] = { 1072 N->getOperand(0), Base, Disp, Index, 1073 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain() 1074 }; 1075 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops); 1076 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1)); 1077 return Res; 1078 } 1079 1080 SDNode *SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) { 1081 SDValue Value = Store->getValue(); 1082 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1083 return 0; 1084 if (Store->getMemoryVT().getSizeInBits() != 1085 Value.getValueType().getSizeInBits()) 1086 return 0; 1087 1088 SDValue ElemV = Value.getOperand(1); 1089 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1090 if (!ElemN) 1091 return 0; 1092 1093 SDValue Vec = Value.getOperand(0); 1094 EVT VT = Vec.getValueType(); 1095 unsigned Elem = ElemN->getZExtValue(); 1096 if (Elem >= VT.getVectorNumElements()) 1097 return 0; 1098 1099 SDValue Base, Disp, Index; 1100 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) || 1101 Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1102 return 0; 1103 1104 SDLoc DL(Store); 1105 SDValue Ops[] = { 1106 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32), 1107 Store->getChain() 1108 }; 1109 return CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 1110 } 1111 1112 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, 1113 LoadSDNode *Load) const { 1114 // Check that the two memory operands have the same size. 1115 if (Load->getMemoryVT() != Store->getMemoryVT()) 1116 return false; 1117 1118 // Volatility stops an access from being decomposed. 1119 if (Load->isVolatile() || Store->isVolatile()) 1120 return false; 1121 1122 // There's no chance of overlap if the load is invariant. 1123 if (Load->isInvariant()) 1124 return true; 1125 1126 // Otherwise we need to check whether there's an alias. 1127 const Value *V1 = Load->getMemOperand()->getValue(); 1128 const Value *V2 = Store->getMemOperand()->getValue(); 1129 if (!V1 || !V2) 1130 return false; 1131 1132 // Reject equality. 1133 uint64_t Size = Load->getMemoryVT().getStoreSize(); 1134 int64_t End1 = Load->getSrcValueOffset() + Size; 1135 int64_t End2 = Store->getSrcValueOffset() + Size; 1136 if (V1 == V2 && End1 == End2) 1137 return false; 1138 1139 return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()), 1140 MemoryLocation(V2, End2, Store->getAAInfo())); 1141 } 1142 1143 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const { 1144 auto *Store = cast<StoreSDNode>(N); 1145 auto *Load = cast<LoadSDNode>(Store->getValue()); 1146 1147 // Prefer not to use MVC if either address can use ... RELATIVE LONG 1148 // instructions. 1149 uint64_t Size = Load->getMemoryVT().getStoreSize(); 1150 if (Size > 1 && Size <= 8) { 1151 // Prefer LHRL, LRL and LGRL. 1152 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode())) 1153 return false; 1154 // Prefer STHRL, STRL and STGRL. 1155 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode())) 1156 return false; 1157 } 1158 1159 return canUseBlockOperation(Store, Load); 1160 } 1161 1162 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N, 1163 unsigned I) const { 1164 auto *StoreA = cast<StoreSDNode>(N); 1165 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I)); 1166 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I)); 1167 return !LoadA->isVolatile() && canUseBlockOperation(StoreA, LoadB); 1168 } 1169 1170 SDNode *SystemZDAGToDAGISel::Select(SDNode *Node) { 1171 // Dump information about the Node being selected 1172 DEBUG(errs() << "Selecting: "; Node->dump(CurDAG); errs() << "\n"); 1173 1174 // If we have a custom node, we already have selected! 1175 if (Node->isMachineOpcode()) { 1176 DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n"); 1177 Node->setNodeId(-1); 1178 return nullptr; 1179 } 1180 1181 unsigned Opcode = Node->getOpcode(); 1182 SDNode *ResNode = nullptr; 1183 switch (Opcode) { 1184 case ISD::OR: 1185 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1186 ResNode = tryRxSBG(Node, SystemZ::ROSBG); 1187 goto or_xor; 1188 1189 case ISD::XOR: 1190 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1191 ResNode = tryRxSBG(Node, SystemZ::RXSBG); 1192 // Fall through. 1193 or_xor: 1194 // If this is a 64-bit operation in which both 32-bit halves are nonzero, 1195 // split the operation into two. 1196 if (!ResNode && Node->getValueType(0) == MVT::i64) 1197 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) { 1198 uint64_t Val = Op1->getZExtValue(); 1199 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) 1200 Node = splitLargeImmediate(Opcode, Node, Node->getOperand(0), 1201 Val - uint32_t(Val), uint32_t(Val)); 1202 } 1203 break; 1204 1205 case ISD::AND: 1206 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1207 ResNode = tryRxSBG(Node, SystemZ::RNSBG); 1208 // Fall through. 1209 case ISD::ROTL: 1210 case ISD::SHL: 1211 case ISD::SRL: 1212 case ISD::ZERO_EXTEND: 1213 if (!ResNode) 1214 ResNode = tryRISBGZero(Node); 1215 break; 1216 1217 case ISD::Constant: 1218 // If this is a 64-bit constant that is out of the range of LLILF, 1219 // LLIHF and LGFI, split it into two 32-bit pieces. 1220 if (Node->getValueType(0) == MVT::i64) { 1221 uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue(); 1222 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) 1223 Node = splitLargeImmediate(ISD::OR, Node, SDValue(), 1224 Val - uint32_t(Val), uint32_t(Val)); 1225 } 1226 break; 1227 1228 case SystemZISD::SELECT_CCMASK: { 1229 SDValue Op0 = Node->getOperand(0); 1230 SDValue Op1 = Node->getOperand(1); 1231 // Prefer to put any load first, so that it can be matched as a 1232 // conditional load. 1233 if (Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) { 1234 SDValue CCValid = Node->getOperand(2); 1235 SDValue CCMask = Node->getOperand(3); 1236 uint64_t ConstCCValid = 1237 cast<ConstantSDNode>(CCValid.getNode())->getZExtValue(); 1238 uint64_t ConstCCMask = 1239 cast<ConstantSDNode>(CCMask.getNode())->getZExtValue(); 1240 // Invert the condition. 1241 CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node), 1242 CCMask.getValueType()); 1243 SDValue Op4 = Node->getOperand(4); 1244 Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4); 1245 } 1246 break; 1247 } 1248 1249 case ISD::INSERT_VECTOR_ELT: { 1250 EVT VT = Node->getValueType(0); 1251 unsigned ElemBitSize = VT.getVectorElementType().getSizeInBits(); 1252 if (ElemBitSize == 32) 1253 ResNode = tryGather(Node, SystemZ::VGEF); 1254 else if (ElemBitSize == 64) 1255 ResNode = tryGather(Node, SystemZ::VGEG); 1256 break; 1257 } 1258 1259 case ISD::STORE: { 1260 auto *Store = cast<StoreSDNode>(Node); 1261 unsigned ElemBitSize = Store->getValue().getValueType().getSizeInBits(); 1262 if (ElemBitSize == 32) 1263 ResNode = tryScatter(Store, SystemZ::VSCEF); 1264 else if (ElemBitSize == 64) 1265 ResNode = tryScatter(Store, SystemZ::VSCEG); 1266 break; 1267 } 1268 } 1269 1270 // Select the default instruction 1271 if (!ResNode) 1272 ResNode = SelectCode(Node); 1273 1274 DEBUG(errs() << "=> "; 1275 if (ResNode == nullptr || ResNode == Node) 1276 Node->dump(CurDAG); 1277 else 1278 ResNode->dump(CurDAG); 1279 errs() << "\n"; 1280 ); 1281 return ResNode; 1282 } 1283 1284 bool SystemZDAGToDAGISel:: 1285 SelectInlineAsmMemoryOperand(const SDValue &Op, 1286 unsigned ConstraintID, 1287 std::vector<SDValue> &OutOps) { 1288 switch(ConstraintID) { 1289 default: 1290 llvm_unreachable("Unexpected asm memory constraint"); 1291 case InlineAsm::Constraint_i: 1292 case InlineAsm::Constraint_m: 1293 case InlineAsm::Constraint_Q: 1294 case InlineAsm::Constraint_R: 1295 case InlineAsm::Constraint_S: 1296 case InlineAsm::Constraint_T: 1297 // Accept addresses with short displacements, which are compatible 1298 // with Q, R, S and T. But keep the index operand for future expansion. 1299 SDValue Base, Disp, Index; 1300 if (selectBDXAddr(SystemZAddressingMode::FormBD, 1301 SystemZAddressingMode::Disp12Only, 1302 Op, Base, Disp, Index)) { 1303 OutOps.push_back(Base); 1304 OutOps.push_back(Disp); 1305 OutOps.push_back(Index); 1306 return false; 1307 } 1308 break; 1309 } 1310 return true; 1311 } 1312