1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines an instruction selector for the ARM target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "ARM.h" 15 #include "ARMBaseInstrInfo.h" 16 #include "ARMTargetMachine.h" 17 #include "MCTargetDesc/ARMAddressingModes.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/CodeGen/MachineFunction.h" 20 #include "llvm/CodeGen/MachineInstrBuilder.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/CodeGen/SelectionDAG.h" 23 #include "llvm/CodeGen/SelectionDAGISel.h" 24 #include "llvm/IR/CallingConv.h" 25 #include "llvm/IR/Constants.h" 26 #include "llvm/IR/DerivedTypes.h" 27 #include "llvm/IR/Function.h" 28 #include "llvm/IR/Intrinsics.h" 29 #include "llvm/IR/LLVMContext.h" 30 #include "llvm/Support/CommandLine.h" 31 #include "llvm/Support/Compiler.h" 32 #include "llvm/Support/Debug.h" 33 #include "llvm/Support/ErrorHandling.h" 34 #include "llvm/Target/TargetLowering.h" 35 #include "llvm/Target/TargetOptions.h" 36 37 using namespace llvm; 38 39 #define DEBUG_TYPE "arm-isel" 40 41 static cl::opt<bool> 42 DisableShifterOp("disable-shifter-op", cl::Hidden, 43 cl::desc("Disable isel of shifter-op"), 44 cl::init(false)); 45 46 static cl::opt<bool> 47 CheckVMLxHazard("check-vmlx-hazard", cl::Hidden, 48 cl::desc("Check fp vmla / vmls hazard at isel time"), 49 cl::init(true)); 50 51 //===--------------------------------------------------------------------===// 52 /// ARMDAGToDAGISel - ARM specific code to select ARM machine 53 /// instructions for SelectionDAG operations. 54 /// 55 namespace { 56 57 enum AddrMode2Type { 58 AM2_BASE, // Simple AM2 (+-imm12) 59 AM2_SHOP // Shifter-op AM2 60 }; 61 62 class ARMDAGToDAGISel : public SelectionDAGISel { 63 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can 64 /// make the right decision when generating code for different targets. 65 const ARMSubtarget *Subtarget; 66 67 public: 68 explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel) 69 : SelectionDAGISel(tm, OptLevel) {} 70 71 bool runOnMachineFunction(MachineFunction &MF) override { 72 // Reset the subtarget each time through. 73 Subtarget = &MF.getTarget().getSubtarget<ARMSubtarget>(); 74 SelectionDAGISel::runOnMachineFunction(MF); 75 return true; 76 } 77 78 const char *getPassName() const override { 79 return "ARM Instruction Selection"; 80 } 81 82 void PreprocessISelDAG() override; 83 84 /// getI32Imm - Return a target constant of type i32 with the specified 85 /// value. 86 inline SDValue getI32Imm(unsigned Imm) { 87 return CurDAG->getTargetConstant(Imm, MVT::i32); 88 } 89 90 SDNode *Select(SDNode *N) override; 91 92 93 bool hasNoVMLxHazardUse(SDNode *N) const; 94 bool isShifterOpProfitable(const SDValue &Shift, 95 ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt); 96 bool SelectRegShifterOperand(SDValue N, SDValue &A, 97 SDValue &B, SDValue &C, 98 bool CheckProfitability = true); 99 bool SelectImmShifterOperand(SDValue N, SDValue &A, 100 SDValue &B, bool CheckProfitability = true); 101 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, 102 SDValue &B, SDValue &C) { 103 // Don't apply the profitability check 104 return SelectRegShifterOperand(N, A, B, C, false); 105 } 106 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, 107 SDValue &B) { 108 // Don't apply the profitability check 109 return SelectImmShifterOperand(N, A, B, false); 110 } 111 112 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 113 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc); 114 115 AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base, 116 SDValue &Offset, SDValue &Opc); 117 bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset, 118 SDValue &Opc) { 119 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE; 120 } 121 122 bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset, 123 SDValue &Opc) { 124 return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP; 125 } 126 127 bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset, 128 SDValue &Opc) { 129 SelectAddrMode2Worker(N, Base, Offset, Opc); 130 // return SelectAddrMode2ShOp(N, Base, Offset, Opc); 131 // This always matches one way or another. 132 return true; 133 } 134 135 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) { 136 const ConstantSDNode *CN = cast<ConstantSDNode>(N); 137 Pred = CurDAG->getTargetConstant(CN->getZExtValue(), MVT::i32); 138 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32); 139 return true; 140 } 141 142 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 143 SDValue &Offset, SDValue &Opc); 144 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 145 SDValue &Offset, SDValue &Opc); 146 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 147 SDValue &Offset, SDValue &Opc); 148 bool SelectAddrOffsetNone(SDValue N, SDValue &Base); 149 bool SelectAddrMode3(SDValue N, SDValue &Base, 150 SDValue &Offset, SDValue &Opc); 151 bool SelectAddrMode3Offset(SDNode *Op, SDValue N, 152 SDValue &Offset, SDValue &Opc); 153 bool SelectAddrMode5(SDValue N, SDValue &Base, 154 SDValue &Offset); 155 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align); 156 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset); 157 158 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label); 159 160 // Thumb Addressing Modes: 161 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset); 162 bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset, 163 unsigned Scale); 164 bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset); 165 bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset); 166 bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset); 167 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base, 168 SDValue &OffImm); 169 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 170 SDValue &OffImm); 171 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 172 SDValue &OffImm); 173 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 174 SDValue &OffImm); 175 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm); 176 177 // Thumb 2 Addressing Modes: 178 bool SelectT2ShifterOperandReg(SDValue N, 179 SDValue &BaseReg, SDValue &Opc); 180 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm); 181 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, 182 SDValue &OffImm); 183 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 184 SDValue &OffImm); 185 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base, 186 SDValue &OffReg, SDValue &ShImm); 187 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm); 188 189 inline bool is_so_imm(unsigned Imm) const { 190 return ARM_AM::getSOImmVal(Imm) != -1; 191 } 192 193 inline bool is_so_imm_not(unsigned Imm) const { 194 return ARM_AM::getSOImmVal(~Imm) != -1; 195 } 196 197 inline bool is_t2_so_imm(unsigned Imm) const { 198 return ARM_AM::getT2SOImmVal(Imm) != -1; 199 } 200 201 inline bool is_t2_so_imm_not(unsigned Imm) const { 202 return ARM_AM::getT2SOImmVal(~Imm) != -1; 203 } 204 205 // Include the pieces autogenerated from the target description. 206 #include "ARMGenDAGISel.inc" 207 208 private: 209 /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for 210 /// ARM. 211 SDNode *SelectARMIndexedLoad(SDNode *N); 212 SDNode *SelectT2IndexedLoad(SDNode *N); 213 214 /// SelectVLD - Select NEON load intrinsics. NumVecs should be 215 /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for 216 /// loads of D registers and even subregs and odd subregs of Q registers. 217 /// For NumVecs <= 2, QOpcodes1 is not used. 218 SDNode *SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 219 const uint16_t *DOpcodes, 220 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 221 222 /// SelectVST - Select NEON store intrinsics. NumVecs should 223 /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for 224 /// stores of D registers and even subregs and odd subregs of Q registers. 225 /// For NumVecs <= 2, QOpcodes1 is not used. 226 SDNode *SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 227 const uint16_t *DOpcodes, 228 const uint16_t *QOpcodes0, const uint16_t *QOpcodes1); 229 230 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should 231 /// be 2, 3 or 4. The opcode arrays specify the instructions used for 232 /// load/store of D registers and Q registers. 233 SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, 234 bool isUpdating, unsigned NumVecs, 235 const uint16_t *DOpcodes, const uint16_t *QOpcodes); 236 237 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs 238 /// should be 2, 3 or 4. The opcode array specifies the instructions used 239 /// for loading D registers. (Q registers are not supported.) 240 SDNode *SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs, 241 const uint16_t *Opcodes); 242 243 /// SelectVTBL - Select NEON VTBL and VTBX intrinsics. NumVecs should be 2, 244 /// 3 or 4. These are custom-selected so that a REG_SEQUENCE can be 245 /// generated to force the table registers to be consecutive. 246 SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc); 247 248 /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM. 249 SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned); 250 251 // Select special operations if node forms integer ABS pattern 252 SDNode *SelectABSOp(SDNode *N); 253 254 SDNode *SelectInlineAsm(SDNode *N); 255 256 SDNode *SelectConcatVector(SDNode *N); 257 258 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 259 /// inline asm expressions. 260 bool SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 261 std::vector<SDValue> &OutOps) override; 262 263 // Form pairs of consecutive R, S, D, or Q registers. 264 SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1); 265 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1); 266 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1); 267 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1); 268 269 // Form sequences of 4 consecutive S, D, or Q registers. 270 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 271 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 272 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3); 273 274 // Get the alignment operand for a NEON VLD or VST instruction. 275 SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector); 276 }; 277 } 278 279 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant 280 /// operand. If so Imm will receive the 32-bit value. 281 static bool isInt32Immediate(SDNode *N, unsigned &Imm) { 282 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) { 283 Imm = cast<ConstantSDNode>(N)->getZExtValue(); 284 return true; 285 } 286 return false; 287 } 288 289 // isInt32Immediate - This method tests to see if a constant operand. 290 // If so Imm will receive the 32 bit value. 291 static bool isInt32Immediate(SDValue N, unsigned &Imm) { 292 return isInt32Immediate(N.getNode(), Imm); 293 } 294 295 // isOpcWithIntImmediate - This method tests to see if the node is a specific 296 // opcode and that it has a immediate integer right operand. 297 // If so Imm will receive the 32 bit value. 298 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) { 299 return N->getOpcode() == Opc && 300 isInt32Immediate(N->getOperand(1).getNode(), Imm); 301 } 302 303 /// \brief Check whether a particular node is a constant value representable as 304 /// (N * Scale) where (N in [\p RangeMin, \p RangeMax). 305 /// 306 /// \param ScaledConstant [out] - On success, the pre-scaled constant value. 307 static bool isScaledConstantInRange(SDValue Node, int Scale, 308 int RangeMin, int RangeMax, 309 int &ScaledConstant) { 310 assert(Scale > 0 && "Invalid scale!"); 311 312 // Check that this is a constant. 313 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node); 314 if (!C) 315 return false; 316 317 ScaledConstant = (int) C->getZExtValue(); 318 if ((ScaledConstant % Scale) != 0) 319 return false; 320 321 ScaledConstant /= Scale; 322 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax; 323 } 324 325 void ARMDAGToDAGISel::PreprocessISelDAG() { 326 if (!Subtarget->hasV6T2Ops()) 327 return; 328 329 bool isThumb2 = Subtarget->isThumb(); 330 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 331 E = CurDAG->allnodes_end(); I != E; ) { 332 SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 333 334 if (N->getOpcode() != ISD::ADD) 335 continue; 336 337 // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with 338 // leading zeros, followed by consecutive set bits, followed by 1 or 2 339 // trailing zeros, e.g. 1020. 340 // Transform the expression to 341 // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number 342 // of trailing zeros of c2. The left shift would be folded as an shifter 343 // operand of 'add' and the 'and' and 'srl' would become a bits extraction 344 // node (UBFX). 345 346 SDValue N0 = N->getOperand(0); 347 SDValue N1 = N->getOperand(1); 348 unsigned And_imm = 0; 349 if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) { 350 if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm)) 351 std::swap(N0, N1); 352 } 353 if (!And_imm) 354 continue; 355 356 // Check if the AND mask is an immediate of the form: 000.....1111111100 357 unsigned TZ = countTrailingZeros(And_imm); 358 if (TZ != 1 && TZ != 2) 359 // Be conservative here. Shifter operands aren't always free. e.g. On 360 // Swift, left shifter operand of 1 / 2 for free but others are not. 361 // e.g. 362 // ubfx r3, r1, #16, #8 363 // ldr.w r3, [r0, r3, lsl #2] 364 // vs. 365 // mov.w r9, #1020 366 // and.w r2, r9, r1, lsr #14 367 // ldr r2, [r0, r2] 368 continue; 369 And_imm >>= TZ; 370 if (And_imm & (And_imm + 1)) 371 continue; 372 373 // Look for (and (srl X, c1), c2). 374 SDValue Srl = N1.getOperand(0); 375 unsigned Srl_imm = 0; 376 if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) || 377 (Srl_imm <= 2)) 378 continue; 379 380 // Make sure first operand is not a shifter operand which would prevent 381 // folding of the left shift. 382 SDValue CPTmp0; 383 SDValue CPTmp1; 384 SDValue CPTmp2; 385 if (isThumb2) { 386 if (SelectT2ShifterOperandReg(N0, CPTmp0, CPTmp1)) 387 continue; 388 } else { 389 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) || 390 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2)) 391 continue; 392 } 393 394 // Now make the transformation. 395 Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32, 396 Srl.getOperand(0), 397 CurDAG->getConstant(Srl_imm+TZ, MVT::i32)); 398 N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32, 399 Srl, CurDAG->getConstant(And_imm, MVT::i32)); 400 N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32, 401 N1, CurDAG->getConstant(TZ, MVT::i32)); 402 CurDAG->UpdateNodeOperands(N, N0, N1); 403 } 404 } 405 406 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS 407 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at 408 /// least on current ARM implementations) which should be avoidded. 409 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const { 410 if (OptLevel == CodeGenOpt::None) 411 return true; 412 413 if (!CheckVMLxHazard) 414 return true; 415 416 if (!Subtarget->isCortexA7() && !Subtarget->isCortexA8() && 417 !Subtarget->isCortexA9() && !Subtarget->isSwift()) 418 return true; 419 420 if (!N->hasOneUse()) 421 return false; 422 423 SDNode *Use = *N->use_begin(); 424 if (Use->getOpcode() == ISD::CopyToReg) 425 return true; 426 if (Use->isMachineOpcode()) { 427 const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>( 428 CurDAG->getTarget().getInstrInfo()); 429 430 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode()); 431 if (MCID.mayStore()) 432 return true; 433 unsigned Opcode = MCID.getOpcode(); 434 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) 435 return true; 436 // vmlx feeding into another vmlx. We actually want to unfold 437 // the use later in the MLxExpansion pass. e.g. 438 // vmla 439 // vmla (stall 8 cycles) 440 // 441 // vmul (5 cycles) 442 // vadd (5 cycles) 443 // vmla 444 // This adds up to about 18 - 19 cycles. 445 // 446 // vmla 447 // vmul (stall 4 cycles) 448 // vadd adds up to about 14 cycles. 449 return TII->isFpMLxInstruction(Opcode); 450 } 451 452 return false; 453 } 454 455 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift, 456 ARM_AM::ShiftOpc ShOpcVal, 457 unsigned ShAmt) { 458 if (!Subtarget->isLikeA9() && !Subtarget->isSwift()) 459 return true; 460 if (Shift.hasOneUse()) 461 return true; 462 // R << 2 is free. 463 return ShOpcVal == ARM_AM::lsl && 464 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1)); 465 } 466 467 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N, 468 SDValue &BaseReg, 469 SDValue &Opc, 470 bool CheckProfitability) { 471 if (DisableShifterOp) 472 return false; 473 474 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 475 476 // Don't match base register only case. That is matched to a separate 477 // lower complexity pattern with explicit register operand. 478 if (ShOpcVal == ARM_AM::no_shift) return false; 479 480 BaseReg = N.getOperand(0); 481 unsigned ShImmVal = 0; 482 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 483 if (!RHS) return false; 484 ShImmVal = RHS->getZExtValue() & 31; 485 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 486 MVT::i32); 487 return true; 488 } 489 490 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N, 491 SDValue &BaseReg, 492 SDValue &ShReg, 493 SDValue &Opc, 494 bool CheckProfitability) { 495 if (DisableShifterOp) 496 return false; 497 498 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 499 500 // Don't match base register only case. That is matched to a separate 501 // lower complexity pattern with explicit register operand. 502 if (ShOpcVal == ARM_AM::no_shift) return false; 503 504 BaseReg = N.getOperand(0); 505 unsigned ShImmVal = 0; 506 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 507 if (RHS) return false; 508 509 ShReg = N.getOperand(1); 510 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal)) 511 return false; 512 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal), 513 MVT::i32); 514 return true; 515 } 516 517 518 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N, 519 SDValue &Base, 520 SDValue &OffImm) { 521 // Match simple R + imm12 operands. 522 523 // Base only. 524 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 525 !CurDAG->isBaseWithConstantOffset(N)) { 526 if (N.getOpcode() == ISD::FrameIndex) { 527 // Match frame index. 528 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 529 Base = CurDAG->getTargetFrameIndex(FI, 530 getTargetLowering()->getPointerTy()); 531 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 532 return true; 533 } 534 535 if (N.getOpcode() == ARMISD::Wrapper && 536 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 537 Base = N.getOperand(0); 538 } else 539 Base = N; 540 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 541 return true; 542 } 543 544 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 545 int RHSC = (int)RHS->getZExtValue(); 546 if (N.getOpcode() == ISD::SUB) 547 RHSC = -RHSC; 548 549 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 550 Base = N.getOperand(0); 551 if (Base.getOpcode() == ISD::FrameIndex) { 552 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 553 Base = CurDAG->getTargetFrameIndex(FI, 554 getTargetLowering()->getPointerTy()); 555 } 556 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 557 return true; 558 } 559 } 560 561 // Base only. 562 Base = N; 563 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 564 return true; 565 } 566 567 568 569 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, 570 SDValue &Opc) { 571 if (N.getOpcode() == ISD::MUL && 572 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) { 573 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 574 // X * [3,5,9] -> X + X * [2,4,8] etc. 575 int RHSC = (int)RHS->getZExtValue(); 576 if (RHSC & 1) { 577 RHSC = RHSC & ~1; 578 ARM_AM::AddrOpc AddSub = ARM_AM::add; 579 if (RHSC < 0) { 580 AddSub = ARM_AM::sub; 581 RHSC = - RHSC; 582 } 583 if (isPowerOf2_32(RHSC)) { 584 unsigned ShAmt = Log2_32(RHSC); 585 Base = Offset = N.getOperand(0); 586 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 587 ARM_AM::lsl), 588 MVT::i32); 589 return true; 590 } 591 } 592 } 593 } 594 595 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 596 // ISD::OR that is equivalent to an ISD::ADD. 597 !CurDAG->isBaseWithConstantOffset(N)) 598 return false; 599 600 // Leave simple R +/- imm12 operands for LDRi12 601 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) { 602 int RHSC; 603 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 604 -0x1000+1, 0x1000, RHSC)) // 12 bits. 605 return false; 606 } 607 608 // Otherwise this is R +/- [possibly shifted] R. 609 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add; 610 ARM_AM::ShiftOpc ShOpcVal = 611 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 612 unsigned ShAmt = 0; 613 614 Base = N.getOperand(0); 615 Offset = N.getOperand(1); 616 617 if (ShOpcVal != ARM_AM::no_shift) { 618 // Check to see if the RHS of the shift is a constant, if not, we can't fold 619 // it. 620 if (ConstantSDNode *Sh = 621 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 622 ShAmt = Sh->getZExtValue(); 623 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 624 Offset = N.getOperand(1).getOperand(0); 625 else { 626 ShAmt = 0; 627 ShOpcVal = ARM_AM::no_shift; 628 } 629 } else { 630 ShOpcVal = ARM_AM::no_shift; 631 } 632 } 633 634 // Try matching (R shl C) + (R). 635 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 636 !(Subtarget->isLikeA9() || Subtarget->isSwift() || 637 N.getOperand(0).hasOneUse())) { 638 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 639 if (ShOpcVal != ARM_AM::no_shift) { 640 // Check to see if the RHS of the shift is a constant, if not, we can't 641 // fold it. 642 if (ConstantSDNode *Sh = 643 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 644 ShAmt = Sh->getZExtValue(); 645 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 646 Offset = N.getOperand(0).getOperand(0); 647 Base = N.getOperand(1); 648 } else { 649 ShAmt = 0; 650 ShOpcVal = ARM_AM::no_shift; 651 } 652 } else { 653 ShOpcVal = ARM_AM::no_shift; 654 } 655 } 656 } 657 658 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 659 MVT::i32); 660 return true; 661 } 662 663 664 //----- 665 666 AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N, 667 SDValue &Base, 668 SDValue &Offset, 669 SDValue &Opc) { 670 if (N.getOpcode() == ISD::MUL && 671 (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) { 672 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 673 // X * [3,5,9] -> X + X * [2,4,8] etc. 674 int RHSC = (int)RHS->getZExtValue(); 675 if (RHSC & 1) { 676 RHSC = RHSC & ~1; 677 ARM_AM::AddrOpc AddSub = ARM_AM::add; 678 if (RHSC < 0) { 679 AddSub = ARM_AM::sub; 680 RHSC = - RHSC; 681 } 682 if (isPowerOf2_32(RHSC)) { 683 unsigned ShAmt = Log2_32(RHSC); 684 Base = Offset = N.getOperand(0); 685 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, 686 ARM_AM::lsl), 687 MVT::i32); 688 return AM2_SHOP; 689 } 690 } 691 } 692 } 693 694 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 695 // ISD::OR that is equivalent to an ADD. 696 !CurDAG->isBaseWithConstantOffset(N)) { 697 Base = N; 698 if (N.getOpcode() == ISD::FrameIndex) { 699 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 700 Base = CurDAG->getTargetFrameIndex(FI, 701 getTargetLowering()->getPointerTy()); 702 } else if (N.getOpcode() == ARMISD::Wrapper && 703 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 704 Base = N.getOperand(0); 705 } 706 Offset = CurDAG->getRegister(0, MVT::i32); 707 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 708 ARM_AM::no_shift), 709 MVT::i32); 710 return AM2_BASE; 711 } 712 713 // Match simple R +/- imm12 operands. 714 if (N.getOpcode() != ISD::SUB) { 715 int RHSC; 716 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 717 -0x1000+1, 0x1000, RHSC)) { // 12 bits. 718 Base = N.getOperand(0); 719 if (Base.getOpcode() == ISD::FrameIndex) { 720 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 721 Base = CurDAG->getTargetFrameIndex(FI, 722 getTargetLowering()->getPointerTy()); 723 } 724 Offset = CurDAG->getRegister(0, MVT::i32); 725 726 ARM_AM::AddrOpc AddSub = ARM_AM::add; 727 if (RHSC < 0) { 728 AddSub = ARM_AM::sub; 729 RHSC = - RHSC; 730 } 731 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC, 732 ARM_AM::no_shift), 733 MVT::i32); 734 return AM2_BASE; 735 } 736 } 737 738 if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) { 739 // Compute R +/- (R << N) and reuse it. 740 Base = N; 741 Offset = CurDAG->getRegister(0, MVT::i32); 742 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0, 743 ARM_AM::no_shift), 744 MVT::i32); 745 return AM2_BASE; 746 } 747 748 // Otherwise this is R +/- [possibly shifted] R. 749 ARM_AM::AddrOpc AddSub = N.getOpcode() != ISD::SUB ? ARM_AM::add:ARM_AM::sub; 750 ARM_AM::ShiftOpc ShOpcVal = 751 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode()); 752 unsigned ShAmt = 0; 753 754 Base = N.getOperand(0); 755 Offset = N.getOperand(1); 756 757 if (ShOpcVal != ARM_AM::no_shift) { 758 // Check to see if the RHS of the shift is a constant, if not, we can't fold 759 // it. 760 if (ConstantSDNode *Sh = 761 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) { 762 ShAmt = Sh->getZExtValue(); 763 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt)) 764 Offset = N.getOperand(1).getOperand(0); 765 else { 766 ShAmt = 0; 767 ShOpcVal = ARM_AM::no_shift; 768 } 769 } else { 770 ShOpcVal = ARM_AM::no_shift; 771 } 772 } 773 774 // Try matching (R shl C) + (R). 775 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift && 776 !(Subtarget->isLikeA9() || Subtarget->isSwift() || 777 N.getOperand(0).hasOneUse())) { 778 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode()); 779 if (ShOpcVal != ARM_AM::no_shift) { 780 // Check to see if the RHS of the shift is a constant, if not, we can't 781 // fold it. 782 if (ConstantSDNode *Sh = 783 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) { 784 ShAmt = Sh->getZExtValue(); 785 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) { 786 Offset = N.getOperand(0).getOperand(0); 787 Base = N.getOperand(1); 788 } else { 789 ShAmt = 0; 790 ShOpcVal = ARM_AM::no_shift; 791 } 792 } else { 793 ShOpcVal = ARM_AM::no_shift; 794 } 795 } 796 } 797 798 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 799 MVT::i32); 800 return AM2_SHOP; 801 } 802 803 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N, 804 SDValue &Offset, SDValue &Opc) { 805 unsigned Opcode = Op->getOpcode(); 806 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 807 ? cast<LoadSDNode>(Op)->getAddressingMode() 808 : cast<StoreSDNode>(Op)->getAddressingMode(); 809 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 810 ? ARM_AM::add : ARM_AM::sub; 811 int Val; 812 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) 813 return false; 814 815 Offset = N; 816 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 817 unsigned ShAmt = 0; 818 if (ShOpcVal != ARM_AM::no_shift) { 819 // Check to see if the RHS of the shift is a constant, if not, we can't fold 820 // it. 821 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 822 ShAmt = Sh->getZExtValue(); 823 if (isShifterOpProfitable(N, ShOpcVal, ShAmt)) 824 Offset = N.getOperand(0); 825 else { 826 ShAmt = 0; 827 ShOpcVal = ARM_AM::no_shift; 828 } 829 } else { 830 ShOpcVal = ARM_AM::no_shift; 831 } 832 } 833 834 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal), 835 MVT::i32); 836 return true; 837 } 838 839 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N, 840 SDValue &Offset, SDValue &Opc) { 841 unsigned Opcode = Op->getOpcode(); 842 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 843 ? cast<LoadSDNode>(Op)->getAddressingMode() 844 : cast<StoreSDNode>(Op)->getAddressingMode(); 845 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 846 ? ARM_AM::add : ARM_AM::sub; 847 int Val; 848 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 849 if (AddSub == ARM_AM::sub) Val *= -1; 850 Offset = CurDAG->getRegister(0, MVT::i32); 851 Opc = CurDAG->getTargetConstant(Val, MVT::i32); 852 return true; 853 } 854 855 return false; 856 } 857 858 859 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N, 860 SDValue &Offset, SDValue &Opc) { 861 unsigned Opcode = Op->getOpcode(); 862 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 863 ? cast<LoadSDNode>(Op)->getAddressingMode() 864 : cast<StoreSDNode>(Op)->getAddressingMode(); 865 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 866 ? ARM_AM::add : ARM_AM::sub; 867 int Val; 868 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits. 869 Offset = CurDAG->getRegister(0, MVT::i32); 870 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val, 871 ARM_AM::no_shift), 872 MVT::i32); 873 return true; 874 } 875 876 return false; 877 } 878 879 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) { 880 Base = N; 881 return true; 882 } 883 884 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N, 885 SDValue &Base, SDValue &Offset, 886 SDValue &Opc) { 887 if (N.getOpcode() == ISD::SUB) { 888 // X - C is canonicalize to X + -C, no need to handle it here. 889 Base = N.getOperand(0); 890 Offset = N.getOperand(1); 891 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32); 892 return true; 893 } 894 895 if (!CurDAG->isBaseWithConstantOffset(N)) { 896 Base = N; 897 if (N.getOpcode() == ISD::FrameIndex) { 898 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 899 Base = CurDAG->getTargetFrameIndex(FI, 900 getTargetLowering()->getPointerTy()); 901 } 902 Offset = CurDAG->getRegister(0, MVT::i32); 903 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32); 904 return true; 905 } 906 907 // If the RHS is +/- imm8, fold into addr mode. 908 int RHSC; 909 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1, 910 -256 + 1, 256, RHSC)) { // 8 bits. 911 Base = N.getOperand(0); 912 if (Base.getOpcode() == ISD::FrameIndex) { 913 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 914 Base = CurDAG->getTargetFrameIndex(FI, 915 getTargetLowering()->getPointerTy()); 916 } 917 Offset = CurDAG->getRegister(0, MVT::i32); 918 919 ARM_AM::AddrOpc AddSub = ARM_AM::add; 920 if (RHSC < 0) { 921 AddSub = ARM_AM::sub; 922 RHSC = -RHSC; 923 } 924 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32); 925 return true; 926 } 927 928 Base = N.getOperand(0); 929 Offset = N.getOperand(1); 930 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32); 931 return true; 932 } 933 934 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N, 935 SDValue &Offset, SDValue &Opc) { 936 unsigned Opcode = Op->getOpcode(); 937 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 938 ? cast<LoadSDNode>(Op)->getAddressingMode() 939 : cast<StoreSDNode>(Op)->getAddressingMode(); 940 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC) 941 ? ARM_AM::add : ARM_AM::sub; 942 int Val; 943 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits. 944 Offset = CurDAG->getRegister(0, MVT::i32); 945 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32); 946 return true; 947 } 948 949 Offset = N; 950 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32); 951 return true; 952 } 953 954 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N, 955 SDValue &Base, SDValue &Offset) { 956 if (!CurDAG->isBaseWithConstantOffset(N)) { 957 Base = N; 958 if (N.getOpcode() == ISD::FrameIndex) { 959 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 960 Base = CurDAG->getTargetFrameIndex(FI, 961 getTargetLowering()->getPointerTy()); 962 } else if (N.getOpcode() == ARMISD::Wrapper && 963 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 964 Base = N.getOperand(0); 965 } 966 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 967 MVT::i32); 968 return true; 969 } 970 971 // If the RHS is +/- imm8, fold into addr mode. 972 int RHSC; 973 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 974 -256 + 1, 256, RHSC)) { 975 Base = N.getOperand(0); 976 if (Base.getOpcode() == ISD::FrameIndex) { 977 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 978 Base = CurDAG->getTargetFrameIndex(FI, 979 getTargetLowering()->getPointerTy()); 980 } 981 982 ARM_AM::AddrOpc AddSub = ARM_AM::add; 983 if (RHSC < 0) { 984 AddSub = ARM_AM::sub; 985 RHSC = -RHSC; 986 } 987 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC), 988 MVT::i32); 989 return true; 990 } 991 992 Base = N; 993 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0), 994 MVT::i32); 995 return true; 996 } 997 998 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr, 999 SDValue &Align) { 1000 Addr = N; 1001 1002 unsigned Alignment = 0; 1003 if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) { 1004 // This case occurs only for VLD1-lane/dup and VST1-lane instructions. 1005 // The maximum alignment is equal to the memory size being referenced. 1006 unsigned LSNAlign = LSN->getAlignment(); 1007 unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8; 1008 if (LSNAlign >= MemSize && MemSize > 1) 1009 Alignment = MemSize; 1010 } else { 1011 // All other uses of addrmode6 are for intrinsics. For now just record 1012 // the raw alignment value; it will be refined later based on the legal 1013 // alignment operands for the intrinsic. 1014 Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment(); 1015 } 1016 1017 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 1018 return true; 1019 } 1020 1021 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N, 1022 SDValue &Offset) { 1023 LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op); 1024 ISD::MemIndexedMode AM = LdSt->getAddressingMode(); 1025 if (AM != ISD::POST_INC) 1026 return false; 1027 Offset = N; 1028 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) { 1029 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits()) 1030 Offset = CurDAG->getRegister(0, MVT::i32); 1031 } 1032 return true; 1033 } 1034 1035 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N, 1036 SDValue &Offset, SDValue &Label) { 1037 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) { 1038 Offset = N.getOperand(0); 1039 SDValue N1 = N.getOperand(1); 1040 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(), 1041 MVT::i32); 1042 return true; 1043 } 1044 1045 return false; 1046 } 1047 1048 1049 //===----------------------------------------------------------------------===// 1050 // Thumb Addressing Modes 1051 //===----------------------------------------------------------------------===// 1052 1053 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, 1054 SDValue &Base, SDValue &Offset){ 1055 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) { 1056 ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N); 1057 if (!NC || !NC->isNullValue()) 1058 return false; 1059 1060 Base = Offset = N; 1061 return true; 1062 } 1063 1064 Base = N.getOperand(0); 1065 Offset = N.getOperand(1); 1066 return true; 1067 } 1068 1069 bool 1070 ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base, 1071 SDValue &Offset, unsigned Scale) { 1072 if (Scale == 4) { 1073 SDValue TmpBase, TmpOffImm; 1074 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1075 return false; // We want to select tLDRspi / tSTRspi instead. 1076 1077 if (N.getOpcode() == ARMISD::Wrapper && 1078 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1079 return false; // We want to select tLDRpci instead. 1080 } 1081 1082 if (!CurDAG->isBaseWithConstantOffset(N)) 1083 return false; 1084 1085 // Thumb does not have [sp, r] address mode. 1086 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1087 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1088 if ((LHSR && LHSR->getReg() == ARM::SP) || 1089 (RHSR && RHSR->getReg() == ARM::SP)) 1090 return false; 1091 1092 // FIXME: Why do we explicitly check for a match here and then return false? 1093 // Presumably to allow something else to match, but shouldn't this be 1094 // documented? 1095 int RHSC; 1096 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) 1097 return false; 1098 1099 Base = N.getOperand(0); 1100 Offset = N.getOperand(1); 1101 return true; 1102 } 1103 1104 bool 1105 ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N, 1106 SDValue &Base, 1107 SDValue &Offset) { 1108 return SelectThumbAddrModeRI(N, Base, Offset, 1); 1109 } 1110 1111 bool 1112 ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N, 1113 SDValue &Base, 1114 SDValue &Offset) { 1115 return SelectThumbAddrModeRI(N, Base, Offset, 2); 1116 } 1117 1118 bool 1119 ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N, 1120 SDValue &Base, 1121 SDValue &Offset) { 1122 return SelectThumbAddrModeRI(N, Base, Offset, 4); 1123 } 1124 1125 bool 1126 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, 1127 SDValue &Base, SDValue &OffImm) { 1128 if (Scale == 4) { 1129 SDValue TmpBase, TmpOffImm; 1130 if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm)) 1131 return false; // We want to select tLDRspi / tSTRspi instead. 1132 1133 if (N.getOpcode() == ARMISD::Wrapper && 1134 N.getOperand(0).getOpcode() == ISD::TargetConstantPool) 1135 return false; // We want to select tLDRpci instead. 1136 } 1137 1138 if (!CurDAG->isBaseWithConstantOffset(N)) { 1139 if (N.getOpcode() == ARMISD::Wrapper && 1140 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 1141 Base = N.getOperand(0); 1142 } else { 1143 Base = N; 1144 } 1145 1146 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1147 return true; 1148 } 1149 1150 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1151 RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1)); 1152 if ((LHSR && LHSR->getReg() == ARM::SP) || 1153 (RHSR && RHSR->getReg() == ARM::SP)) { 1154 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0)); 1155 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1156 unsigned LHSC = LHS ? LHS->getZExtValue() : 0; 1157 unsigned RHSC = RHS ? RHS->getZExtValue() : 0; 1158 1159 // Thumb does not have [sp, #imm5] address mode for non-zero imm5. 1160 if (LHSC != 0 || RHSC != 0) return false; 1161 1162 Base = N; 1163 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1164 return true; 1165 } 1166 1167 // If the RHS is + imm5 * scale, fold into addr mode. 1168 int RHSC; 1169 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) { 1170 Base = N.getOperand(0); 1171 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1172 return true; 1173 } 1174 1175 Base = N.getOperand(0); 1176 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1177 return true; 1178 } 1179 1180 bool 1181 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base, 1182 SDValue &OffImm) { 1183 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm); 1184 } 1185 1186 bool 1187 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base, 1188 SDValue &OffImm) { 1189 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm); 1190 } 1191 1192 bool 1193 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base, 1194 SDValue &OffImm) { 1195 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm); 1196 } 1197 1198 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N, 1199 SDValue &Base, SDValue &OffImm) { 1200 if (N.getOpcode() == ISD::FrameIndex) { 1201 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1202 Base = CurDAG->getTargetFrameIndex(FI, 1203 getTargetLowering()->getPointerTy()); 1204 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1205 return true; 1206 } 1207 1208 if (!CurDAG->isBaseWithConstantOffset(N)) 1209 return false; 1210 1211 RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0)); 1212 if (N.getOperand(0).getOpcode() == ISD::FrameIndex || 1213 (LHSR && LHSR->getReg() == ARM::SP)) { 1214 // If the RHS is + imm8 * scale, fold into addr mode. 1215 int RHSC; 1216 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) { 1217 Base = N.getOperand(0); 1218 if (Base.getOpcode() == ISD::FrameIndex) { 1219 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1220 Base = CurDAG->getTargetFrameIndex(FI, 1221 getTargetLowering()->getPointerTy()); 1222 } 1223 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1224 return true; 1225 } 1226 } 1227 1228 return false; 1229 } 1230 1231 1232 //===----------------------------------------------------------------------===// 1233 // Thumb 2 Addressing Modes 1234 //===----------------------------------------------------------------------===// 1235 1236 1237 bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg, 1238 SDValue &Opc) { 1239 if (DisableShifterOp) 1240 return false; 1241 1242 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode()); 1243 1244 // Don't match base register only case. That is matched to a separate 1245 // lower complexity pattern with explicit register operand. 1246 if (ShOpcVal == ARM_AM::no_shift) return false; 1247 1248 BaseReg = N.getOperand(0); 1249 unsigned ShImmVal = 0; 1250 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1251 ShImmVal = RHS->getZExtValue() & 31; 1252 Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal)); 1253 return true; 1254 } 1255 1256 return false; 1257 } 1258 1259 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N, 1260 SDValue &Base, SDValue &OffImm) { 1261 // Match simple R + imm12 operands. 1262 1263 // Base only. 1264 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1265 !CurDAG->isBaseWithConstantOffset(N)) { 1266 if (N.getOpcode() == ISD::FrameIndex) { 1267 // Match frame index. 1268 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 1269 Base = CurDAG->getTargetFrameIndex(FI, 1270 getTargetLowering()->getPointerTy()); 1271 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1272 return true; 1273 } 1274 1275 if (N.getOpcode() == ARMISD::Wrapper && 1276 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress) { 1277 Base = N.getOperand(0); 1278 if (Base.getOpcode() == ISD::TargetConstantPool) 1279 return false; // We want to select t2LDRpci instead. 1280 } else 1281 Base = N; 1282 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1283 return true; 1284 } 1285 1286 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1287 if (SelectT2AddrModeImm8(N, Base, OffImm)) 1288 // Let t2LDRi8 handle (R - imm8). 1289 return false; 1290 1291 int RHSC = (int)RHS->getZExtValue(); 1292 if (N.getOpcode() == ISD::SUB) 1293 RHSC = -RHSC; 1294 1295 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned) 1296 Base = N.getOperand(0); 1297 if (Base.getOpcode() == ISD::FrameIndex) { 1298 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1299 Base = CurDAG->getTargetFrameIndex(FI, 1300 getTargetLowering()->getPointerTy()); 1301 } 1302 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1303 return true; 1304 } 1305 } 1306 1307 // Base only. 1308 Base = N; 1309 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1310 return true; 1311 } 1312 1313 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, 1314 SDValue &Base, SDValue &OffImm) { 1315 // Match simple R - imm8 operands. 1316 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB && 1317 !CurDAG->isBaseWithConstantOffset(N)) 1318 return false; 1319 1320 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1321 int RHSC = (int)RHS->getSExtValue(); 1322 if (N.getOpcode() == ISD::SUB) 1323 RHSC = -RHSC; 1324 1325 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative) 1326 Base = N.getOperand(0); 1327 if (Base.getOpcode() == ISD::FrameIndex) { 1328 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1329 Base = CurDAG->getTargetFrameIndex(FI, 1330 getTargetLowering()->getPointerTy()); 1331 } 1332 OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32); 1333 return true; 1334 } 1335 } 1336 1337 return false; 1338 } 1339 1340 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N, 1341 SDValue &OffImm){ 1342 unsigned Opcode = Op->getOpcode(); 1343 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD) 1344 ? cast<LoadSDNode>(Op)->getAddressingMode() 1345 : cast<StoreSDNode>(Op)->getAddressingMode(); 1346 int RHSC; 1347 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits. 1348 OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC)) 1349 ? CurDAG->getTargetConstant(RHSC, MVT::i32) 1350 : CurDAG->getTargetConstant(-RHSC, MVT::i32); 1351 return true; 1352 } 1353 1354 return false; 1355 } 1356 1357 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N, 1358 SDValue &Base, 1359 SDValue &OffReg, SDValue &ShImm) { 1360 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12. 1361 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) 1362 return false; 1363 1364 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8. 1365 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 1366 int RHSC = (int)RHS->getZExtValue(); 1367 if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned) 1368 return false; 1369 else if (RHSC < 0 && RHSC >= -255) // 8 bits 1370 return false; 1371 } 1372 1373 // Look for (R + R) or (R + (R << [1,2,3])). 1374 unsigned ShAmt = 0; 1375 Base = N.getOperand(0); 1376 OffReg = N.getOperand(1); 1377 1378 // Swap if it is ((R << c) + R). 1379 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode()); 1380 if (ShOpcVal != ARM_AM::lsl) { 1381 ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode()); 1382 if (ShOpcVal == ARM_AM::lsl) 1383 std::swap(Base, OffReg); 1384 } 1385 1386 if (ShOpcVal == ARM_AM::lsl) { 1387 // Check to see if the RHS of the shift is a constant, if not, we can't fold 1388 // it. 1389 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) { 1390 ShAmt = Sh->getZExtValue(); 1391 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)) 1392 OffReg = OffReg.getOperand(0); 1393 else { 1394 ShAmt = 0; 1395 ShOpcVal = ARM_AM::no_shift; 1396 } 1397 } else { 1398 ShOpcVal = ARM_AM::no_shift; 1399 } 1400 } 1401 1402 ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32); 1403 1404 return true; 1405 } 1406 1407 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base, 1408 SDValue &OffImm) { 1409 // This *must* succeed since it's used for the irreplaceable ldrex and strex 1410 // instructions. 1411 Base = N; 1412 OffImm = CurDAG->getTargetConstant(0, MVT::i32); 1413 1414 if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N)) 1415 return true; 1416 1417 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1)); 1418 if (!RHS) 1419 return true; 1420 1421 uint32_t RHSC = (int)RHS->getZExtValue(); 1422 if (RHSC > 1020 || RHSC % 4 != 0) 1423 return true; 1424 1425 Base = N.getOperand(0); 1426 if (Base.getOpcode() == ISD::FrameIndex) { 1427 int FI = cast<FrameIndexSDNode>(Base)->getIndex(); 1428 Base = CurDAG->getTargetFrameIndex(FI, getTargetLowering()->getPointerTy()); 1429 } 1430 1431 OffImm = CurDAG->getTargetConstant(RHSC / 4, MVT::i32); 1432 return true; 1433 } 1434 1435 //===--------------------------------------------------------------------===// 1436 1437 /// getAL - Returns a ARMCC::AL immediate node. 1438 static inline SDValue getAL(SelectionDAG *CurDAG) { 1439 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32); 1440 } 1441 1442 SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) { 1443 LoadSDNode *LD = cast<LoadSDNode>(N); 1444 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1445 if (AM == ISD::UNINDEXED) 1446 return nullptr; 1447 1448 EVT LoadedVT = LD->getMemoryVT(); 1449 SDValue Offset, AMOpc; 1450 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1451 unsigned Opcode = 0; 1452 bool Match = false; 1453 if (LoadedVT == MVT::i32 && isPre && 1454 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1455 Opcode = ARM::LDR_PRE_IMM; 1456 Match = true; 1457 } else if (LoadedVT == MVT::i32 && !isPre && 1458 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1459 Opcode = ARM::LDR_POST_IMM; 1460 Match = true; 1461 } else if (LoadedVT == MVT::i32 && 1462 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1463 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG; 1464 Match = true; 1465 1466 } else if (LoadedVT == MVT::i16 && 1467 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1468 Match = true; 1469 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD) 1470 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST) 1471 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST); 1472 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) { 1473 if (LD->getExtensionType() == ISD::SEXTLOAD) { 1474 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) { 1475 Match = true; 1476 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST; 1477 } 1478 } else { 1479 if (isPre && 1480 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) { 1481 Match = true; 1482 Opcode = ARM::LDRB_PRE_IMM; 1483 } else if (!isPre && 1484 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) { 1485 Match = true; 1486 Opcode = ARM::LDRB_POST_IMM; 1487 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) { 1488 Match = true; 1489 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG; 1490 } 1491 } 1492 } 1493 1494 if (Match) { 1495 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) { 1496 SDValue Chain = LD->getChain(); 1497 SDValue Base = LD->getBasePtr(); 1498 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG), 1499 CurDAG->getRegister(0, MVT::i32), Chain }; 1500 return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, 1501 MVT::i32, MVT::Other, Ops); 1502 } else { 1503 SDValue Chain = LD->getChain(); 1504 SDValue Base = LD->getBasePtr(); 1505 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG), 1506 CurDAG->getRegister(0, MVT::i32), Chain }; 1507 return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, 1508 MVT::i32, MVT::Other, Ops); 1509 } 1510 } 1511 1512 return nullptr; 1513 } 1514 1515 SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) { 1516 LoadSDNode *LD = cast<LoadSDNode>(N); 1517 ISD::MemIndexedMode AM = LD->getAddressingMode(); 1518 if (AM == ISD::UNINDEXED) 1519 return nullptr; 1520 1521 EVT LoadedVT = LD->getMemoryVT(); 1522 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD; 1523 SDValue Offset; 1524 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC); 1525 unsigned Opcode = 0; 1526 bool Match = false; 1527 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) { 1528 switch (LoadedVT.getSimpleVT().SimpleTy) { 1529 case MVT::i32: 1530 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST; 1531 break; 1532 case MVT::i16: 1533 if (isSExtLd) 1534 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST; 1535 else 1536 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST; 1537 break; 1538 case MVT::i8: 1539 case MVT::i1: 1540 if (isSExtLd) 1541 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST; 1542 else 1543 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST; 1544 break; 1545 default: 1546 return nullptr; 1547 } 1548 Match = true; 1549 } 1550 1551 if (Match) { 1552 SDValue Chain = LD->getChain(); 1553 SDValue Base = LD->getBasePtr(); 1554 SDValue Ops[]= { Base, Offset, getAL(CurDAG), 1555 CurDAG->getRegister(0, MVT::i32), Chain }; 1556 return CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32, 1557 MVT::Other, Ops); 1558 } 1559 1560 return nullptr; 1561 } 1562 1563 /// \brief Form a GPRPair pseudo register from a pair of GPR regs. 1564 SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) { 1565 SDLoc dl(V0.getNode()); 1566 SDValue RegClass = 1567 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, MVT::i32); 1568 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); 1569 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); 1570 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1571 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1572 } 1573 1574 /// \brief Form a D register from a pair of S registers. 1575 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1576 SDLoc dl(V0.getNode()); 1577 SDValue RegClass = 1578 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, MVT::i32); 1579 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1580 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1581 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1582 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1583 } 1584 1585 /// \brief Form a quad register from a pair of D registers. 1586 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1587 SDLoc dl(V0.getNode()); 1588 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, MVT::i32); 1589 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1590 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1591 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1592 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1593 } 1594 1595 /// \brief Form 4 consecutive D registers from a pair of Q registers. 1596 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) { 1597 SDLoc dl(V0.getNode()); 1598 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1599 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1600 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1601 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 }; 1602 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1603 } 1604 1605 /// \brief Form 4 consecutive S registers. 1606 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, 1607 SDValue V2, SDValue V3) { 1608 SDLoc dl(V0.getNode()); 1609 SDValue RegClass = 1610 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, MVT::i32); 1611 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32); 1612 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32); 1613 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32); 1614 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32); 1615 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1616 V2, SubReg2, V3, SubReg3 }; 1617 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1618 } 1619 1620 /// \brief Form 4 consecutive D registers. 1621 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, 1622 SDValue V2, SDValue V3) { 1623 SDLoc dl(V0.getNode()); 1624 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, MVT::i32); 1625 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32); 1626 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32); 1627 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32); 1628 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32); 1629 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1630 V2, SubReg2, V3, SubReg3 }; 1631 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1632 } 1633 1634 /// \brief Form 4 consecutive Q registers. 1635 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, 1636 SDValue V2, SDValue V3) { 1637 SDLoc dl(V0.getNode()); 1638 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, MVT::i32); 1639 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32); 1640 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32); 1641 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32); 1642 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32); 1643 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1, 1644 V2, SubReg2, V3, SubReg3 }; 1645 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops); 1646 } 1647 1648 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand 1649 /// of a NEON VLD or VST instruction. The supported values depend on the 1650 /// number of registers being loaded. 1651 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs, 1652 bool is64BitVector) { 1653 unsigned NumRegs = NumVecs; 1654 if (!is64BitVector && NumVecs < 3) 1655 NumRegs *= 2; 1656 1657 unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 1658 if (Alignment >= 32 && NumRegs == 4) 1659 Alignment = 32; 1660 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4)) 1661 Alignment = 16; 1662 else if (Alignment >= 8) 1663 Alignment = 8; 1664 else 1665 Alignment = 0; 1666 1667 return CurDAG->getTargetConstant(Alignment, MVT::i32); 1668 } 1669 1670 static bool isVLDfixed(unsigned Opc) 1671 { 1672 switch (Opc) { 1673 default: return false; 1674 case ARM::VLD1d8wb_fixed : return true; 1675 case ARM::VLD1d16wb_fixed : return true; 1676 case ARM::VLD1d64Qwb_fixed : return true; 1677 case ARM::VLD1d32wb_fixed : return true; 1678 case ARM::VLD1d64wb_fixed : return true; 1679 case ARM::VLD1d64TPseudoWB_fixed : return true; 1680 case ARM::VLD1d64QPseudoWB_fixed : return true; 1681 case ARM::VLD1q8wb_fixed : return true; 1682 case ARM::VLD1q16wb_fixed : return true; 1683 case ARM::VLD1q32wb_fixed : return true; 1684 case ARM::VLD1q64wb_fixed : return true; 1685 case ARM::VLD2d8wb_fixed : return true; 1686 case ARM::VLD2d16wb_fixed : return true; 1687 case ARM::VLD2d32wb_fixed : return true; 1688 case ARM::VLD2q8PseudoWB_fixed : return true; 1689 case ARM::VLD2q16PseudoWB_fixed : return true; 1690 case ARM::VLD2q32PseudoWB_fixed : return true; 1691 case ARM::VLD2DUPd8wb_fixed : return true; 1692 case ARM::VLD2DUPd16wb_fixed : return true; 1693 case ARM::VLD2DUPd32wb_fixed : return true; 1694 } 1695 } 1696 1697 static bool isVSTfixed(unsigned Opc) 1698 { 1699 switch (Opc) { 1700 default: return false; 1701 case ARM::VST1d8wb_fixed : return true; 1702 case ARM::VST1d16wb_fixed : return true; 1703 case ARM::VST1d32wb_fixed : return true; 1704 case ARM::VST1d64wb_fixed : return true; 1705 case ARM::VST1q8wb_fixed : return true; 1706 case ARM::VST1q16wb_fixed : return true; 1707 case ARM::VST1q32wb_fixed : return true; 1708 case ARM::VST1q64wb_fixed : return true; 1709 case ARM::VST1d64TPseudoWB_fixed : return true; 1710 case ARM::VST1d64QPseudoWB_fixed : return true; 1711 case ARM::VST2d8wb_fixed : return true; 1712 case ARM::VST2d16wb_fixed : return true; 1713 case ARM::VST2d32wb_fixed : return true; 1714 case ARM::VST2q8PseudoWB_fixed : return true; 1715 case ARM::VST2q16PseudoWB_fixed : return true; 1716 case ARM::VST2q32PseudoWB_fixed : return true; 1717 } 1718 } 1719 1720 // Get the register stride update opcode of a VLD/VST instruction that 1721 // is otherwise equivalent to the given fixed stride updating instruction. 1722 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) { 1723 assert((isVLDfixed(Opc) || isVSTfixed(Opc)) 1724 && "Incorrect fixed stride updating instruction."); 1725 switch (Opc) { 1726 default: break; 1727 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register; 1728 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register; 1729 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register; 1730 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register; 1731 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register; 1732 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register; 1733 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register; 1734 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register; 1735 case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register; 1736 case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register; 1737 case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register; 1738 case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register; 1739 1740 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register; 1741 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register; 1742 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register; 1743 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register; 1744 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register; 1745 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; 1746 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; 1747 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; 1748 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; 1749 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; 1750 1751 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; 1752 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register; 1753 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register; 1754 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register; 1755 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register; 1756 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register; 1757 1758 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register; 1759 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register; 1760 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register; 1761 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register; 1762 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register; 1763 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register; 1764 1765 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register; 1766 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register; 1767 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register; 1768 } 1769 return Opc; // If not one we handle, return it unchanged. 1770 } 1771 1772 SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, 1773 const uint16_t *DOpcodes, 1774 const uint16_t *QOpcodes0, 1775 const uint16_t *QOpcodes1) { 1776 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range"); 1777 SDLoc dl(N); 1778 1779 SDValue MemAddr, Align; 1780 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1781 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1782 return nullptr; 1783 1784 SDValue Chain = N->getOperand(0); 1785 EVT VT = N->getValueType(0); 1786 bool is64BitVector = VT.is64BitVector(); 1787 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1788 1789 unsigned OpcodeIndex; 1790 switch (VT.getSimpleVT().SimpleTy) { 1791 default: llvm_unreachable("unhandled vld type"); 1792 // Double-register operations: 1793 case MVT::v8i8: OpcodeIndex = 0; break; 1794 case MVT::v4i16: OpcodeIndex = 1; break; 1795 case MVT::v2f32: 1796 case MVT::v2i32: OpcodeIndex = 2; break; 1797 case MVT::v1i64: OpcodeIndex = 3; break; 1798 // Quad-register operations: 1799 case MVT::v16i8: OpcodeIndex = 0; break; 1800 case MVT::v8i16: OpcodeIndex = 1; break; 1801 case MVT::v4f32: 1802 case MVT::v4i32: OpcodeIndex = 2; break; 1803 case MVT::v2i64: OpcodeIndex = 3; 1804 assert(NumVecs == 1 && "v2i64 type only supported for VLD1"); 1805 break; 1806 } 1807 1808 EVT ResTy; 1809 if (NumVecs == 1) 1810 ResTy = VT; 1811 else { 1812 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 1813 if (!is64BitVector) 1814 ResTyElts *= 2; 1815 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts); 1816 } 1817 std::vector<EVT> ResTys; 1818 ResTys.push_back(ResTy); 1819 if (isUpdating) 1820 ResTys.push_back(MVT::i32); 1821 ResTys.push_back(MVT::Other); 1822 1823 SDValue Pred = getAL(CurDAG); 1824 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1825 SDNode *VLd; 1826 SmallVector<SDValue, 7> Ops; 1827 1828 // Double registers and VLD1/VLD2 quad registers are directly supported. 1829 if (is64BitVector || NumVecs <= 2) { 1830 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1831 QOpcodes0[OpcodeIndex]); 1832 Ops.push_back(MemAddr); 1833 Ops.push_back(Align); 1834 if (isUpdating) { 1835 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1836 // FIXME: VLD1/VLD2 fixed increment doesn't need Reg0. Remove the reg0 1837 // case entirely when the rest are updated to that form, too. 1838 if ((NumVecs <= 2) && !isa<ConstantSDNode>(Inc.getNode())) 1839 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1840 // FIXME: We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so 1841 // check for that explicitly too. Horribly hacky, but temporary. 1842 if ((NumVecs > 2 && !isVLDfixed(Opc)) || 1843 !isa<ConstantSDNode>(Inc.getNode())) 1844 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 1845 } 1846 Ops.push_back(Pred); 1847 Ops.push_back(Reg0); 1848 Ops.push_back(Chain); 1849 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 1850 1851 } else { 1852 // Otherwise, quad registers are loaded with two separate instructions, 1853 // where one loads the even registers and the other loads the odd registers. 1854 EVT AddrTy = MemAddr.getValueType(); 1855 1856 // Load the even subregs. This is always an updating load, so that it 1857 // provides the address to the second load for the odd subregs. 1858 SDValue ImplDef = 1859 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0); 1860 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain }; 1861 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 1862 ResTy, AddrTy, MVT::Other, OpsA); 1863 Chain = SDValue(VLdA, 2); 1864 1865 // Load the odd subregs. 1866 Ops.push_back(SDValue(VLdA, 1)); 1867 Ops.push_back(Align); 1868 if (isUpdating) { 1869 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1870 assert(isa<ConstantSDNode>(Inc.getNode()) && 1871 "only constant post-increment update allowed for VLD3/4"); 1872 (void)Inc; 1873 Ops.push_back(Reg0); 1874 } 1875 Ops.push_back(SDValue(VLdA, 0)); 1876 Ops.push_back(Pred); 1877 Ops.push_back(Reg0); 1878 Ops.push_back(Chain); 1879 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); 1880 } 1881 1882 // Transfer memoperands. 1883 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1884 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1885 cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); 1886 1887 if (NumVecs == 1) 1888 return VLd; 1889 1890 // Extract out the subregisters. 1891 SDValue SuperReg = SDValue(VLd, 0); 1892 assert(ARM::dsub_7 == ARM::dsub_0+7 && 1893 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 1894 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0); 1895 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 1896 ReplaceUses(SDValue(N, Vec), 1897 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 1898 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1)); 1899 if (isUpdating) 1900 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2)); 1901 return nullptr; 1902 } 1903 1904 SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, 1905 const uint16_t *DOpcodes, 1906 const uint16_t *QOpcodes0, 1907 const uint16_t *QOpcodes1) { 1908 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range"); 1909 SDLoc dl(N); 1910 1911 SDValue MemAddr, Align; 1912 unsigned AddrOpIdx = isUpdating ? 1 : 2; 1913 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 1914 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 1915 return nullptr; 1916 1917 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1918 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 1919 1920 SDValue Chain = N->getOperand(0); 1921 EVT VT = N->getOperand(Vec0Idx).getValueType(); 1922 bool is64BitVector = VT.is64BitVector(); 1923 Align = GetVLDSTAlign(Align, NumVecs, is64BitVector); 1924 1925 unsigned OpcodeIndex; 1926 switch (VT.getSimpleVT().SimpleTy) { 1927 default: llvm_unreachable("unhandled vst type"); 1928 // Double-register operations: 1929 case MVT::v8i8: OpcodeIndex = 0; break; 1930 case MVT::v4i16: OpcodeIndex = 1; break; 1931 case MVT::v2f32: 1932 case MVT::v2i32: OpcodeIndex = 2; break; 1933 case MVT::v1i64: OpcodeIndex = 3; break; 1934 // Quad-register operations: 1935 case MVT::v16i8: OpcodeIndex = 0; break; 1936 case MVT::v8i16: OpcodeIndex = 1; break; 1937 case MVT::v4f32: 1938 case MVT::v4i32: OpcodeIndex = 2; break; 1939 case MVT::v2i64: OpcodeIndex = 3; 1940 assert(NumVecs == 1 && "v2i64 type only supported for VST1"); 1941 break; 1942 } 1943 1944 std::vector<EVT> ResTys; 1945 if (isUpdating) 1946 ResTys.push_back(MVT::i32); 1947 ResTys.push_back(MVT::Other); 1948 1949 SDValue Pred = getAL(CurDAG); 1950 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 1951 SmallVector<SDValue, 7> Ops; 1952 1953 // Double registers and VST1/VST2 quad registers are directly supported. 1954 if (is64BitVector || NumVecs <= 2) { 1955 SDValue SrcReg; 1956 if (NumVecs == 1) { 1957 SrcReg = N->getOperand(Vec0Idx); 1958 } else if (is64BitVector) { 1959 // Form a REG_SEQUENCE to force register allocation. 1960 SDValue V0 = N->getOperand(Vec0Idx + 0); 1961 SDValue V1 = N->getOperand(Vec0Idx + 1); 1962 if (NumVecs == 2) 1963 SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 1964 else { 1965 SDValue V2 = N->getOperand(Vec0Idx + 2); 1966 // If it's a vst3, form a quad D-register and leave the last part as 1967 // an undef. 1968 SDValue V3 = (NumVecs == 3) 1969 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0) 1970 : N->getOperand(Vec0Idx + 3); 1971 SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 1972 } 1973 } else { 1974 // Form a QQ register. 1975 SDValue Q0 = N->getOperand(Vec0Idx); 1976 SDValue Q1 = N->getOperand(Vec0Idx + 1); 1977 SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0); 1978 } 1979 1980 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 1981 QOpcodes0[OpcodeIndex]); 1982 Ops.push_back(MemAddr); 1983 Ops.push_back(Align); 1984 if (isUpdating) { 1985 SDValue Inc = N->getOperand(AddrOpIdx + 1); 1986 // FIXME: VST1/VST2 fixed increment doesn't need Reg0. Remove the reg0 1987 // case entirely when the rest are updated to that form, too. 1988 if (NumVecs <= 2 && !isa<ConstantSDNode>(Inc.getNode())) 1989 Opc = getVLDSTRegisterUpdateOpcode(Opc); 1990 // FIXME: We use a VST1 for v1i64 even if the pseudo says vld2/3/4, so 1991 // check for that explicitly too. Horribly hacky, but temporary. 1992 if (!isa<ConstantSDNode>(Inc.getNode())) 1993 Ops.push_back(Inc); 1994 else if (NumVecs > 2 && !isVSTfixed(Opc)) 1995 Ops.push_back(Reg0); 1996 } 1997 Ops.push_back(SrcReg); 1998 Ops.push_back(Pred); 1999 Ops.push_back(Reg0); 2000 Ops.push_back(Chain); 2001 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2002 2003 // Transfer memoperands. 2004 cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); 2005 2006 return VSt; 2007 } 2008 2009 // Otherwise, quad registers are stored with two separate instructions, 2010 // where one stores the even registers and the other stores the odd registers. 2011 2012 // Form the QQQQ REG_SEQUENCE. 2013 SDValue V0 = N->getOperand(Vec0Idx + 0); 2014 SDValue V1 = N->getOperand(Vec0Idx + 1); 2015 SDValue V2 = N->getOperand(Vec0Idx + 2); 2016 SDValue V3 = (NumVecs == 3) 2017 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2018 : N->getOperand(Vec0Idx + 3); 2019 SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2020 2021 // Store the even D registers. This is always an updating store, so that it 2022 // provides the address to the second store for the odd subregs. 2023 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain }; 2024 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, 2025 MemAddr.getValueType(), 2026 MVT::Other, OpsA); 2027 cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); 2028 Chain = SDValue(VStA, 1); 2029 2030 // Store the odd D registers. 2031 Ops.push_back(SDValue(VStA, 0)); 2032 Ops.push_back(Align); 2033 if (isUpdating) { 2034 SDValue Inc = N->getOperand(AddrOpIdx + 1); 2035 assert(isa<ConstantSDNode>(Inc.getNode()) && 2036 "only constant post-increment update allowed for VST3/4"); 2037 (void)Inc; 2038 Ops.push_back(Reg0); 2039 } 2040 Ops.push_back(RegSeq); 2041 Ops.push_back(Pred); 2042 Ops.push_back(Reg0); 2043 Ops.push_back(Chain); 2044 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, 2045 Ops); 2046 cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); 2047 return VStB; 2048 } 2049 2050 SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, 2051 bool isUpdating, unsigned NumVecs, 2052 const uint16_t *DOpcodes, 2053 const uint16_t *QOpcodes) { 2054 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range"); 2055 SDLoc dl(N); 2056 2057 SDValue MemAddr, Align; 2058 unsigned AddrOpIdx = isUpdating ? 1 : 2; 2059 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1) 2060 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) 2061 return nullptr; 2062 2063 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2064 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2065 2066 SDValue Chain = N->getOperand(0); 2067 unsigned Lane = 2068 cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue(); 2069 EVT VT = N->getOperand(Vec0Idx).getValueType(); 2070 bool is64BitVector = VT.is64BitVector(); 2071 2072 unsigned Alignment = 0; 2073 if (NumVecs != 3) { 2074 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2075 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2076 if (Alignment > NumBytes) 2077 Alignment = NumBytes; 2078 if (Alignment < 8 && Alignment < NumBytes) 2079 Alignment = 0; 2080 // Alignment must be a power of two; make sure of that. 2081 Alignment = (Alignment & -Alignment); 2082 if (Alignment == 1) 2083 Alignment = 0; 2084 } 2085 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2086 2087 unsigned OpcodeIndex; 2088 switch (VT.getSimpleVT().SimpleTy) { 2089 default: llvm_unreachable("unhandled vld/vst lane type"); 2090 // Double-register operations: 2091 case MVT::v8i8: OpcodeIndex = 0; break; 2092 case MVT::v4i16: OpcodeIndex = 1; break; 2093 case MVT::v2f32: 2094 case MVT::v2i32: OpcodeIndex = 2; break; 2095 // Quad-register operations: 2096 case MVT::v8i16: OpcodeIndex = 0; break; 2097 case MVT::v4f32: 2098 case MVT::v4i32: OpcodeIndex = 1; break; 2099 } 2100 2101 std::vector<EVT> ResTys; 2102 if (IsLoad) { 2103 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2104 if (!is64BitVector) 2105 ResTyElts *= 2; 2106 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), 2107 MVT::i64, ResTyElts)); 2108 } 2109 if (isUpdating) 2110 ResTys.push_back(MVT::i32); 2111 ResTys.push_back(MVT::Other); 2112 2113 SDValue Pred = getAL(CurDAG); 2114 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2115 2116 SmallVector<SDValue, 8> Ops; 2117 Ops.push_back(MemAddr); 2118 Ops.push_back(Align); 2119 if (isUpdating) { 2120 SDValue Inc = N->getOperand(AddrOpIdx + 1); 2121 Ops.push_back(isa<ConstantSDNode>(Inc.getNode()) ? Reg0 : Inc); 2122 } 2123 2124 SDValue SuperReg; 2125 SDValue V0 = N->getOperand(Vec0Idx + 0); 2126 SDValue V1 = N->getOperand(Vec0Idx + 1); 2127 if (NumVecs == 2) { 2128 if (is64BitVector) 2129 SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0); 2130 else 2131 SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0); 2132 } else { 2133 SDValue V2 = N->getOperand(Vec0Idx + 2); 2134 SDValue V3 = (NumVecs == 3) 2135 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2136 : N->getOperand(Vec0Idx + 3); 2137 if (is64BitVector) 2138 SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2139 else 2140 SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0); 2141 } 2142 Ops.push_back(SuperReg); 2143 Ops.push_back(getI32Imm(Lane)); 2144 Ops.push_back(Pred); 2145 Ops.push_back(Reg0); 2146 Ops.push_back(Chain); 2147 2148 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : 2149 QOpcodes[OpcodeIndex]); 2150 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2151 cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); 2152 if (!IsLoad) 2153 return VLdLn; 2154 2155 // Extract the subregisters. 2156 SuperReg = SDValue(VLdLn, 0); 2157 assert(ARM::dsub_7 == ARM::dsub_0+7 && 2158 ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering"); 2159 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0; 2160 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2161 ReplaceUses(SDValue(N, Vec), 2162 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg)); 2163 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1)); 2164 if (isUpdating) 2165 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2)); 2166 return nullptr; 2167 } 2168 2169 SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, 2170 unsigned NumVecs, 2171 const uint16_t *Opcodes) { 2172 assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range"); 2173 SDLoc dl(N); 2174 2175 SDValue MemAddr, Align; 2176 if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align)) 2177 return nullptr; 2178 2179 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 2180 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 2181 2182 SDValue Chain = N->getOperand(0); 2183 EVT VT = N->getValueType(0); 2184 2185 unsigned Alignment = 0; 2186 if (NumVecs != 3) { 2187 Alignment = cast<ConstantSDNode>(Align)->getZExtValue(); 2188 unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8; 2189 if (Alignment > NumBytes) 2190 Alignment = NumBytes; 2191 if (Alignment < 8 && Alignment < NumBytes) 2192 Alignment = 0; 2193 // Alignment must be a power of two; make sure of that. 2194 Alignment = (Alignment & -Alignment); 2195 if (Alignment == 1) 2196 Alignment = 0; 2197 } 2198 Align = CurDAG->getTargetConstant(Alignment, MVT::i32); 2199 2200 unsigned OpcodeIndex; 2201 switch (VT.getSimpleVT().SimpleTy) { 2202 default: llvm_unreachable("unhandled vld-dup type"); 2203 case MVT::v8i8: OpcodeIndex = 0; break; 2204 case MVT::v4i16: OpcodeIndex = 1; break; 2205 case MVT::v2f32: 2206 case MVT::v2i32: OpcodeIndex = 2; break; 2207 } 2208 2209 SDValue Pred = getAL(CurDAG); 2210 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2211 SDValue SuperReg; 2212 unsigned Opc = Opcodes[OpcodeIndex]; 2213 SmallVector<SDValue, 6> Ops; 2214 Ops.push_back(MemAddr); 2215 Ops.push_back(Align); 2216 if (isUpdating) { 2217 // fixed-stride update instructions don't have an explicit writeback 2218 // operand. It's implicit in the opcode itself. 2219 SDValue Inc = N->getOperand(2); 2220 if (!isa<ConstantSDNode>(Inc.getNode())) 2221 Ops.push_back(Inc); 2222 // FIXME: VLD3 and VLD4 haven't been updated to that form yet. 2223 else if (NumVecs > 2) 2224 Ops.push_back(Reg0); 2225 } 2226 Ops.push_back(Pred); 2227 Ops.push_back(Reg0); 2228 Ops.push_back(Chain); 2229 2230 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs; 2231 std::vector<EVT> ResTys; 2232 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts)); 2233 if (isUpdating) 2234 ResTys.push_back(MVT::i32); 2235 ResTys.push_back(MVT::Other); 2236 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); 2237 cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); 2238 SuperReg = SDValue(VLdDup, 0); 2239 2240 // Extract the subregisters. 2241 assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering"); 2242 unsigned SubIdx = ARM::dsub_0; 2243 for (unsigned Vec = 0; Vec < NumVecs; ++Vec) 2244 ReplaceUses(SDValue(N, Vec), 2245 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg)); 2246 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1)); 2247 if (isUpdating) 2248 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2)); 2249 return nullptr; 2250 } 2251 2252 SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, 2253 unsigned Opc) { 2254 assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range"); 2255 SDLoc dl(N); 2256 EVT VT = N->getValueType(0); 2257 unsigned FirstTblReg = IsExt ? 2 : 1; 2258 2259 // Form a REG_SEQUENCE to force register allocation. 2260 SDValue RegSeq; 2261 SDValue V0 = N->getOperand(FirstTblReg + 0); 2262 SDValue V1 = N->getOperand(FirstTblReg + 1); 2263 if (NumVecs == 2) 2264 RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 2265 else { 2266 SDValue V2 = N->getOperand(FirstTblReg + 2); 2267 // If it's a vtbl3, form a quad D-register and leave the last part as 2268 // an undef. 2269 SDValue V3 = (NumVecs == 3) 2270 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0) 2271 : N->getOperand(FirstTblReg + 3); 2272 RegSeq = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0); 2273 } 2274 2275 SmallVector<SDValue, 6> Ops; 2276 if (IsExt) 2277 Ops.push_back(N->getOperand(1)); 2278 Ops.push_back(RegSeq); 2279 Ops.push_back(N->getOperand(FirstTblReg + NumVecs)); 2280 Ops.push_back(getAL(CurDAG)); // predicate 2281 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register 2282 return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2283 } 2284 2285 SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N, 2286 bool isSigned) { 2287 if (!Subtarget->hasV6T2Ops()) 2288 return nullptr; 2289 2290 unsigned Opc = isSigned 2291 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX) 2292 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX); 2293 2294 // For unsigned extracts, check for a shift right and mask 2295 unsigned And_imm = 0; 2296 if (N->getOpcode() == ISD::AND) { 2297 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) { 2298 2299 // The immediate is a mask of the low bits iff imm & (imm+1) == 0 2300 if (And_imm & (And_imm + 1)) 2301 return nullptr; 2302 2303 unsigned Srl_imm = 0; 2304 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, 2305 Srl_imm)) { 2306 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2307 2308 // Note: The width operand is encoded as width-1. 2309 unsigned Width = CountTrailingOnes_32(And_imm) - 1; 2310 unsigned LSB = Srl_imm; 2311 2312 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2313 2314 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) { 2315 // It's cheaper to use a right shift to extract the top bits. 2316 if (Subtarget->isThumb()) { 2317 Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri; 2318 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2319 CurDAG->getTargetConstant(LSB, MVT::i32), 2320 getAL(CurDAG), Reg0, Reg0 }; 2321 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2322 } 2323 2324 // ARM models shift instructions as MOVsi with shifter operand. 2325 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL); 2326 SDValue ShOpc = 2327 CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), 2328 MVT::i32); 2329 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc, 2330 getAL(CurDAG), Reg0, Reg0 }; 2331 return CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops); 2332 } 2333 2334 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2335 CurDAG->getTargetConstant(LSB, MVT::i32), 2336 CurDAG->getTargetConstant(Width, MVT::i32), 2337 getAL(CurDAG), Reg0 }; 2338 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2339 } 2340 } 2341 return nullptr; 2342 } 2343 2344 // Otherwise, we're looking for a shift of a shift 2345 unsigned Shl_imm = 0; 2346 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) { 2347 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!"); 2348 unsigned Srl_imm = 0; 2349 if (isInt32Immediate(N->getOperand(1), Srl_imm)) { 2350 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!"); 2351 // Note: The width operand is encoded as width-1. 2352 unsigned Width = 32 - Srl_imm - 1; 2353 int LSB = Srl_imm - Shl_imm; 2354 if (LSB < 0) 2355 return nullptr; 2356 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2357 SDValue Ops[] = { N->getOperand(0).getOperand(0), 2358 CurDAG->getTargetConstant(LSB, MVT::i32), 2359 CurDAG->getTargetConstant(Width, MVT::i32), 2360 getAL(CurDAG), Reg0 }; 2361 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2362 } 2363 } 2364 return nullptr; 2365 } 2366 2367 /// Target-specific DAG combining for ISD::XOR. 2368 /// Target-independent combining lowers SELECT_CC nodes of the form 2369 /// select_cc setg[ge] X, 0, X, -X 2370 /// select_cc setgt X, -1, X, -X 2371 /// select_cc setl[te] X, 0, -X, X 2372 /// select_cc setlt X, 1, -X, X 2373 /// which represent Integer ABS into: 2374 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y) 2375 /// ARM instruction selection detects the latter and matches it to 2376 /// ARM::ABS or ARM::t2ABS machine node. 2377 SDNode *ARMDAGToDAGISel::SelectABSOp(SDNode *N){ 2378 SDValue XORSrc0 = N->getOperand(0); 2379 SDValue XORSrc1 = N->getOperand(1); 2380 EVT VT = N->getValueType(0); 2381 2382 if (Subtarget->isThumb1Only()) 2383 return nullptr; 2384 2385 if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA) 2386 return nullptr; 2387 2388 SDValue ADDSrc0 = XORSrc0.getOperand(0); 2389 SDValue ADDSrc1 = XORSrc0.getOperand(1); 2390 SDValue SRASrc0 = XORSrc1.getOperand(0); 2391 SDValue SRASrc1 = XORSrc1.getOperand(1); 2392 ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1); 2393 EVT XType = SRASrc0.getValueType(); 2394 unsigned Size = XType.getSizeInBits() - 1; 2395 2396 if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 && 2397 XType.isInteger() && SRAConstant != nullptr && 2398 Size == SRAConstant->getZExtValue()) { 2399 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS; 2400 return CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0); 2401 } 2402 2403 return nullptr; 2404 } 2405 2406 SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) { 2407 // The only time a CONCAT_VECTORS operation can have legal types is when 2408 // two 64-bit vectors are concatenated to a 128-bit vector. 2409 EVT VT = N->getValueType(0); 2410 if (!VT.is128BitVector() || N->getNumOperands() != 2) 2411 llvm_unreachable("unexpected CONCAT_VECTORS"); 2412 return createDRegPairNode(VT, N->getOperand(0), N->getOperand(1)); 2413 } 2414 2415 SDNode *ARMDAGToDAGISel::Select(SDNode *N) { 2416 SDLoc dl(N); 2417 2418 if (N->isMachineOpcode()) { 2419 N->setNodeId(-1); 2420 return nullptr; // Already selected. 2421 } 2422 2423 switch (N->getOpcode()) { 2424 default: break; 2425 case ISD::INLINEASM: { 2426 SDNode *ResNode = SelectInlineAsm(N); 2427 if (ResNode) 2428 return ResNode; 2429 break; 2430 } 2431 case ISD::XOR: { 2432 // Select special operations if XOR node forms integer ABS pattern 2433 SDNode *ResNode = SelectABSOp(N); 2434 if (ResNode) 2435 return ResNode; 2436 // Other cases are autogenerated. 2437 break; 2438 } 2439 case ISD::Constant: { 2440 unsigned Val = cast<ConstantSDNode>(N)->getZExtValue(); 2441 bool UseCP = true; 2442 if (Subtarget->useMovt(*MF)) 2443 // Thumb2-aware targets have the MOVT instruction, so all immediates can 2444 // be done with MOV + MOVT, at worst. 2445 UseCP = false; 2446 else { 2447 if (Subtarget->isThumb()) { 2448 UseCP = (Val > 255 && // MOV 2449 ~Val > 255 && // MOV + MVN 2450 !ARM_AM::isThumbImmShiftedVal(Val) && // MOV + LSL 2451 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW 2452 } else 2453 UseCP = (ARM_AM::getSOImmVal(Val) == -1 && // MOV 2454 ARM_AM::getSOImmVal(~Val) == -1 && // MVN 2455 !ARM_AM::isSOImmTwoPartVal(Val) && // two instrs. 2456 !(Subtarget->hasV6T2Ops() && Val <= 0xffff)); // MOVW 2457 } 2458 2459 if (UseCP) { 2460 SDValue CPIdx = 2461 CurDAG->getTargetConstantPool(ConstantInt::get( 2462 Type::getInt32Ty(*CurDAG->getContext()), Val), 2463 getTargetLowering()->getPointerTy()); 2464 2465 SDNode *ResNode; 2466 if (Subtarget->isThumb()) { 2467 SDValue Pred = getAL(CurDAG); 2468 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2469 SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() }; 2470 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other, 2471 Ops); 2472 } else { 2473 SDValue Ops[] = { 2474 CPIdx, 2475 CurDAG->getTargetConstant(0, MVT::i32), 2476 getAL(CurDAG), 2477 CurDAG->getRegister(0, MVT::i32), 2478 CurDAG->getEntryNode() 2479 }; 2480 ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other, 2481 Ops); 2482 } 2483 ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0)); 2484 return nullptr; 2485 } 2486 2487 // Other cases are autogenerated. 2488 break; 2489 } 2490 case ISD::FrameIndex: { 2491 // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm. 2492 int FI = cast<FrameIndexSDNode>(N)->getIndex(); 2493 SDValue TFI = CurDAG->getTargetFrameIndex(FI, 2494 getTargetLowering()->getPointerTy()); 2495 if (Subtarget->isThumb1Only()) { 2496 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2497 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2498 return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, Ops); 2499 } else { 2500 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ? 2501 ARM::t2ADDri : ARM::ADDri); 2502 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32), 2503 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2504 CurDAG->getRegister(0, MVT::i32) }; 2505 return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops); 2506 } 2507 } 2508 case ISD::SRL: 2509 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2510 return I; 2511 break; 2512 case ISD::SRA: 2513 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true)) 2514 return I; 2515 break; 2516 case ISD::MUL: 2517 if (Subtarget->isThumb1Only()) 2518 break; 2519 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) { 2520 unsigned RHSV = C->getZExtValue(); 2521 if (!RHSV) break; 2522 if (isPowerOf2_32(RHSV-1)) { // 2^n+1? 2523 unsigned ShImm = Log2_32(RHSV-1); 2524 if (ShImm >= 32) 2525 break; 2526 SDValue V = N->getOperand(0); 2527 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2528 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2529 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2530 if (Subtarget->isThumb()) { 2531 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2532 return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops); 2533 } else { 2534 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2535 return CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops); 2536 } 2537 } 2538 if (isPowerOf2_32(RHSV+1)) { // 2^n-1? 2539 unsigned ShImm = Log2_32(RHSV+1); 2540 if (ShImm >= 32) 2541 break; 2542 SDValue V = N->getOperand(0); 2543 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm); 2544 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32); 2545 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32); 2546 if (Subtarget->isThumb()) { 2547 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2548 return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops); 2549 } else { 2550 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 }; 2551 return CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops); 2552 } 2553 } 2554 } 2555 break; 2556 case ISD::AND: { 2557 // Check for unsigned bitfield extract 2558 if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false)) 2559 return I; 2560 2561 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits 2562 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits 2563 // are entirely contributed by c2 and lower 16-bits are entirely contributed 2564 // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)). 2565 // Select it to: "movt x, ((c1 & 0xffff) >> 16) 2566 EVT VT = N->getValueType(0); 2567 if (VT != MVT::i32) 2568 break; 2569 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2()) 2570 ? ARM::t2MOVTi16 2571 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0); 2572 if (!Opc) 2573 break; 2574 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1); 2575 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); 2576 if (!N1C) 2577 break; 2578 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) { 2579 SDValue N2 = N0.getOperand(1); 2580 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); 2581 if (!N2C) 2582 break; 2583 unsigned N1CVal = N1C->getZExtValue(); 2584 unsigned N2CVal = N2C->getZExtValue(); 2585 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) && 2586 (N1CVal & 0xffffU) == 0xffffU && 2587 (N2CVal & 0xffffU) == 0x0U) { 2588 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16, 2589 MVT::i32); 2590 SDValue Ops[] = { N0.getOperand(0), Imm16, 2591 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2592 return CurDAG->getMachineNode(Opc, dl, VT, Ops); 2593 } 2594 } 2595 break; 2596 } 2597 case ARMISD::VMOVRRD: 2598 return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32, 2599 N->getOperand(0), getAL(CurDAG), 2600 CurDAG->getRegister(0, MVT::i32)); 2601 case ISD::UMUL_LOHI: { 2602 if (Subtarget->isThumb1Only()) 2603 break; 2604 if (Subtarget->isThumb()) { 2605 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2606 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2607 return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops); 2608 } else { 2609 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2610 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2611 CurDAG->getRegister(0, MVT::i32) }; 2612 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2613 ARM::UMULL : ARM::UMULLv5, 2614 dl, MVT::i32, MVT::i32, Ops); 2615 } 2616 } 2617 case ISD::SMUL_LOHI: { 2618 if (Subtarget->isThumb1Only()) 2619 break; 2620 if (Subtarget->isThumb()) { 2621 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2622 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) }; 2623 return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops); 2624 } else { 2625 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), 2626 getAL(CurDAG), CurDAG->getRegister(0, MVT::i32), 2627 CurDAG->getRegister(0, MVT::i32) }; 2628 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2629 ARM::SMULL : ARM::SMULLv5, 2630 dl, MVT::i32, MVT::i32, Ops); 2631 } 2632 } 2633 case ARMISD::UMLAL:{ 2634 if (Subtarget->isThumb()) { 2635 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2636 N->getOperand(3), getAL(CurDAG), 2637 CurDAG->getRegister(0, MVT::i32)}; 2638 return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops); 2639 }else{ 2640 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2641 N->getOperand(3), getAL(CurDAG), 2642 CurDAG->getRegister(0, MVT::i32), 2643 CurDAG->getRegister(0, MVT::i32) }; 2644 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2645 ARM::UMLAL : ARM::UMLALv5, 2646 dl, MVT::i32, MVT::i32, Ops); 2647 } 2648 } 2649 case ARMISD::SMLAL:{ 2650 if (Subtarget->isThumb()) { 2651 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2652 N->getOperand(3), getAL(CurDAG), 2653 CurDAG->getRegister(0, MVT::i32)}; 2654 return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops); 2655 }else{ 2656 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 2657 N->getOperand(3), getAL(CurDAG), 2658 CurDAG->getRegister(0, MVT::i32), 2659 CurDAG->getRegister(0, MVT::i32) }; 2660 return CurDAG->getMachineNode(Subtarget->hasV6Ops() ? 2661 ARM::SMLAL : ARM::SMLALv5, 2662 dl, MVT::i32, MVT::i32, Ops); 2663 } 2664 } 2665 case ISD::LOAD: { 2666 SDNode *ResNode = nullptr; 2667 if (Subtarget->isThumb() && Subtarget->hasThumb2()) 2668 ResNode = SelectT2IndexedLoad(N); 2669 else 2670 ResNode = SelectARMIndexedLoad(N); 2671 if (ResNode) 2672 return ResNode; 2673 // Other cases are autogenerated. 2674 break; 2675 } 2676 case ARMISD::BRCOND: { 2677 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2678 // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2679 // Pattern complexity = 6 cost = 1 size = 0 2680 2681 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2682 // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc) 2683 // Pattern complexity = 6 cost = 1 size = 0 2684 2685 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc) 2686 // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc) 2687 // Pattern complexity = 6 cost = 1 size = 0 2688 2689 unsigned Opc = Subtarget->isThumb() ? 2690 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc; 2691 SDValue Chain = N->getOperand(0); 2692 SDValue N1 = N->getOperand(1); 2693 SDValue N2 = N->getOperand(2); 2694 SDValue N3 = N->getOperand(3); 2695 SDValue InFlag = N->getOperand(4); 2696 assert(N1.getOpcode() == ISD::BasicBlock); 2697 assert(N2.getOpcode() == ISD::Constant); 2698 assert(N3.getOpcode() == ISD::Register); 2699 2700 SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned) 2701 cast<ConstantSDNode>(N2)->getZExtValue()), 2702 MVT::i32); 2703 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag }; 2704 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, 2705 MVT::Glue, Ops); 2706 Chain = SDValue(ResNode, 0); 2707 if (N->getNumValues() == 2) { 2708 InFlag = SDValue(ResNode, 1); 2709 ReplaceUses(SDValue(N, 1), InFlag); 2710 } 2711 ReplaceUses(SDValue(N, 0), 2712 SDValue(Chain.getNode(), Chain.getResNo())); 2713 return nullptr; 2714 } 2715 case ARMISD::VZIP: { 2716 unsigned Opc = 0; 2717 EVT VT = N->getValueType(0); 2718 switch (VT.getSimpleVT().SimpleTy) { 2719 default: return nullptr; 2720 case MVT::v8i8: Opc = ARM::VZIPd8; break; 2721 case MVT::v4i16: Opc = ARM::VZIPd16; break; 2722 case MVT::v2f32: 2723 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2724 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2725 case MVT::v16i8: Opc = ARM::VZIPq8; break; 2726 case MVT::v8i16: Opc = ARM::VZIPq16; break; 2727 case MVT::v4f32: 2728 case MVT::v4i32: Opc = ARM::VZIPq32; break; 2729 } 2730 SDValue Pred = getAL(CurDAG); 2731 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2732 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2733 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2734 } 2735 case ARMISD::VUZP: { 2736 unsigned Opc = 0; 2737 EVT VT = N->getValueType(0); 2738 switch (VT.getSimpleVT().SimpleTy) { 2739 default: return nullptr; 2740 case MVT::v8i8: Opc = ARM::VUZPd8; break; 2741 case MVT::v4i16: Opc = ARM::VUZPd16; break; 2742 case MVT::v2f32: 2743 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm. 2744 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2745 case MVT::v16i8: Opc = ARM::VUZPq8; break; 2746 case MVT::v8i16: Opc = ARM::VUZPq16; break; 2747 case MVT::v4f32: 2748 case MVT::v4i32: Opc = ARM::VUZPq32; break; 2749 } 2750 SDValue Pred = getAL(CurDAG); 2751 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2752 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2753 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2754 } 2755 case ARMISD::VTRN: { 2756 unsigned Opc = 0; 2757 EVT VT = N->getValueType(0); 2758 switch (VT.getSimpleVT().SimpleTy) { 2759 default: return nullptr; 2760 case MVT::v8i8: Opc = ARM::VTRNd8; break; 2761 case MVT::v4i16: Opc = ARM::VTRNd16; break; 2762 case MVT::v2f32: 2763 case MVT::v2i32: Opc = ARM::VTRNd32; break; 2764 case MVT::v16i8: Opc = ARM::VTRNq8; break; 2765 case MVT::v8i16: Opc = ARM::VTRNq16; break; 2766 case MVT::v4f32: 2767 case MVT::v4i32: Opc = ARM::VTRNq32; break; 2768 } 2769 SDValue Pred = getAL(CurDAG); 2770 SDValue PredReg = CurDAG->getRegister(0, MVT::i32); 2771 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg }; 2772 return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops); 2773 } 2774 case ARMISD::BUILD_VECTOR: { 2775 EVT VecVT = N->getValueType(0); 2776 EVT EltVT = VecVT.getVectorElementType(); 2777 unsigned NumElts = VecVT.getVectorNumElements(); 2778 if (EltVT == MVT::f64) { 2779 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR"); 2780 return createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2781 } 2782 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"); 2783 if (NumElts == 2) 2784 return createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)); 2785 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR"); 2786 return createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1), 2787 N->getOperand(2), N->getOperand(3)); 2788 } 2789 2790 case ARMISD::VLD2DUP: { 2791 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16, 2792 ARM::VLD2DUPd32 }; 2793 return SelectVLDDup(N, false, 2, Opcodes); 2794 } 2795 2796 case ARMISD::VLD3DUP: { 2797 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo, 2798 ARM::VLD3DUPd16Pseudo, 2799 ARM::VLD3DUPd32Pseudo }; 2800 return SelectVLDDup(N, false, 3, Opcodes); 2801 } 2802 2803 case ARMISD::VLD4DUP: { 2804 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo, 2805 ARM::VLD4DUPd16Pseudo, 2806 ARM::VLD4DUPd32Pseudo }; 2807 return SelectVLDDup(N, false, 4, Opcodes); 2808 } 2809 2810 case ARMISD::VLD2DUP_UPD: { 2811 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed, 2812 ARM::VLD2DUPd16wb_fixed, 2813 ARM::VLD2DUPd32wb_fixed }; 2814 return SelectVLDDup(N, true, 2, Opcodes); 2815 } 2816 2817 case ARMISD::VLD3DUP_UPD: { 2818 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD, 2819 ARM::VLD3DUPd16Pseudo_UPD, 2820 ARM::VLD3DUPd32Pseudo_UPD }; 2821 return SelectVLDDup(N, true, 3, Opcodes); 2822 } 2823 2824 case ARMISD::VLD4DUP_UPD: { 2825 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD, 2826 ARM::VLD4DUPd16Pseudo_UPD, 2827 ARM::VLD4DUPd32Pseudo_UPD }; 2828 return SelectVLDDup(N, true, 4, Opcodes); 2829 } 2830 2831 case ARMISD::VLD1_UPD: { 2832 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed, 2833 ARM::VLD1d16wb_fixed, 2834 ARM::VLD1d32wb_fixed, 2835 ARM::VLD1d64wb_fixed }; 2836 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed, 2837 ARM::VLD1q16wb_fixed, 2838 ARM::VLD1q32wb_fixed, 2839 ARM::VLD1q64wb_fixed }; 2840 return SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr); 2841 } 2842 2843 case ARMISD::VLD2_UPD: { 2844 static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed, 2845 ARM::VLD2d16wb_fixed, 2846 ARM::VLD2d32wb_fixed, 2847 ARM::VLD1q64wb_fixed}; 2848 static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed, 2849 ARM::VLD2q16PseudoWB_fixed, 2850 ARM::VLD2q32PseudoWB_fixed }; 2851 return SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr); 2852 } 2853 2854 case ARMISD::VLD3_UPD: { 2855 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD, 2856 ARM::VLD3d16Pseudo_UPD, 2857 ARM::VLD3d32Pseudo_UPD, 2858 ARM::VLD1d64TPseudoWB_fixed}; 2859 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 2860 ARM::VLD3q16Pseudo_UPD, 2861 ARM::VLD3q32Pseudo_UPD }; 2862 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD, 2863 ARM::VLD3q16oddPseudo_UPD, 2864 ARM::VLD3q32oddPseudo_UPD }; 2865 return SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2866 } 2867 2868 case ARMISD::VLD4_UPD: { 2869 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD, 2870 ARM::VLD4d16Pseudo_UPD, 2871 ARM::VLD4d32Pseudo_UPD, 2872 ARM::VLD1d64QPseudoWB_fixed}; 2873 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 2874 ARM::VLD4q16Pseudo_UPD, 2875 ARM::VLD4q32Pseudo_UPD }; 2876 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD, 2877 ARM::VLD4q16oddPseudo_UPD, 2878 ARM::VLD4q32oddPseudo_UPD }; 2879 return SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 2880 } 2881 2882 case ARMISD::VLD2LN_UPD: { 2883 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD, 2884 ARM::VLD2LNd16Pseudo_UPD, 2885 ARM::VLD2LNd32Pseudo_UPD }; 2886 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD, 2887 ARM::VLD2LNq32Pseudo_UPD }; 2888 return SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes); 2889 } 2890 2891 case ARMISD::VLD3LN_UPD: { 2892 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD, 2893 ARM::VLD3LNd16Pseudo_UPD, 2894 ARM::VLD3LNd32Pseudo_UPD }; 2895 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD, 2896 ARM::VLD3LNq32Pseudo_UPD }; 2897 return SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes); 2898 } 2899 2900 case ARMISD::VLD4LN_UPD: { 2901 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD, 2902 ARM::VLD4LNd16Pseudo_UPD, 2903 ARM::VLD4LNd32Pseudo_UPD }; 2904 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD, 2905 ARM::VLD4LNq32Pseudo_UPD }; 2906 return SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes); 2907 } 2908 2909 case ARMISD::VST1_UPD: { 2910 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed, 2911 ARM::VST1d16wb_fixed, 2912 ARM::VST1d32wb_fixed, 2913 ARM::VST1d64wb_fixed }; 2914 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed, 2915 ARM::VST1q16wb_fixed, 2916 ARM::VST1q32wb_fixed, 2917 ARM::VST1q64wb_fixed }; 2918 return SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr); 2919 } 2920 2921 case ARMISD::VST2_UPD: { 2922 static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed, 2923 ARM::VST2d16wb_fixed, 2924 ARM::VST2d32wb_fixed, 2925 ARM::VST1q64wb_fixed}; 2926 static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed, 2927 ARM::VST2q16PseudoWB_fixed, 2928 ARM::VST2q32PseudoWB_fixed }; 2929 return SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); 2930 } 2931 2932 case ARMISD::VST3_UPD: { 2933 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD, 2934 ARM::VST3d16Pseudo_UPD, 2935 ARM::VST3d32Pseudo_UPD, 2936 ARM::VST1d64TPseudoWB_fixed}; 2937 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 2938 ARM::VST3q16Pseudo_UPD, 2939 ARM::VST3q32Pseudo_UPD }; 2940 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD, 2941 ARM::VST3q16oddPseudo_UPD, 2942 ARM::VST3q32oddPseudo_UPD }; 2943 return SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); 2944 } 2945 2946 case ARMISD::VST4_UPD: { 2947 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD, 2948 ARM::VST4d16Pseudo_UPD, 2949 ARM::VST4d32Pseudo_UPD, 2950 ARM::VST1d64QPseudoWB_fixed}; 2951 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 2952 ARM::VST4q16Pseudo_UPD, 2953 ARM::VST4q32Pseudo_UPD }; 2954 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD, 2955 ARM::VST4q16oddPseudo_UPD, 2956 ARM::VST4q32oddPseudo_UPD }; 2957 return SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); 2958 } 2959 2960 case ARMISD::VST2LN_UPD: { 2961 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, 2962 ARM::VST2LNd16Pseudo_UPD, 2963 ARM::VST2LNd32Pseudo_UPD }; 2964 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD, 2965 ARM::VST2LNq32Pseudo_UPD }; 2966 return SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes); 2967 } 2968 2969 case ARMISD::VST3LN_UPD: { 2970 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD, 2971 ARM::VST3LNd16Pseudo_UPD, 2972 ARM::VST3LNd32Pseudo_UPD }; 2973 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD, 2974 ARM::VST3LNq32Pseudo_UPD }; 2975 return SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes); 2976 } 2977 2978 case ARMISD::VST4LN_UPD: { 2979 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD, 2980 ARM::VST4LNd16Pseudo_UPD, 2981 ARM::VST4LNd32Pseudo_UPD }; 2982 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD, 2983 ARM::VST4LNq32Pseudo_UPD }; 2984 return SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes); 2985 } 2986 2987 case ISD::INTRINSIC_VOID: 2988 case ISD::INTRINSIC_W_CHAIN: { 2989 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue(); 2990 switch (IntNo) { 2991 default: 2992 break; 2993 2994 case Intrinsic::arm_ldaexd: 2995 case Intrinsic::arm_ldrexd: { 2996 SDLoc dl(N); 2997 SDValue Chain = N->getOperand(0); 2998 SDValue MemAddr = N->getOperand(2); 2999 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3000 3001 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd; 3002 unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD) 3003 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD); 3004 3005 // arm_ldrexd returns a i64 value in {i32, i32} 3006 std::vector<EVT> ResTys; 3007 if (isThumb) { 3008 ResTys.push_back(MVT::i32); 3009 ResTys.push_back(MVT::i32); 3010 } else 3011 ResTys.push_back(MVT::Untyped); 3012 ResTys.push_back(MVT::Other); 3013 3014 // Place arguments in the right order. 3015 SmallVector<SDValue, 7> Ops; 3016 Ops.push_back(MemAddr); 3017 Ops.push_back(getAL(CurDAG)); 3018 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3019 Ops.push_back(Chain); 3020 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3021 // Transfer memoperands. 3022 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3023 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3024 cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); 3025 3026 // Remap uses. 3027 SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1); 3028 if (!SDValue(N, 0).use_empty()) { 3029 SDValue Result; 3030 if (isThumb) 3031 Result = SDValue(Ld, 0); 3032 else { 3033 SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_0, MVT::i32); 3034 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3035 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3036 Result = SDValue(ResNode,0); 3037 } 3038 ReplaceUses(SDValue(N, 0), Result); 3039 } 3040 if (!SDValue(N, 1).use_empty()) { 3041 SDValue Result; 3042 if (isThumb) 3043 Result = SDValue(Ld, 1); 3044 else { 3045 SDValue SubRegIdx = CurDAG->getTargetConstant(ARM::gsub_1, MVT::i32); 3046 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 3047 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx); 3048 Result = SDValue(ResNode,0); 3049 } 3050 ReplaceUses(SDValue(N, 1), Result); 3051 } 3052 ReplaceUses(SDValue(N, 2), OutChain); 3053 return nullptr; 3054 } 3055 case Intrinsic::arm_stlexd: 3056 case Intrinsic::arm_strexd: { 3057 SDLoc dl(N); 3058 SDValue Chain = N->getOperand(0); 3059 SDValue Val0 = N->getOperand(2); 3060 SDValue Val1 = N->getOperand(3); 3061 SDValue MemAddr = N->getOperand(4); 3062 3063 // Store exclusive double return a i32 value which is the return status 3064 // of the issued store. 3065 EVT ResTys[] = { MVT::i32, MVT::Other }; 3066 3067 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2(); 3068 // Place arguments in the right order. 3069 SmallVector<SDValue, 7> Ops; 3070 if (isThumb) { 3071 Ops.push_back(Val0); 3072 Ops.push_back(Val1); 3073 } else 3074 // arm_strexd uses GPRPair. 3075 Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0)); 3076 Ops.push_back(MemAddr); 3077 Ops.push_back(getAL(CurDAG)); 3078 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); 3079 Ops.push_back(Chain); 3080 3081 bool IsRelease = IntNo == Intrinsic::arm_stlexd; 3082 unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD) 3083 : (IsRelease ? ARM::STLEXD : ARM::STREXD); 3084 3085 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); 3086 // Transfer memoperands. 3087 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 3088 MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); 3089 cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); 3090 3091 return St; 3092 } 3093 3094 case Intrinsic::arm_neon_vld1: { 3095 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16, 3096 ARM::VLD1d32, ARM::VLD1d64 }; 3097 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16, 3098 ARM::VLD1q32, ARM::VLD1q64}; 3099 return SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr); 3100 } 3101 3102 case Intrinsic::arm_neon_vld2: { 3103 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16, 3104 ARM::VLD2d32, ARM::VLD1q64 }; 3105 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo, 3106 ARM::VLD2q32Pseudo }; 3107 return SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr); 3108 } 3109 3110 case Intrinsic::arm_neon_vld3: { 3111 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo, 3112 ARM::VLD3d16Pseudo, 3113 ARM::VLD3d32Pseudo, 3114 ARM::VLD1d64TPseudo }; 3115 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD, 3116 ARM::VLD3q16Pseudo_UPD, 3117 ARM::VLD3q32Pseudo_UPD }; 3118 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo, 3119 ARM::VLD3q16oddPseudo, 3120 ARM::VLD3q32oddPseudo }; 3121 return SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3122 } 3123 3124 case Intrinsic::arm_neon_vld4: { 3125 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo, 3126 ARM::VLD4d16Pseudo, 3127 ARM::VLD4d32Pseudo, 3128 ARM::VLD1d64QPseudo }; 3129 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD, 3130 ARM::VLD4q16Pseudo_UPD, 3131 ARM::VLD4q32Pseudo_UPD }; 3132 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo, 3133 ARM::VLD4q16oddPseudo, 3134 ARM::VLD4q32oddPseudo }; 3135 return SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3136 } 3137 3138 case Intrinsic::arm_neon_vld2lane: { 3139 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo, 3140 ARM::VLD2LNd16Pseudo, 3141 ARM::VLD2LNd32Pseudo }; 3142 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo, 3143 ARM::VLD2LNq32Pseudo }; 3144 return SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes); 3145 } 3146 3147 case Intrinsic::arm_neon_vld3lane: { 3148 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo, 3149 ARM::VLD3LNd16Pseudo, 3150 ARM::VLD3LNd32Pseudo }; 3151 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo, 3152 ARM::VLD3LNq32Pseudo }; 3153 return SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes); 3154 } 3155 3156 case Intrinsic::arm_neon_vld4lane: { 3157 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo, 3158 ARM::VLD4LNd16Pseudo, 3159 ARM::VLD4LNd32Pseudo }; 3160 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo, 3161 ARM::VLD4LNq32Pseudo }; 3162 return SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes); 3163 } 3164 3165 case Intrinsic::arm_neon_vst1: { 3166 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16, 3167 ARM::VST1d32, ARM::VST1d64 }; 3168 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16, 3169 ARM::VST1q32, ARM::VST1q64 }; 3170 return SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr); 3171 } 3172 3173 case Intrinsic::arm_neon_vst2: { 3174 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16, 3175 ARM::VST2d32, ARM::VST1q64 }; 3176 static uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo, 3177 ARM::VST2q32Pseudo }; 3178 return SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr); 3179 } 3180 3181 case Intrinsic::arm_neon_vst3: { 3182 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo, 3183 ARM::VST3d16Pseudo, 3184 ARM::VST3d32Pseudo, 3185 ARM::VST1d64TPseudo }; 3186 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD, 3187 ARM::VST3q16Pseudo_UPD, 3188 ARM::VST3q32Pseudo_UPD }; 3189 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo, 3190 ARM::VST3q16oddPseudo, 3191 ARM::VST3q32oddPseudo }; 3192 return SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1); 3193 } 3194 3195 case Intrinsic::arm_neon_vst4: { 3196 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo, 3197 ARM::VST4d16Pseudo, 3198 ARM::VST4d32Pseudo, 3199 ARM::VST1d64QPseudo }; 3200 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD, 3201 ARM::VST4q16Pseudo_UPD, 3202 ARM::VST4q32Pseudo_UPD }; 3203 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo, 3204 ARM::VST4q16oddPseudo, 3205 ARM::VST4q32oddPseudo }; 3206 return SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1); 3207 } 3208 3209 case Intrinsic::arm_neon_vst2lane: { 3210 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo, 3211 ARM::VST2LNd16Pseudo, 3212 ARM::VST2LNd32Pseudo }; 3213 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo, 3214 ARM::VST2LNq32Pseudo }; 3215 return SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes); 3216 } 3217 3218 case Intrinsic::arm_neon_vst3lane: { 3219 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo, 3220 ARM::VST3LNd16Pseudo, 3221 ARM::VST3LNd32Pseudo }; 3222 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo, 3223 ARM::VST3LNq32Pseudo }; 3224 return SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes); 3225 } 3226 3227 case Intrinsic::arm_neon_vst4lane: { 3228 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo, 3229 ARM::VST4LNd16Pseudo, 3230 ARM::VST4LNd32Pseudo }; 3231 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo, 3232 ARM::VST4LNq32Pseudo }; 3233 return SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes); 3234 } 3235 } 3236 break; 3237 } 3238 3239 case ISD::INTRINSIC_WO_CHAIN: { 3240 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 3241 switch (IntNo) { 3242 default: 3243 break; 3244 3245 case Intrinsic::arm_neon_vtbl2: 3246 return SelectVTBL(N, false, 2, ARM::VTBL2); 3247 case Intrinsic::arm_neon_vtbl3: 3248 return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo); 3249 case Intrinsic::arm_neon_vtbl4: 3250 return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo); 3251 3252 case Intrinsic::arm_neon_vtbx2: 3253 return SelectVTBL(N, true, 2, ARM::VTBX2); 3254 case Intrinsic::arm_neon_vtbx3: 3255 return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo); 3256 case Intrinsic::arm_neon_vtbx4: 3257 return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo); 3258 } 3259 break; 3260 } 3261 3262 case ARMISD::VTBL1: { 3263 SDLoc dl(N); 3264 EVT VT = N->getValueType(0); 3265 SmallVector<SDValue, 6> Ops; 3266 3267 Ops.push_back(N->getOperand(0)); 3268 Ops.push_back(N->getOperand(1)); 3269 Ops.push_back(getAL(CurDAG)); // Predicate 3270 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3271 return CurDAG->getMachineNode(ARM::VTBL1, dl, VT, Ops); 3272 } 3273 case ARMISD::VTBL2: { 3274 SDLoc dl(N); 3275 EVT VT = N->getValueType(0); 3276 3277 // Form a REG_SEQUENCE to force register allocation. 3278 SDValue V0 = N->getOperand(0); 3279 SDValue V1 = N->getOperand(1); 3280 SDValue RegSeq = SDValue(createDRegPairNode(MVT::v16i8, V0, V1), 0); 3281 3282 SmallVector<SDValue, 6> Ops; 3283 Ops.push_back(RegSeq); 3284 Ops.push_back(N->getOperand(2)); 3285 Ops.push_back(getAL(CurDAG)); // Predicate 3286 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // Predicate Register 3287 return CurDAG->getMachineNode(ARM::VTBL2, dl, VT, Ops); 3288 } 3289 3290 case ISD::CONCAT_VECTORS: 3291 return SelectConcatVector(N); 3292 } 3293 3294 return SelectCode(N); 3295 } 3296 3297 SDNode *ARMDAGToDAGISel::SelectInlineAsm(SDNode *N){ 3298 std::vector<SDValue> AsmNodeOperands; 3299 unsigned Flag, Kind; 3300 bool Changed = false; 3301 unsigned NumOps = N->getNumOperands(); 3302 3303 // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint. 3304 // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require 3305 // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs 3306 // respectively. Since there is no constraint to explicitly specify a 3307 // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb, 3308 // the 64-bit data may be referred by H, Q, R modifiers, so we still pack 3309 // them into a GPRPair. 3310 3311 SDLoc dl(N); 3312 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1) 3313 : SDValue(nullptr,0); 3314 3315 SmallVector<bool, 8> OpChanged; 3316 // Glue node will be appended late. 3317 for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) { 3318 SDValue op = N->getOperand(i); 3319 AsmNodeOperands.push_back(op); 3320 3321 if (i < InlineAsm::Op_FirstOperand) 3322 continue; 3323 3324 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) { 3325 Flag = C->getZExtValue(); 3326 Kind = InlineAsm::getKind(Flag); 3327 } 3328 else 3329 continue; 3330 3331 // Immediate operands to inline asm in the SelectionDAG are modeled with 3332 // two operands. The first is a constant of value InlineAsm::Kind_Imm, and 3333 // the second is a constant with the value of the immediate. If we get here 3334 // and we have a Kind_Imm, skip the next operand, and continue. 3335 if (Kind == InlineAsm::Kind_Imm) { 3336 SDValue op = N->getOperand(++i); 3337 AsmNodeOperands.push_back(op); 3338 continue; 3339 } 3340 3341 unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag); 3342 if (NumRegs) 3343 OpChanged.push_back(false); 3344 3345 unsigned DefIdx = 0; 3346 bool IsTiedToChangedOp = false; 3347 // If it's a use that is tied with a previous def, it has no 3348 // reg class constraint. 3349 if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx)) 3350 IsTiedToChangedOp = OpChanged[DefIdx]; 3351 3352 if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef 3353 && Kind != InlineAsm::Kind_RegDefEarlyClobber) 3354 continue; 3355 3356 unsigned RC; 3357 bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC); 3358 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID)) 3359 || NumRegs != 2) 3360 continue; 3361 3362 assert((i+2 < NumOps) && "Invalid number of operands in inline asm"); 3363 SDValue V0 = N->getOperand(i+1); 3364 SDValue V1 = N->getOperand(i+2); 3365 unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg(); 3366 unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg(); 3367 SDValue PairedReg; 3368 MachineRegisterInfo &MRI = MF->getRegInfo(); 3369 3370 if (Kind == InlineAsm::Kind_RegDef || 3371 Kind == InlineAsm::Kind_RegDefEarlyClobber) { 3372 // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to 3373 // the original GPRs. 3374 3375 unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3376 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3377 SDValue Chain = SDValue(N,0); 3378 3379 SDNode *GU = N->getGluedUser(); 3380 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped, 3381 Chain.getValue(1)); 3382 3383 // Extract values from a GPRPair reg and copy to the original GPR reg. 3384 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32, 3385 RegCopy); 3386 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32, 3387 RegCopy); 3388 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0, 3389 RegCopy.getValue(1)); 3390 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1)); 3391 3392 // Update the original glue user. 3393 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1); 3394 Ops.push_back(T1.getValue(1)); 3395 CurDAG->UpdateNodeOperands(GU, Ops); 3396 GU = T1.getNode(); 3397 } 3398 else { 3399 // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a 3400 // GPRPair and then pass the GPRPair to the inline asm. 3401 SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain]; 3402 3403 // As REG_SEQ doesn't take RegisterSDNode, we copy them first. 3404 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32, 3405 Chain.getValue(1)); 3406 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32, 3407 T0.getValue(1)); 3408 SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0); 3409 3410 // Copy REG_SEQ into a GPRPair-typed VR and replace the original two 3411 // i32 VRs of inline asm with it. 3412 unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass); 3413 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped); 3414 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1)); 3415 3416 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 3417 Glue = Chain.getValue(1); 3418 } 3419 3420 Changed = true; 3421 3422 if(PairedReg.getNode()) { 3423 OpChanged[OpChanged.size() -1 ] = true; 3424 Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/); 3425 if (IsTiedToChangedOp) 3426 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx); 3427 else 3428 Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID); 3429 // Replace the current flag. 3430 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant( 3431 Flag, MVT::i32); 3432 // Add the new register node and skip the original two GPRs. 3433 AsmNodeOperands.push_back(PairedReg); 3434 // Skip the next two GPRs. 3435 i += 2; 3436 } 3437 } 3438 3439 if (Glue.getNode()) 3440 AsmNodeOperands.push_back(Glue); 3441 if (!Changed) 3442 return nullptr; 3443 3444 SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N), 3445 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); 3446 New->setNodeId(-1); 3447 return New.getNode(); 3448 } 3449 3450 3451 bool ARMDAGToDAGISel:: 3452 SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 3453 std::vector<SDValue> &OutOps) { 3454 assert(ConstraintCode == 'm' && "unexpected asm memory constraint"); 3455 // Require the address to be in a register. That is safe for all ARM 3456 // variants and it is hard to do anything much smarter without knowing 3457 // how the operand is used. 3458 OutOps.push_back(Op); 3459 return false; 3460 } 3461 3462 /// createARMISelDag - This pass converts a legalized DAG into a 3463 /// ARM-specific DAG, ready for instruction scheduling. 3464 /// 3465 FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM, 3466 CodeGenOpt::Level OptLevel) { 3467 return new ARMDAGToDAGISel(TM, OptLevel); 3468 } 3469