1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //==-----------------------------------------------------------------------===// 9 // 10 /// \file 11 /// Defines an instruction selector for the AMDGPU target. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "AMDGPU.h" 16 #include "AMDGPUArgumentUsageInfo.h" 17 #include "AMDGPUISelLowering.h" // For AMDGPUISD 18 #include "AMDGPUInstrInfo.h" 19 #include "AMDGPUPerfHintAnalysis.h" 20 #include "AMDGPURegisterInfo.h" 21 #include "AMDGPUSubtarget.h" 22 #include "AMDGPUTargetMachine.h" 23 #include "SIDefines.h" 24 #include "SIISelLowering.h" 25 #include "SIInstrInfo.h" 26 #include "SIMachineFunctionInfo.h" 27 #include "SIRegisterInfo.h" 28 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 29 #include "llvm/ADT/APInt.h" 30 #include "llvm/ADT/SmallVector.h" 31 #include "llvm/ADT/StringRef.h" 32 #include "llvm/Analysis/DivergenceAnalysis.h" 33 #include "llvm/Analysis/ValueTracking.h" 34 #include "llvm/CodeGen/FunctionLoweringInfo.h" 35 #include "llvm/CodeGen/ISDOpcodes.h" 36 #include "llvm/CodeGen/MachineFunction.h" 37 #include "llvm/CodeGen/MachineRegisterInfo.h" 38 #include "llvm/CodeGen/SelectionDAG.h" 39 #include "llvm/CodeGen/SelectionDAGISel.h" 40 #include "llvm/CodeGen/SelectionDAGNodes.h" 41 #include "llvm/CodeGen/ValueTypes.h" 42 #include "llvm/IR/BasicBlock.h" 43 #include "llvm/IR/Instruction.h" 44 #include "llvm/MC/MCInstrDesc.h" 45 #include "llvm/Support/Casting.h" 46 #include "llvm/Support/CodeGen.h" 47 #include "llvm/Support/ErrorHandling.h" 48 #include "llvm/Support/MachineValueType.h" 49 #include "llvm/Support/MathExtras.h" 50 #include <cassert> 51 #include <cstdint> 52 #include <new> 53 #include <vector> 54 55 using namespace llvm; 56 57 namespace llvm { 58 59 class R600InstrInfo; 60 61 } // end namespace llvm 62 63 //===----------------------------------------------------------------------===// 64 // Instruction Selector Implementation 65 //===----------------------------------------------------------------------===// 66 67 namespace { 68 69 /// AMDGPU specific code to select AMDGPU machine instructions for 70 /// SelectionDAG operations. 71 class AMDGPUDAGToDAGISel : public SelectionDAGISel { 72 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 73 // make the right decision when generating code for different targets. 74 const GCNSubtarget *Subtarget; 75 AMDGPUAS AMDGPUASI; 76 bool EnableLateStructurizeCFG; 77 78 public: 79 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr, 80 CodeGenOpt::Level OptLevel = CodeGenOpt::Default) 81 : SelectionDAGISel(*TM, OptLevel) { 82 AMDGPUASI = AMDGPU::getAMDGPUAS(*TM); 83 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG; 84 } 85 ~AMDGPUDAGToDAGISel() override = default; 86 87 void getAnalysisUsage(AnalysisUsage &AU) const override { 88 AU.addRequired<AMDGPUArgumentUsageInfo>(); 89 AU.addRequired<AMDGPUPerfHintAnalysis>(); 90 AU.addRequired<DivergenceAnalysis>(); 91 SelectionDAGISel::getAnalysisUsage(AU); 92 } 93 94 bool runOnMachineFunction(MachineFunction &MF) override; 95 void Select(SDNode *N) override; 96 StringRef getPassName() const override; 97 void PostprocessISelDAG() override; 98 99 protected: 100 void SelectBuildVector(SDNode *N, unsigned RegClassID); 101 102 private: 103 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const; 104 bool isNoNanSrc(SDValue N) const; 105 bool isInlineImmediate(const SDNode *N) const; 106 107 bool isUniformBr(const SDNode *N) const; 108 109 SDNode *glueCopyToM0(SDNode *N) const; 110 111 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const; 112 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 113 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset); 114 bool isDSOffsetLegal(const SDValue &Base, unsigned Offset, 115 unsigned OffsetBits) const; 116 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const; 117 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0, 118 SDValue &Offset1) const; 119 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 120 SDValue &SOffset, SDValue &Offset, SDValue &Offen, 121 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC, 122 SDValue &TFE) const; 123 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr, 124 SDValue &SOffset, SDValue &Offset, SDValue &GLC, 125 SDValue &SLC, SDValue &TFE) const; 126 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 127 SDValue &VAddr, SDValue &SOffset, SDValue &Offset, 128 SDValue &SLC) const; 129 bool SelectMUBUFScratchOffen(SDNode *Parent, 130 SDValue Addr, SDValue &RSrc, SDValue &VAddr, 131 SDValue &SOffset, SDValue &ImmOffset) const; 132 bool SelectMUBUFScratchOffset(SDNode *Parent, 133 SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 134 SDValue &Offset) const; 135 136 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset, 137 SDValue &Offset, SDValue &GLC, SDValue &SLC, 138 SDValue &TFE) const; 139 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 140 SDValue &Offset, SDValue &SLC) const; 141 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset, 142 SDValue &Offset) const; 143 bool SelectMUBUFConstant(SDValue Constant, 144 SDValue &SOffset, 145 SDValue &ImmOffset) const; 146 bool SelectMUBUFIntrinsicOffset(SDValue Offset, SDValue &SOffset, 147 SDValue &ImmOffset) const; 148 bool SelectMUBUFIntrinsicVOffset(SDValue Offset, SDValue &SOffset, 149 SDValue &ImmOffset, SDValue &VOffset) const; 150 151 bool SelectFlatAtomic(SDValue Addr, SDValue &VAddr, 152 SDValue &Offset, SDValue &SLC) const; 153 bool SelectFlatAtomicSigned(SDValue Addr, SDValue &VAddr, 154 SDValue &Offset, SDValue &SLC) const; 155 156 template <bool IsSigned> 157 bool SelectFlatOffset(SDValue Addr, SDValue &VAddr, 158 SDValue &Offset, SDValue &SLC) const; 159 160 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset, 161 bool &Imm) const; 162 SDValue Expand32BitAddress(SDValue Addr) const; 163 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, 164 bool &Imm) const; 165 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 166 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 167 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const; 168 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const; 169 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const; 170 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const; 171 172 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const; 173 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const; 174 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 175 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const; 176 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods, 177 SDValue &Clamp, SDValue &Omod) const; 178 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 179 SDValue &Clamp, SDValue &Omod) const; 180 181 bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods, 182 SDValue &Clamp, 183 SDValue &Omod) const; 184 185 bool SelectVOP3OMods(SDValue In, SDValue &Src, 186 SDValue &Clamp, SDValue &Omod) const; 187 188 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 189 bool SelectVOP3PMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 190 SDValue &Clamp) const; 191 192 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const; 193 bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods, 194 SDValue &Clamp) const; 195 196 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 197 bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods, 198 SDValue &Clamp) const; 199 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const; 200 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const; 201 202 bool SelectHi16Elt(SDValue In, SDValue &Src) const; 203 204 void SelectADD_SUB_I64(SDNode *N); 205 void SelectUADDO_USUBO(SDNode *N); 206 void SelectDIV_SCALE(SDNode *N); 207 void SelectMAD_64_32(SDNode *N); 208 void SelectFMA_W_CHAIN(SDNode *N); 209 void SelectFMUL_W_CHAIN(SDNode *N); 210 211 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, 212 uint32_t Offset, uint32_t Width); 213 void SelectS_BFEFromShifts(SDNode *N); 214 void SelectS_BFE(SDNode *N); 215 bool isCBranchSCC(const SDNode *N) const; 216 void SelectBRCOND(SDNode *N); 217 void SelectFMAD_FMA(SDNode *N); 218 void SelectATOMIC_CMP_SWAP(SDNode *N); 219 220 protected: 221 // Include the pieces autogenerated from the target description. 222 #include "AMDGPUGenDAGISel.inc" 223 }; 224 225 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel { 226 const R600Subtarget *Subtarget; 227 AMDGPUAS AMDGPUASI; 228 229 bool isConstantLoad(const MemSDNode *N, int cbID) const; 230 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr); 231 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg, 232 SDValue& Offset); 233 public: 234 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) : 235 AMDGPUDAGToDAGISel(TM, OptLevel) { 236 AMDGPUASI = AMDGPU::getAMDGPUAS(*TM); 237 } 238 239 void Select(SDNode *N) override; 240 241 bool SelectADDRIndirect(SDValue Addr, SDValue &Base, 242 SDValue &Offset) override; 243 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 244 SDValue &Offset) override; 245 246 bool runOnMachineFunction(MachineFunction &MF) override; 247 protected: 248 // Include the pieces autogenerated from the target description. 249 #include "R600GenDAGISel.inc" 250 }; 251 252 } // end anonymous namespace 253 254 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "isel", 255 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 256 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo) 257 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis) 258 INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis) 259 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "isel", 260 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false) 261 262 /// This pass converts a legalized DAG into a AMDGPU-specific 263 // DAG, ready for instruction scheduling. 264 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM, 265 CodeGenOpt::Level OptLevel) { 266 return new AMDGPUDAGToDAGISel(TM, OptLevel); 267 } 268 269 /// This pass converts a legalized DAG into a R600-specific 270 // DAG, ready for instruction scheduling. 271 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM, 272 CodeGenOpt::Level OptLevel) { 273 return new R600DAGToDAGISel(TM, OptLevel); 274 } 275 276 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 277 Subtarget = &MF.getSubtarget<GCNSubtarget>(); 278 return SelectionDAGISel::runOnMachineFunction(MF); 279 } 280 281 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const { 282 if (TM.Options.NoNaNsFPMath) 283 return true; 284 285 // TODO: Move into isKnownNeverNaN 286 if (N->getFlags().isDefined()) 287 return N->getFlags().hasNoNaNs(); 288 289 return CurDAG->isKnownNeverNaN(N); 290 } 291 292 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N) const { 293 const SIInstrInfo *TII = Subtarget->getInstrInfo(); 294 295 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) 296 return TII->isInlineConstant(C->getAPIntValue()); 297 298 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) 299 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); 300 301 return false; 302 } 303 304 /// Determine the register class for \p OpNo 305 /// \returns The register class of the virtual register that will be used for 306 /// the given operand number \OpNo or NULL if the register class cannot be 307 /// determined. 308 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, 309 unsigned OpNo) const { 310 if (!N->isMachineOpcode()) { 311 if (N->getOpcode() == ISD::CopyToReg) { 312 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg(); 313 if (TargetRegisterInfo::isVirtualRegister(Reg)) { 314 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); 315 return MRI.getRegClass(Reg); 316 } 317 318 const SIRegisterInfo *TRI 319 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo(); 320 return TRI->getPhysRegClass(Reg); 321 } 322 323 return nullptr; 324 } 325 326 switch (N->getMachineOpcode()) { 327 default: { 328 const MCInstrDesc &Desc = 329 Subtarget->getInstrInfo()->get(N->getMachineOpcode()); 330 unsigned OpIdx = Desc.getNumDefs() + OpNo; 331 if (OpIdx >= Desc.getNumOperands()) 332 return nullptr; 333 int RegClass = Desc.OpInfo[OpIdx].RegClass; 334 if (RegClass == -1) 335 return nullptr; 336 337 return Subtarget->getRegisterInfo()->getRegClass(RegClass); 338 } 339 case AMDGPU::REG_SEQUENCE: { 340 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue(); 341 const TargetRegisterClass *SuperRC = 342 Subtarget->getRegisterInfo()->getRegClass(RCID); 343 344 SDValue SubRegOp = N->getOperand(OpNo + 1); 345 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue(); 346 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC, 347 SubRegIdx); 348 } 349 } 350 } 351 352 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const { 353 if (cast<MemSDNode>(N)->getAddressSpace() != AMDGPUASI.LOCAL_ADDRESS || 354 !Subtarget->ldsRequiresM0Init()) 355 return N; 356 357 const SITargetLowering& Lowering = 358 *static_cast<const SITargetLowering*>(getTargetLowering()); 359 360 // Write max value to m0 before each load operation 361 362 SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N), 363 CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32)); 364 365 SDValue Glue = M0.getValue(1); 366 367 SmallVector <SDValue, 8> Ops; 368 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { 369 Ops.push_back(N->getOperand(i)); 370 } 371 Ops.push_back(Glue); 372 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops); 373 } 374 375 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) { 376 switch (NumVectorElts) { 377 case 1: 378 return AMDGPU::SReg_32_XM0RegClassID; 379 case 2: 380 return AMDGPU::SReg_64RegClassID; 381 case 4: 382 return AMDGPU::SReg_128RegClassID; 383 case 8: 384 return AMDGPU::SReg_256RegClassID; 385 case 16: 386 return AMDGPU::SReg_512RegClassID; 387 } 388 389 llvm_unreachable("invalid vector size"); 390 } 391 392 static bool getConstantValue(SDValue N, uint32_t &Out) { 393 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) { 394 Out = C->getAPIntValue().getZExtValue(); 395 return true; 396 } 397 398 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) { 399 Out = C->getValueAPF().bitcastToAPInt().getZExtValue(); 400 return true; 401 } 402 403 return false; 404 } 405 406 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) { 407 EVT VT = N->getValueType(0); 408 unsigned NumVectorElts = VT.getVectorNumElements(); 409 EVT EltVT = VT.getVectorElementType(); 410 SDLoc DL(N); 411 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 412 413 if (NumVectorElts == 1) { 414 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0), 415 RegClass); 416 return; 417 } 418 419 assert(NumVectorElts <= 16 && "Vectors with more than 16 elements not " 420 "supported yet"); 421 // 16 = Max Num Vector Elements 422 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index) 423 // 1 = Vector Register Class 424 SmallVector<SDValue, 16 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1); 425 426 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32); 427 bool IsRegSeq = true; 428 unsigned NOps = N->getNumOperands(); 429 for (unsigned i = 0; i < NOps; i++) { 430 // XXX: Why is this here? 431 if (isa<RegisterSDNode>(N->getOperand(i))) { 432 IsRegSeq = false; 433 break; 434 } 435 unsigned Sub = AMDGPURegisterInfo::getSubRegFromChannel(i); 436 RegSeqArgs[1 + (2 * i)] = N->getOperand(i); 437 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32); 438 } 439 if (NOps != NumVectorElts) { 440 // Fill in the missing undef elements if this was a scalar_to_vector. 441 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts); 442 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 443 DL, EltVT); 444 for (unsigned i = NOps; i < NumVectorElts; ++i) { 445 unsigned Sub = AMDGPURegisterInfo::getSubRegFromChannel(i); 446 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0); 447 RegSeqArgs[1 + (2 * i) + 1] = 448 CurDAG->getTargetConstant(Sub, DL, MVT::i32); 449 } 450 } 451 452 if (!IsRegSeq) 453 SelectCode(N); 454 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs); 455 } 456 457 void AMDGPUDAGToDAGISel::Select(SDNode *N) { 458 unsigned int Opc = N->getOpcode(); 459 if (N->isMachineOpcode()) { 460 N->setNodeId(-1); 461 return; // Already selected. 462 } 463 464 if (isa<AtomicSDNode>(N) || 465 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC || 466 Opc == AMDGPUISD::ATOMIC_LOAD_FADD || 467 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN || 468 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) 469 N = glueCopyToM0(N); 470 471 switch (Opc) { 472 default: 473 break; 474 // We are selecting i64 ADD here instead of custom lower it during 475 // DAG legalization, so we can fold some i64 ADDs used for address 476 // calculation into the LOAD and STORE instructions. 477 case ISD::ADDC: 478 case ISD::ADDE: 479 case ISD::SUBC: 480 case ISD::SUBE: { 481 if (N->getValueType(0) != MVT::i64) 482 break; 483 484 SelectADD_SUB_I64(N); 485 return; 486 } 487 case ISD::UADDO: 488 case ISD::USUBO: { 489 SelectUADDO_USUBO(N); 490 return; 491 } 492 case AMDGPUISD::FMUL_W_CHAIN: { 493 SelectFMUL_W_CHAIN(N); 494 return; 495 } 496 case AMDGPUISD::FMA_W_CHAIN: { 497 SelectFMA_W_CHAIN(N); 498 return; 499 } 500 501 case ISD::SCALAR_TO_VECTOR: 502 case ISD::BUILD_VECTOR: { 503 EVT VT = N->getValueType(0); 504 unsigned NumVectorElts = VT.getVectorNumElements(); 505 if (VT.getScalarSizeInBits() == 16) { 506 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) { 507 uint32_t LHSVal, RHSVal; 508 if (getConstantValue(N->getOperand(0), LHSVal) && 509 getConstantValue(N->getOperand(1), RHSVal)) { 510 uint32_t K = LHSVal | (RHSVal << 16); 511 CurDAG->SelectNodeTo(N, AMDGPU::S_MOV_B32, VT, 512 CurDAG->getTargetConstant(K, SDLoc(N), MVT::i32)); 513 return; 514 } 515 } 516 517 break; 518 } 519 520 assert(VT.getVectorElementType().bitsEq(MVT::i32)); 521 unsigned RegClassID = selectSGPRVectorRegClassID(NumVectorElts); 522 SelectBuildVector(N, RegClassID); 523 return; 524 } 525 case ISD::BUILD_PAIR: { 526 SDValue RC, SubReg0, SubReg1; 527 SDLoc DL(N); 528 if (N->getValueType(0) == MVT::i128) { 529 RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, DL, MVT::i32); 530 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32); 531 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32); 532 } else if (N->getValueType(0) == MVT::i64) { 533 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32); 534 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 535 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 536 } else { 537 llvm_unreachable("Unhandled value type for BUILD_PAIR"); 538 } 539 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0, 540 N->getOperand(1), SubReg1 }; 541 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 542 N->getValueType(0), Ops)); 543 return; 544 } 545 546 case ISD::Constant: 547 case ISD::ConstantFP: { 548 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N)) 549 break; 550 551 uint64_t Imm; 552 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N)) 553 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue(); 554 else { 555 ConstantSDNode *C = cast<ConstantSDNode>(N); 556 Imm = C->getZExtValue(); 557 } 558 559 SDLoc DL(N); 560 SDNode *Lo = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 561 CurDAG->getConstant(Imm & 0xFFFFFFFF, DL, 562 MVT::i32)); 563 SDNode *Hi = CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 564 CurDAG->getConstant(Imm >> 32, DL, MVT::i32)); 565 const SDValue Ops[] = { 566 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 567 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32), 568 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32) 569 }; 570 571 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, 572 N->getValueType(0), Ops)); 573 return; 574 } 575 case ISD::LOAD: 576 case ISD::STORE: 577 case ISD::ATOMIC_LOAD: 578 case ISD::ATOMIC_STORE: { 579 N = glueCopyToM0(N); 580 break; 581 } 582 583 case AMDGPUISD::BFE_I32: 584 case AMDGPUISD::BFE_U32: { 585 // There is a scalar version available, but unlike the vector version which 586 // has a separate operand for the offset and width, the scalar version packs 587 // the width and offset into a single operand. Try to move to the scalar 588 // version if the offsets are constant, so that we can try to keep extended 589 // loads of kernel arguments in SGPRs. 590 591 // TODO: Technically we could try to pattern match scalar bitshifts of 592 // dynamic values, but it's probably not useful. 593 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1)); 594 if (!Offset) 595 break; 596 597 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2)); 598 if (!Width) 599 break; 600 601 bool Signed = Opc == AMDGPUISD::BFE_I32; 602 603 uint32_t OffsetVal = Offset->getZExtValue(); 604 uint32_t WidthVal = Width->getZExtValue(); 605 606 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32, 607 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal)); 608 return; 609 } 610 case AMDGPUISD::DIV_SCALE: { 611 SelectDIV_SCALE(N); 612 return; 613 } 614 case AMDGPUISD::MAD_I64_I32: 615 case AMDGPUISD::MAD_U64_U32: { 616 SelectMAD_64_32(N); 617 return; 618 } 619 case ISD::CopyToReg: { 620 const SITargetLowering& Lowering = 621 *static_cast<const SITargetLowering*>(getTargetLowering()); 622 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG); 623 break; 624 } 625 case ISD::AND: 626 case ISD::SRL: 627 case ISD::SRA: 628 case ISD::SIGN_EXTEND_INREG: 629 if (N->getValueType(0) != MVT::i32) 630 break; 631 632 SelectS_BFE(N); 633 return; 634 case ISD::BRCOND: 635 SelectBRCOND(N); 636 return; 637 case ISD::FMAD: 638 case ISD::FMA: 639 SelectFMAD_FMA(N); 640 return; 641 case AMDGPUISD::ATOMIC_CMP_SWAP: 642 SelectATOMIC_CMP_SWAP(N); 643 return; 644 } 645 646 SelectCode(N); 647 } 648 649 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { 650 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock(); 651 const Instruction *Term = BB->getTerminator(); 652 return Term->getMetadata("amdgpu.uniform") || 653 Term->getMetadata("structurizecfg.uniform"); 654 } 655 656 StringRef AMDGPUDAGToDAGISel::getPassName() const { 657 return "AMDGPU DAG->DAG Pattern Instruction Selection"; 658 } 659 660 //===----------------------------------------------------------------------===// 661 // Complex Patterns 662 //===----------------------------------------------------------------------===// 663 664 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 665 SDValue &Offset) { 666 return false; 667 } 668 669 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 670 SDValue &Offset) { 671 ConstantSDNode *C; 672 SDLoc DL(Addr); 673 674 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 675 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 676 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 677 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 678 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 679 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 680 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 681 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 682 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 683 Base = Addr.getOperand(0); 684 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 685 } else { 686 Base = Addr; 687 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 688 } 689 690 return true; 691 } 692 693 // FIXME: Should only handle addcarry/subcarry 694 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { 695 SDLoc DL(N); 696 SDValue LHS = N->getOperand(0); 697 SDValue RHS = N->getOperand(1); 698 699 unsigned Opcode = N->getOpcode(); 700 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); 701 bool ProduceCarry = 702 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; 703 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE; 704 705 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); 706 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); 707 708 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 709 DL, MVT::i32, LHS, Sub0); 710 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 711 DL, MVT::i32, LHS, Sub1); 712 713 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 714 DL, MVT::i32, RHS, Sub0); 715 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, 716 DL, MVT::i32, RHS, Sub1); 717 718 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); 719 720 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; 721 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; 722 723 SDNode *AddLo; 724 if (!ConsumeCarry) { 725 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; 726 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); 727 } else { 728 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; 729 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); 730 } 731 SDValue AddHiArgs[] = { 732 SDValue(Hi0, 0), 733 SDValue(Hi1, 0), 734 SDValue(AddLo, 1) 735 }; 736 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); 737 738 SDValue RegSequenceArgs[] = { 739 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), 740 SDValue(AddLo,0), 741 Sub0, 742 SDValue(AddHi,0), 743 Sub1, 744 }; 745 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, 746 MVT::i64, RegSequenceArgs); 747 748 if (ProduceCarry) { 749 // Replace the carry-use 750 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1)); 751 } 752 753 // Replace the remaining uses. 754 ReplaceNode(N, RegSequence); 755 } 756 757 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) { 758 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned 759 // carry out despite the _i32 name. These were renamed in VI to _U32. 760 // FIXME: We should probably rename the opcodes here. 761 unsigned Opc = N->getOpcode() == ISD::UADDO ? 762 AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; 763 764 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), 765 { N->getOperand(0), N->getOperand(1) }); 766 } 767 768 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { 769 SDLoc SL(N); 770 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod 771 SDValue Ops[10]; 772 773 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); 774 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 775 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); 776 Ops[8] = N->getOperand(0); 777 Ops[9] = N->getOperand(4); 778 779 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops); 780 } 781 782 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { 783 SDLoc SL(N); 784 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod 785 SDValue Ops[8]; 786 787 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); 788 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); 789 Ops[6] = N->getOperand(0); 790 Ops[7] = N->getOperand(3); 791 792 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); 793 } 794 795 // We need to handle this here because tablegen doesn't support matching 796 // instructions with multiple outputs. 797 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) { 798 SDLoc SL(N); 799 EVT VT = N->getValueType(0); 800 801 assert(VT == MVT::f32 || VT == MVT::f64); 802 803 unsigned Opc 804 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32; 805 806 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) }; 807 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 808 } 809 810 // We need to handle this here because tablegen doesn't support matching 811 // instructions with multiple outputs. 812 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) { 813 SDLoc SL(N); 814 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32; 815 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32; 816 817 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1); 818 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2), 819 Clamp }; 820 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); 821 } 822 823 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset, 824 unsigned OffsetBits) const { 825 if ((OffsetBits == 16 && !isUInt<16>(Offset)) || 826 (OffsetBits == 8 && !isUInt<8>(Offset))) 827 return false; 828 829 if (Subtarget->getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS || 830 Subtarget->unsafeDSOffsetFoldingEnabled()) 831 return true; 832 833 // On Southern Islands instruction with a negative base value and an offset 834 // don't seem to work. 835 return CurDAG->SignBitIsZero(Base); 836 } 837 838 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base, 839 SDValue &Offset) const { 840 SDLoc DL(Addr); 841 if (CurDAG->isBaseWithConstantOffset(Addr)) { 842 SDValue N0 = Addr.getOperand(0); 843 SDValue N1 = Addr.getOperand(1); 844 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 845 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) { 846 // (add n0, c0) 847 Base = N0; 848 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 849 return true; 850 } 851 } else if (Addr.getOpcode() == ISD::SUB) { 852 // sub C, x -> add (sub 0, x), C 853 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 854 int64_t ByteOffset = C->getSExtValue(); 855 if (isUInt<16>(ByteOffset)) { 856 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 857 858 // XXX - This is kind of hacky. Create a dummy sub node so we can check 859 // the known bits in isDSOffsetLegal. We need to emit the selected node 860 // here, so this is thrown away. 861 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 862 Zero, Addr.getOperand(1)); 863 864 if (isDSOffsetLegal(Sub, ByteOffset, 16)) { 865 // FIXME: Select to VOP3 version for with-carry. 866 unsigned SubOp = Subtarget->hasAddNoCarry() ? 867 AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32; 868 869 MachineSDNode *MachineSub 870 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, 871 Zero, Addr.getOperand(1)); 872 873 Base = SDValue(MachineSub, 0); 874 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16); 875 return true; 876 } 877 } 878 } 879 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 880 // If we have a constant address, prefer to put the constant into the 881 // offset. This can save moves to load the constant address since multiple 882 // operations can share the zero base address register, and enables merging 883 // into read2 / write2 instructions. 884 885 SDLoc DL(Addr); 886 887 if (isUInt<16>(CAddr->getZExtValue())) { 888 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 889 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 890 DL, MVT::i32, Zero); 891 Base = SDValue(MovZero, 0); 892 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 893 return true; 894 } 895 } 896 897 // default case 898 Base = Addr; 899 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16); 900 return true; 901 } 902 903 // TODO: If offset is too big, put low 16-bit into offset. 904 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, 905 SDValue &Offset0, 906 SDValue &Offset1) const { 907 SDLoc DL(Addr); 908 909 if (CurDAG->isBaseWithConstantOffset(Addr)) { 910 SDValue N0 = Addr.getOperand(0); 911 SDValue N1 = Addr.getOperand(1); 912 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 913 unsigned DWordOffset0 = C1->getZExtValue() / 4; 914 unsigned DWordOffset1 = DWordOffset0 + 1; 915 // (add n0, c0) 916 if (isDSOffsetLegal(N0, DWordOffset1, 8)) { 917 Base = N0; 918 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 919 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 920 return true; 921 } 922 } else if (Addr.getOpcode() == ISD::SUB) { 923 // sub C, x -> add (sub 0, x), C 924 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) { 925 unsigned DWordOffset0 = C->getZExtValue() / 4; 926 unsigned DWordOffset1 = DWordOffset0 + 1; 927 928 if (isUInt<8>(DWordOffset0)) { 929 SDLoc DL(Addr); 930 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 931 932 // XXX - This is kind of hacky. Create a dummy sub node so we can check 933 // the known bits in isDSOffsetLegal. We need to emit the selected node 934 // here, so this is thrown away. 935 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32, 936 Zero, Addr.getOperand(1)); 937 938 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) { 939 unsigned SubOp = Subtarget->hasAddNoCarry() ? 940 AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32; 941 942 MachineSDNode *MachineSub 943 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, 944 Zero, Addr.getOperand(1)); 945 946 Base = SDValue(MachineSub, 0); 947 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 948 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 949 return true; 950 } 951 } 952 } 953 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 954 unsigned DWordOffset0 = CAddr->getZExtValue() / 4; 955 unsigned DWordOffset1 = DWordOffset0 + 1; 956 assert(4 * DWordOffset0 == CAddr->getZExtValue()); 957 958 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) { 959 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32); 960 MachineSDNode *MovZero 961 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 962 DL, MVT::i32, Zero); 963 Base = SDValue(MovZero, 0); 964 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8); 965 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8); 966 return true; 967 } 968 } 969 970 // default case 971 972 // FIXME: This is broken on SI where we still need to check if the base 973 // pointer is positive here. 974 Base = Addr; 975 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); 976 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); 977 return true; 978 } 979 980 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, 981 SDValue &VAddr, SDValue &SOffset, 982 SDValue &Offset, SDValue &Offen, 983 SDValue &Idxen, SDValue &Addr64, 984 SDValue &GLC, SDValue &SLC, 985 SDValue &TFE) const { 986 // Subtarget prefers to use flat instruction 987 if (Subtarget->useFlatForGlobal()) 988 return false; 989 990 SDLoc DL(Addr); 991 992 if (!GLC.getNode()) 993 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 994 if (!SLC.getNode()) 995 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1); 996 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1); 997 998 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1); 999 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1); 1000 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1); 1001 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1002 1003 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1004 SDValue N0 = Addr.getOperand(0); 1005 SDValue N1 = Addr.getOperand(1); 1006 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1007 1008 if (N0.getOpcode() == ISD::ADD) { 1009 // (add (add N2, N3), C1) -> addr64 1010 SDValue N2 = N0.getOperand(0); 1011 SDValue N3 = N0.getOperand(1); 1012 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1013 Ptr = N2; 1014 VAddr = N3; 1015 } else { 1016 // (add N0, C1) -> offset 1017 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1018 Ptr = N0; 1019 } 1020 1021 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) { 1022 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1023 return true; 1024 } 1025 1026 if (isUInt<32>(C1->getZExtValue())) { 1027 // Illegal offset, store it in soffset. 1028 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1029 SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 1030 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)), 1031 0); 1032 return true; 1033 } 1034 } 1035 1036 if (Addr.getOpcode() == ISD::ADD) { 1037 // (add N0, N1) -> addr64 1038 SDValue N0 = Addr.getOperand(0); 1039 SDValue N1 = Addr.getOperand(1); 1040 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1); 1041 Ptr = N0; 1042 VAddr = N1; 1043 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1044 return true; 1045 } 1046 1047 // default case -> offset 1048 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); 1049 Ptr = Addr; 1050 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1051 1052 return true; 1053 } 1054 1055 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1056 SDValue &VAddr, SDValue &SOffset, 1057 SDValue &Offset, SDValue &GLC, 1058 SDValue &SLC, SDValue &TFE) const { 1059 SDValue Ptr, Offen, Idxen, Addr64; 1060 1061 // addr64 bit was removed for volcanic islands. 1062 if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) 1063 return false; 1064 1065 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1066 GLC, SLC, TFE)) 1067 return false; 1068 1069 ConstantSDNode *C = cast<ConstantSDNode>(Addr64); 1070 if (C->getSExtValue()) { 1071 SDLoc DL(Addr); 1072 1073 const SITargetLowering& Lowering = 1074 *static_cast<const SITargetLowering*>(getTargetLowering()); 1075 1076 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0); 1077 return true; 1078 } 1079 1080 return false; 1081 } 1082 1083 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, 1084 SDValue &VAddr, SDValue &SOffset, 1085 SDValue &Offset, 1086 SDValue &SLC) const { 1087 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1); 1088 SDValue GLC, TFE; 1089 1090 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE); 1091 } 1092 1093 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) { 1094 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>(); 1095 return PSV && PSV->isStack(); 1096 } 1097 1098 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { 1099 const MachineFunction &MF = CurDAG->getMachineFunction(); 1100 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1101 1102 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) { 1103 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(), 1104 FI->getValueType(0)); 1105 1106 // If we can resolve this to a frame index access, this is relative to the 1107 // frame pointer SGPR. 1108 return std::make_pair(TFI, CurDAG->getRegister(Info->getFrameOffsetReg(), 1109 MVT::i32)); 1110 } 1111 1112 // If we don't know this private access is a local stack object, it needs to 1113 // be relative to the entry point's scratch wave offset register. 1114 return std::make_pair(N, CurDAG->getRegister(Info->getScratchWaveOffsetReg(), 1115 MVT::i32)); 1116 } 1117 1118 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent, 1119 SDValue Addr, SDValue &Rsrc, 1120 SDValue &VAddr, SDValue &SOffset, 1121 SDValue &ImmOffset) const { 1122 1123 SDLoc DL(Addr); 1124 MachineFunction &MF = CurDAG->getMachineFunction(); 1125 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1126 1127 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1128 1129 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) { 1130 unsigned Imm = CAddr->getZExtValue(); 1131 1132 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32); 1133 MachineSDNode *MovHighBits = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 1134 DL, MVT::i32, HighBits); 1135 VAddr = SDValue(MovHighBits, 0); 1136 1137 // In a call sequence, stores to the argument stack area are relative to the 1138 // stack pointer. 1139 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1140 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ? 1141 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg(); 1142 1143 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32); 1144 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16); 1145 return true; 1146 } 1147 1148 if (CurDAG->isBaseWithConstantOffset(Addr)) { 1149 // (add n0, c1) 1150 1151 SDValue N0 = Addr.getOperand(0); 1152 SDValue N1 = Addr.getOperand(1); 1153 1154 // Offsets in vaddr must be positive if range checking is enabled. 1155 // 1156 // The total computation of vaddr + soffset + offset must not overflow. If 1157 // vaddr is negative, even if offset is 0 the sgpr offset add will end up 1158 // overflowing. 1159 // 1160 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would 1161 // always perform a range check. If a negative vaddr base index was used, 1162 // this would fail the range check. The overall address computation would 1163 // compute a valid address, but this doesn't happen due to the range 1164 // check. For out-of-bounds MUBUF loads, a 0 is returned. 1165 // 1166 // Therefore it should be safe to fold any VGPR offset on gfx9 into the 1167 // MUBUF vaddr, but not on older subtargets which can only do this if the 1168 // sign bit is known 0. 1169 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1170 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) && 1171 (!Subtarget->privateMemoryResourceIsRangeChecked() || 1172 CurDAG->SignBitIsZero(N0))) { 1173 std::tie(VAddr, SOffset) = foldFrameIndex(N0); 1174 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); 1175 return true; 1176 } 1177 } 1178 1179 // (node) 1180 std::tie(VAddr, SOffset) = foldFrameIndex(Addr); 1181 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1182 return true; 1183 } 1184 1185 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent, 1186 SDValue Addr, 1187 SDValue &SRsrc, 1188 SDValue &SOffset, 1189 SDValue &Offset) const { 1190 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr); 1191 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue())) 1192 return false; 1193 1194 SDLoc DL(Addr); 1195 MachineFunction &MF = CurDAG->getMachineFunction(); 1196 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1197 1198 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32); 1199 1200 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo(); 1201 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ? 1202 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg(); 1203 1204 // FIXME: Get from MachinePointerInfo? We should only be using the frame 1205 // offset if we know this is in a call sequence. 1206 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32); 1207 1208 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16); 1209 return true; 1210 } 1211 1212 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1213 SDValue &SOffset, SDValue &Offset, 1214 SDValue &GLC, SDValue &SLC, 1215 SDValue &TFE) const { 1216 SDValue Ptr, VAddr, Offen, Idxen, Addr64; 1217 const SIInstrInfo *TII = 1218 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo()); 1219 1220 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64, 1221 GLC, SLC, TFE)) 1222 return false; 1223 1224 if (!cast<ConstantSDNode>(Offen)->getSExtValue() && 1225 !cast<ConstantSDNode>(Idxen)->getSExtValue() && 1226 !cast<ConstantSDNode>(Addr64)->getSExtValue()) { 1227 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() | 1228 APInt::getAllOnesValue(32).getZExtValue(); // Size 1229 SDLoc DL(Addr); 1230 1231 const SITargetLowering& Lowering = 1232 *static_cast<const SITargetLowering*>(getTargetLowering()); 1233 1234 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0); 1235 return true; 1236 } 1237 return false; 1238 } 1239 1240 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1241 SDValue &Soffset, SDValue &Offset 1242 ) const { 1243 SDValue GLC, SLC, TFE; 1244 1245 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE); 1246 } 1247 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, 1248 SDValue &Soffset, SDValue &Offset, 1249 SDValue &SLC) const { 1250 SDValue GLC, TFE; 1251 1252 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE); 1253 } 1254 1255 bool AMDGPUDAGToDAGISel::SelectMUBUFConstant(SDValue Constant, 1256 SDValue &SOffset, 1257 SDValue &ImmOffset) const { 1258 SDLoc DL(Constant); 1259 const uint32_t Align = 4; 1260 const uint32_t MaxImm = alignDown(4095, Align); 1261 uint32_t Imm = cast<ConstantSDNode>(Constant)->getZExtValue(); 1262 uint32_t Overflow = 0; 1263 1264 if (Imm > MaxImm) { 1265 if (Imm <= MaxImm + 64) { 1266 // Use an SOffset inline constant for 4..64 1267 Overflow = Imm - MaxImm; 1268 Imm = MaxImm; 1269 } else { 1270 // Try to keep the same value in SOffset for adjacent loads, so that 1271 // the corresponding register contents can be re-used. 1272 // 1273 // Load values with all low-bits (except for alignment bits) set into 1274 // SOffset, so that a larger range of values can be covered using 1275 // s_movk_i32. 1276 // 1277 // Atomic operations fail to work correctly when individual address 1278 // components are unaligned, even if their sum is aligned. 1279 uint32_t High = (Imm + Align) & ~4095; 1280 uint32_t Low = (Imm + Align) & 4095; 1281 Imm = Low; 1282 Overflow = High - Align; 1283 } 1284 } 1285 1286 // There is a hardware bug in SI and CI which prevents address clamping in 1287 // MUBUF instructions from working correctly with SOffsets. The immediate 1288 // offset is unaffected. 1289 if (Overflow > 0 && 1290 Subtarget->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) 1291 return false; 1292 1293 ImmOffset = CurDAG->getTargetConstant(Imm, DL, MVT::i16); 1294 1295 if (Overflow <= 64) 1296 SOffset = CurDAG->getTargetConstant(Overflow, DL, MVT::i32); 1297 else 1298 SOffset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, 1299 CurDAG->getTargetConstant(Overflow, DL, MVT::i32)), 1300 0); 1301 1302 return true; 1303 } 1304 1305 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicOffset(SDValue Offset, 1306 SDValue &SOffset, 1307 SDValue &ImmOffset) const { 1308 SDLoc DL(Offset); 1309 1310 if (!isa<ConstantSDNode>(Offset)) 1311 return false; 1312 1313 return SelectMUBUFConstant(Offset, SOffset, ImmOffset); 1314 } 1315 1316 bool AMDGPUDAGToDAGISel::SelectMUBUFIntrinsicVOffset(SDValue Offset, 1317 SDValue &SOffset, 1318 SDValue &ImmOffset, 1319 SDValue &VOffset) const { 1320 SDLoc DL(Offset); 1321 1322 // Don't generate an unnecessary voffset for constant offsets. 1323 if (isa<ConstantSDNode>(Offset)) { 1324 SDValue Tmp1, Tmp2; 1325 1326 // When necessary, use a voffset in <= CI anyway to work around a hardware 1327 // bug. 1328 if (Subtarget->getGeneration() > AMDGPUSubtarget::SEA_ISLANDS || 1329 SelectMUBUFConstant(Offset, Tmp1, Tmp2)) 1330 return false; 1331 } 1332 1333 if (CurDAG->isBaseWithConstantOffset(Offset)) { 1334 SDValue N0 = Offset.getOperand(0); 1335 SDValue N1 = Offset.getOperand(1); 1336 if (cast<ConstantSDNode>(N1)->getSExtValue() >= 0 && 1337 SelectMUBUFConstant(N1, SOffset, ImmOffset)) { 1338 VOffset = N0; 1339 return true; 1340 } 1341 } 1342 1343 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1344 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); 1345 VOffset = Offset; 1346 1347 return true; 1348 } 1349 1350 template <bool IsSigned> 1351 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDValue Addr, 1352 SDValue &VAddr, 1353 SDValue &Offset, 1354 SDValue &SLC) const { 1355 int64_t OffsetVal = 0; 1356 1357 if (Subtarget->hasFlatInstOffsets() && 1358 CurDAG->isBaseWithConstantOffset(Addr)) { 1359 SDValue N0 = Addr.getOperand(0); 1360 SDValue N1 = Addr.getOperand(1); 1361 int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue(); 1362 1363 if ((IsSigned && isInt<13>(COffsetVal)) || 1364 (!IsSigned && isUInt<12>(COffsetVal))) { 1365 Addr = N0; 1366 OffsetVal = COffsetVal; 1367 } 1368 } 1369 1370 VAddr = Addr; 1371 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i16); 1372 SLC = CurDAG->getTargetConstant(0, SDLoc(), MVT::i1); 1373 1374 return true; 1375 } 1376 1377 bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDValue Addr, 1378 SDValue &VAddr, 1379 SDValue &Offset, 1380 SDValue &SLC) const { 1381 return SelectFlatOffset<false>(Addr, VAddr, Offset, SLC); 1382 } 1383 1384 bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDValue Addr, 1385 SDValue &VAddr, 1386 SDValue &Offset, 1387 SDValue &SLC) const { 1388 return SelectFlatOffset<true>(Addr, VAddr, Offset, SLC); 1389 } 1390 1391 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, 1392 SDValue &Offset, bool &Imm) const { 1393 1394 // FIXME: Handle non-constant offsets. 1395 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode); 1396 if (!C) 1397 return false; 1398 1399 SDLoc SL(ByteOffsetNode); 1400 GCNSubtarget::Generation Gen = Subtarget->getGeneration(); 1401 int64_t ByteOffset = C->getSExtValue(); 1402 int64_t EncodedOffset = AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset); 1403 1404 if (AMDGPU::isLegalSMRDImmOffset(*Subtarget, ByteOffset)) { 1405 Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32); 1406 Imm = true; 1407 return true; 1408 } 1409 1410 if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset)) 1411 return false; 1412 1413 if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) { 1414 // 32-bit Immediates are supported on Sea Islands. 1415 Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32); 1416 } else { 1417 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32); 1418 Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, 1419 C32Bit), 0); 1420 } 1421 Imm = false; 1422 return true; 1423 } 1424 1425 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const { 1426 if (Addr.getValueType() != MVT::i32) 1427 return Addr; 1428 1429 // Zero-extend a 32-bit address. 1430 SDLoc SL(Addr); 1431 1432 const MachineFunction &MF = CurDAG->getMachineFunction(); 1433 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); 1434 unsigned AddrHiVal = Info->get32BitAddressHighBits(); 1435 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32); 1436 1437 const SDValue Ops[] = { 1438 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32), 1439 Addr, 1440 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32), 1441 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi), 1442 0), 1443 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32), 1444 }; 1445 1446 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64, 1447 Ops), 0); 1448 } 1449 1450 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, 1451 SDValue &Offset, bool &Imm) const { 1452 SDLoc SL(Addr); 1453 1454 // A 32-bit (address + offset) should not cause unsigned 32-bit integer 1455 // wraparound, because s_load instructions perform the addition in 64 bits. 1456 if ((Addr.getValueType() != MVT::i32 || 1457 Addr->getFlags().hasNoUnsignedWrap()) && 1458 CurDAG->isBaseWithConstantOffset(Addr)) { 1459 SDValue N0 = Addr.getOperand(0); 1460 SDValue N1 = Addr.getOperand(1); 1461 1462 if (SelectSMRDOffset(N1, Offset, Imm)) { 1463 SBase = Expand32BitAddress(N0); 1464 return true; 1465 } 1466 } 1467 SBase = Expand32BitAddress(Addr); 1468 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32); 1469 Imm = true; 1470 return true; 1471 } 1472 1473 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase, 1474 SDValue &Offset) const { 1475 bool Imm; 1476 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm; 1477 } 1478 1479 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase, 1480 SDValue &Offset) const { 1481 1482 if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS) 1483 return false; 1484 1485 bool Imm; 1486 if (!SelectSMRD(Addr, SBase, Offset, Imm)) 1487 return false; 1488 1489 return !Imm && isa<ConstantSDNode>(Offset); 1490 } 1491 1492 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase, 1493 SDValue &Offset) const { 1494 bool Imm; 1495 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm && 1496 !isa<ConstantSDNode>(Offset); 1497 } 1498 1499 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr, 1500 SDValue &Offset) const { 1501 bool Imm; 1502 return SelectSMRDOffset(Addr, Offset, Imm) && Imm; 1503 } 1504 1505 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr, 1506 SDValue &Offset) const { 1507 if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS) 1508 return false; 1509 1510 bool Imm; 1511 if (!SelectSMRDOffset(Addr, Offset, Imm)) 1512 return false; 1513 1514 return !Imm && isa<ConstantSDNode>(Offset); 1515 } 1516 1517 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index, 1518 SDValue &Base, 1519 SDValue &Offset) const { 1520 SDLoc DL(Index); 1521 1522 if (CurDAG->isBaseWithConstantOffset(Index)) { 1523 SDValue N0 = Index.getOperand(0); 1524 SDValue N1 = Index.getOperand(1); 1525 ConstantSDNode *C1 = cast<ConstantSDNode>(N1); 1526 1527 // (add n0, c0) 1528 Base = N0; 1529 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32); 1530 return true; 1531 } 1532 1533 if (isa<ConstantSDNode>(Index)) 1534 return false; 1535 1536 Base = Index; 1537 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 1538 return true; 1539 } 1540 1541 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL, 1542 SDValue Val, uint32_t Offset, 1543 uint32_t Width) { 1544 // Transformation function, pack the offset and width of a BFE into 1545 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second 1546 // source, bits [5:0] contain the offset and bits [22:16] the width. 1547 uint32_t PackedVal = Offset | (Width << 16); 1548 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32); 1549 1550 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst); 1551 } 1552 1553 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) { 1554 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c) 1555 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c) 1556 // Predicate: 0 < b <= c < 32 1557 1558 const SDValue &Shl = N->getOperand(0); 1559 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1)); 1560 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1561 1562 if (B && C) { 1563 uint32_t BVal = B->getZExtValue(); 1564 uint32_t CVal = C->getZExtValue(); 1565 1566 if (0 < BVal && BVal <= CVal && CVal < 32) { 1567 bool Signed = N->getOpcode() == ISD::SRA; 1568 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32; 1569 1570 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal, 1571 32 - CVal)); 1572 return; 1573 } 1574 } 1575 SelectCode(N); 1576 } 1577 1578 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { 1579 switch (N->getOpcode()) { 1580 case ISD::AND: 1581 if (N->getOperand(0).getOpcode() == ISD::SRL) { 1582 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)" 1583 // Predicate: isMask(mask) 1584 const SDValue &Srl = N->getOperand(0); 1585 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1)); 1586 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1587 1588 if (Shift && Mask) { 1589 uint32_t ShiftVal = Shift->getZExtValue(); 1590 uint32_t MaskVal = Mask->getZExtValue(); 1591 1592 if (isMask_32(MaskVal)) { 1593 uint32_t WidthVal = countPopulation(MaskVal); 1594 1595 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 1596 Srl.getOperand(0), ShiftVal, WidthVal)); 1597 return; 1598 } 1599 } 1600 } 1601 break; 1602 case ISD::SRL: 1603 if (N->getOperand(0).getOpcode() == ISD::AND) { 1604 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)" 1605 // Predicate: isMask(mask >> b) 1606 const SDValue &And = N->getOperand(0); 1607 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1)); 1608 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1)); 1609 1610 if (Shift && Mask) { 1611 uint32_t ShiftVal = Shift->getZExtValue(); 1612 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal; 1613 1614 if (isMask_32(MaskVal)) { 1615 uint32_t WidthVal = countPopulation(MaskVal); 1616 1617 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N), 1618 And.getOperand(0), ShiftVal, WidthVal)); 1619 return; 1620 } 1621 } 1622 } else if (N->getOperand(0).getOpcode() == ISD::SHL) { 1623 SelectS_BFEFromShifts(N); 1624 return; 1625 } 1626 break; 1627 case ISD::SRA: 1628 if (N->getOperand(0).getOpcode() == ISD::SHL) { 1629 SelectS_BFEFromShifts(N); 1630 return; 1631 } 1632 break; 1633 1634 case ISD::SIGN_EXTEND_INREG: { 1635 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8 1636 SDValue Src = N->getOperand(0); 1637 if (Src.getOpcode() != ISD::SRL) 1638 break; 1639 1640 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1)); 1641 if (!Amt) 1642 break; 1643 1644 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits(); 1645 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0), 1646 Amt->getZExtValue(), Width)); 1647 return; 1648 } 1649 } 1650 1651 SelectCode(N); 1652 } 1653 1654 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { 1655 assert(N->getOpcode() == ISD::BRCOND); 1656 if (!N->hasOneUse()) 1657 return false; 1658 1659 SDValue Cond = N->getOperand(1); 1660 if (Cond.getOpcode() == ISD::CopyToReg) 1661 Cond = Cond.getOperand(2); 1662 1663 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) 1664 return false; 1665 1666 MVT VT = Cond.getOperand(0).getSimpleValueType(); 1667 if (VT == MVT::i32) 1668 return true; 1669 1670 if (VT == MVT::i64) { 1671 auto ST = static_cast<const GCNSubtarget *>(Subtarget); 1672 1673 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); 1674 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); 1675 } 1676 1677 return false; 1678 } 1679 1680 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { 1681 SDValue Cond = N->getOperand(1); 1682 1683 if (Cond.isUndef()) { 1684 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, 1685 N->getOperand(2), N->getOperand(0)); 1686 return; 1687 } 1688 1689 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N); 1690 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ; 1691 unsigned CondReg = UseSCCBr ? AMDGPU::SCC : AMDGPU::VCC; 1692 SDLoc SL(N); 1693 1694 if (!UseSCCBr) { 1695 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not 1696 // analyzed what generates the vcc value, so we do not know whether vcc 1697 // bits for disabled lanes are 0. Thus we need to mask out bits for 1698 // disabled lanes. 1699 // 1700 // For the case that we select S_CBRANCH_SCC1 and it gets 1701 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls 1702 // SIInstrInfo::moveToVALU which inserts the S_AND). 1703 // 1704 // We could add an analysis of what generates the vcc value here and omit 1705 // the S_AND when is unnecessary. But it would be better to add a separate 1706 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it 1707 // catches both cases. 1708 Cond = SDValue(CurDAG->getMachineNode(AMDGPU::S_AND_B64, SL, MVT::i1, 1709 CurDAG->getRegister(AMDGPU::EXEC, MVT::i1), 1710 Cond), 1711 0); 1712 } 1713 1714 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond); 1715 CurDAG->SelectNodeTo(N, BrOp, MVT::Other, 1716 N->getOperand(2), // Basic Block 1717 VCC.getValue(0)); 1718 } 1719 1720 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { 1721 MVT VT = N->getSimpleValueType(0); 1722 bool IsFMA = N->getOpcode() == ISD::FMA; 1723 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() && 1724 !Subtarget->hasFmaMixInsts()) || 1725 ((IsFMA && Subtarget->hasMadMixInsts()) || 1726 (!IsFMA && Subtarget->hasFmaMixInsts()))) { 1727 SelectCode(N); 1728 return; 1729 } 1730 1731 SDValue Src0 = N->getOperand(0); 1732 SDValue Src1 = N->getOperand(1); 1733 SDValue Src2 = N->getOperand(2); 1734 unsigned Src0Mods, Src1Mods, Src2Mods; 1735 1736 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand 1737 // using the conversion from f16. 1738 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods); 1739 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); 1740 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); 1741 1742 assert((IsFMA || !Subtarget->hasFP32Denormals()) && 1743 "fmad selected with denormals enabled"); 1744 // TODO: We can select this with f32 denormals enabled if all the sources are 1745 // converted from f16 (in which case fmad isn't legal). 1746 1747 if (Sel0 || Sel1 || Sel2) { 1748 // For dummy operands. 1749 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32); 1750 SDValue Ops[] = { 1751 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0, 1752 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1, 1753 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2, 1754 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1), 1755 Zero, Zero 1756 }; 1757 1758 CurDAG->SelectNodeTo(N, 1759 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32, 1760 MVT::f32, Ops); 1761 } else { 1762 SelectCode(N); 1763 } 1764 } 1765 1766 // This is here because there isn't a way to use the generated sub0_sub1 as the 1767 // subreg index to EXTRACT_SUBREG in tablegen. 1768 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { 1769 MemSDNode *Mem = cast<MemSDNode>(N); 1770 unsigned AS = Mem->getAddressSpace(); 1771 if (AS == AMDGPUASI.FLAT_ADDRESS) { 1772 SelectCode(N); 1773 return; 1774 } 1775 1776 MVT VT = N->getSimpleValueType(0); 1777 bool Is32 = (VT == MVT::i32); 1778 SDLoc SL(N); 1779 1780 MachineSDNode *CmpSwap = nullptr; 1781 if (Subtarget->hasAddr64()) { 1782 SDValue SRsrc, VAddr, SOffset, Offset, SLC; 1783 1784 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) { 1785 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN : 1786 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN; 1787 SDValue CmpVal = Mem->getOperand(2); 1788 1789 // XXX - Do we care about glue operands? 1790 1791 SDValue Ops[] = { 1792 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain() 1793 }; 1794 1795 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 1796 } 1797 } 1798 1799 if (!CmpSwap) { 1800 SDValue SRsrc, SOffset, Offset, SLC; 1801 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) { 1802 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN : 1803 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN; 1804 1805 SDValue CmpVal = Mem->getOperand(2); 1806 SDValue Ops[] = { 1807 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain() 1808 }; 1809 1810 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops); 1811 } 1812 } 1813 1814 if (!CmpSwap) { 1815 SelectCode(N); 1816 return; 1817 } 1818 1819 MachineSDNode::mmo_iterator MMOs = MF->allocateMemRefsArray(1); 1820 *MMOs = Mem->getMemOperand(); 1821 CmpSwap->setMemRefs(MMOs, MMOs + 1); 1822 1823 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; 1824 SDValue Extract 1825 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0)); 1826 1827 ReplaceUses(SDValue(N, 0), Extract); 1828 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1)); 1829 CurDAG->RemoveDeadNode(N); 1830 } 1831 1832 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src, 1833 unsigned &Mods) const { 1834 Mods = 0; 1835 Src = In; 1836 1837 if (Src.getOpcode() == ISD::FNEG) { 1838 Mods |= SISrcMods::NEG; 1839 Src = Src.getOperand(0); 1840 } 1841 1842 if (Src.getOpcode() == ISD::FABS) { 1843 Mods |= SISrcMods::ABS; 1844 Src = Src.getOperand(0); 1845 } 1846 1847 return true; 1848 } 1849 1850 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, 1851 SDValue &SrcMods) const { 1852 unsigned Mods; 1853 if (SelectVOP3ModsImpl(In, Src, Mods)) { 1854 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 1855 return true; 1856 } 1857 1858 return false; 1859 } 1860 1861 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, 1862 SDValue &SrcMods) const { 1863 SelectVOP3Mods(In, Src, SrcMods); 1864 return isNoNanSrc(Src); 1865 } 1866 1867 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const { 1868 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG) 1869 return false; 1870 1871 Src = In; 1872 return true; 1873 } 1874 1875 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, 1876 SDValue &SrcMods, SDValue &Clamp, 1877 SDValue &Omod) const { 1878 SDLoc DL(In); 1879 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1880 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 1881 1882 return SelectVOP3Mods(In, Src, SrcMods); 1883 } 1884 1885 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, 1886 SDValue &SrcMods, 1887 SDValue &Clamp, 1888 SDValue &Omod) const { 1889 Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 1890 return SelectVOP3Mods(In, Src, SrcMods); 1891 } 1892 1893 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, 1894 SDValue &Clamp, SDValue &Omod) const { 1895 Src = In; 1896 1897 SDLoc DL(In); 1898 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1); 1899 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1); 1900 1901 return true; 1902 } 1903 1904 static SDValue stripBitcast(SDValue Val) { 1905 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val; 1906 } 1907 1908 // Figure out if this is really an extract of the high 16-bits of a dword. 1909 static bool isExtractHiElt(SDValue In, SDValue &Out) { 1910 In = stripBitcast(In); 1911 if (In.getOpcode() != ISD::TRUNCATE) 1912 return false; 1913 1914 SDValue Srl = In.getOperand(0); 1915 if (Srl.getOpcode() == ISD::SRL) { 1916 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) { 1917 if (ShiftAmt->getZExtValue() == 16) { 1918 Out = stripBitcast(Srl.getOperand(0)); 1919 return true; 1920 } 1921 } 1922 } 1923 1924 return false; 1925 } 1926 1927 // Look through operations that obscure just looking at the low 16-bits of the 1928 // same register. 1929 static SDValue stripExtractLoElt(SDValue In) { 1930 if (In.getOpcode() == ISD::TRUNCATE) { 1931 SDValue Src = In.getOperand(0); 1932 if (Src.getValueType().getSizeInBits() == 32) 1933 return stripBitcast(Src); 1934 } 1935 1936 return In; 1937 } 1938 1939 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src, 1940 SDValue &SrcMods) const { 1941 unsigned Mods = 0; 1942 Src = In; 1943 1944 if (Src.getOpcode() == ISD::FNEG) { 1945 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI); 1946 Src = Src.getOperand(0); 1947 } 1948 1949 if (Src.getOpcode() == ISD::BUILD_VECTOR) { 1950 unsigned VecMods = Mods; 1951 1952 SDValue Lo = stripBitcast(Src.getOperand(0)); 1953 SDValue Hi = stripBitcast(Src.getOperand(1)); 1954 1955 if (Lo.getOpcode() == ISD::FNEG) { 1956 Lo = stripBitcast(Lo.getOperand(0)); 1957 Mods ^= SISrcMods::NEG; 1958 } 1959 1960 if (Hi.getOpcode() == ISD::FNEG) { 1961 Hi = stripBitcast(Hi.getOperand(0)); 1962 Mods ^= SISrcMods::NEG_HI; 1963 } 1964 1965 if (isExtractHiElt(Lo, Lo)) 1966 Mods |= SISrcMods::OP_SEL_0; 1967 1968 if (isExtractHiElt(Hi, Hi)) 1969 Mods |= SISrcMods::OP_SEL_1; 1970 1971 Lo = stripExtractLoElt(Lo); 1972 Hi = stripExtractLoElt(Hi); 1973 1974 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) { 1975 // Really a scalar input. Just select from the low half of the register to 1976 // avoid packing. 1977 1978 Src = Lo; 1979 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 1980 return true; 1981 } 1982 1983 Mods = VecMods; 1984 } 1985 1986 // Packed instructions do not have abs modifiers. 1987 Mods |= SISrcMods::OP_SEL_1; 1988 1989 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 1990 return true; 1991 } 1992 1993 bool AMDGPUDAGToDAGISel::SelectVOP3PMods0(SDValue In, SDValue &Src, 1994 SDValue &SrcMods, 1995 SDValue &Clamp) const { 1996 SDLoc SL(In); 1997 1998 // FIXME: Handle clamp and op_sel 1999 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32); 2000 2001 return SelectVOP3PMods(In, Src, SrcMods); 2002 } 2003 2004 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src, 2005 SDValue &SrcMods) const { 2006 Src = In; 2007 // FIXME: Handle op_sel 2008 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); 2009 return true; 2010 } 2011 2012 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel0(SDValue In, SDValue &Src, 2013 SDValue &SrcMods, 2014 SDValue &Clamp) const { 2015 SDLoc SL(In); 2016 2017 // FIXME: Handle clamp 2018 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32); 2019 2020 return SelectVOP3OpSel(In, Src, SrcMods); 2021 } 2022 2023 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src, 2024 SDValue &SrcMods) const { 2025 // FIXME: Handle op_sel 2026 return SelectVOP3Mods(In, Src, SrcMods); 2027 } 2028 2029 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src, 2030 SDValue &SrcMods, 2031 SDValue &Clamp) const { 2032 SDLoc SL(In); 2033 2034 // FIXME: Handle clamp 2035 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32); 2036 2037 return SelectVOP3OpSelMods(In, Src, SrcMods); 2038 } 2039 2040 // The return value is not whether the match is possible (which it always is), 2041 // but whether or not it a conversion is really used. 2042 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, 2043 unsigned &Mods) const { 2044 Mods = 0; 2045 SelectVOP3ModsImpl(In, Src, Mods); 2046 2047 if (Src.getOpcode() == ISD::FP_EXTEND) { 2048 Src = Src.getOperand(0); 2049 assert(Src.getValueType() == MVT::f16); 2050 Src = stripBitcast(Src); 2051 2052 // Be careful about folding modifiers if we already have an abs. fneg is 2053 // applied last, so we don't want to apply an earlier fneg. 2054 if ((Mods & SISrcMods::ABS) == 0) { 2055 unsigned ModsTmp; 2056 SelectVOP3ModsImpl(Src, Src, ModsTmp); 2057 2058 if ((ModsTmp & SISrcMods::NEG) != 0) 2059 Mods ^= SISrcMods::NEG; 2060 2061 if ((ModsTmp & SISrcMods::ABS) != 0) 2062 Mods |= SISrcMods::ABS; 2063 } 2064 2065 // op_sel/op_sel_hi decide the source type and source. 2066 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16. 2067 // If the sources's op_sel is set, it picks the high half of the source 2068 // register. 2069 2070 Mods |= SISrcMods::OP_SEL_1; 2071 if (isExtractHiElt(Src, Src)) { 2072 Mods |= SISrcMods::OP_SEL_0; 2073 2074 // TODO: Should we try to look for neg/abs here? 2075 } 2076 2077 return true; 2078 } 2079 2080 return false; 2081 } 2082 2083 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src, 2084 SDValue &SrcMods) const { 2085 unsigned Mods = 0; 2086 SelectVOP3PMadMixModsImpl(In, Src, Mods); 2087 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32); 2088 return true; 2089 } 2090 2091 // TODO: Can we identify things like v_mad_mixhi_f16? 2092 bool AMDGPUDAGToDAGISel::SelectHi16Elt(SDValue In, SDValue &Src) const { 2093 if (In.isUndef()) { 2094 Src = In; 2095 return true; 2096 } 2097 2098 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) { 2099 SDLoc SL(In); 2100 SDValue K = CurDAG->getTargetConstant(C->getZExtValue() << 16, SL, MVT::i32); 2101 MachineSDNode *MovK = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 2102 SL, MVT::i32, K); 2103 Src = SDValue(MovK, 0); 2104 return true; 2105 } 2106 2107 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) { 2108 SDLoc SL(In); 2109 SDValue K = CurDAG->getTargetConstant( 2110 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32); 2111 MachineSDNode *MovK = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, 2112 SL, MVT::i32, K); 2113 Src = SDValue(MovK, 0); 2114 return true; 2115 } 2116 2117 return isExtractHiElt(In, Src); 2118 } 2119 2120 void AMDGPUDAGToDAGISel::PostprocessISelDAG() { 2121 const AMDGPUTargetLowering& Lowering = 2122 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering()); 2123 bool IsModified = false; 2124 do { 2125 IsModified = false; 2126 2127 // Go over all selected nodes and try to fold them a bit more 2128 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin(); 2129 while (Position != CurDAG->allnodes_end()) { 2130 SDNode *Node = &*Position++; 2131 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node); 2132 if (!MachineNode) 2133 continue; 2134 2135 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG); 2136 if (ResNode != Node) { 2137 if (ResNode) 2138 ReplaceUses(Node, ResNode); 2139 IsModified = true; 2140 } 2141 } 2142 CurDAG->RemoveDeadNodes(); 2143 } while (IsModified); 2144 } 2145 2146 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { 2147 Subtarget = &MF.getSubtarget<R600Subtarget>(); 2148 return SelectionDAGISel::runOnMachineFunction(MF); 2149 } 2150 2151 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const { 2152 if (!N->readMem()) 2153 return false; 2154 if (CbId == -1) 2155 return N->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS || 2156 N->getAddressSpace() == AMDGPUASI.CONSTANT_ADDRESS_32BIT; 2157 2158 return N->getAddressSpace() == AMDGPUASI.CONSTANT_BUFFER_0 + CbId; 2159 } 2160 2161 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr, 2162 SDValue& IntPtr) { 2163 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) { 2164 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr), 2165 true); 2166 return true; 2167 } 2168 return false; 2169 } 2170 2171 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr, 2172 SDValue& BaseReg, SDValue &Offset) { 2173 if (!isa<ConstantSDNode>(Addr)) { 2174 BaseReg = Addr; 2175 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true); 2176 return true; 2177 } 2178 return false; 2179 } 2180 2181 void R600DAGToDAGISel::Select(SDNode *N) { 2182 unsigned int Opc = N->getOpcode(); 2183 if (N->isMachineOpcode()) { 2184 N->setNodeId(-1); 2185 return; // Already selected. 2186 } 2187 2188 switch (Opc) { 2189 default: break; 2190 case AMDGPUISD::BUILD_VERTICAL_VECTOR: 2191 case ISD::SCALAR_TO_VECTOR: 2192 case ISD::BUILD_VECTOR: { 2193 EVT VT = N->getValueType(0); 2194 unsigned NumVectorElts = VT.getVectorNumElements(); 2195 unsigned RegClassID; 2196 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG 2197 // that adds a 128 bits reg copy when going through TwoAddressInstructions 2198 // pass. We want to avoid 128 bits copies as much as possible because they 2199 // can't be bundled by our scheduler. 2200 switch(NumVectorElts) { 2201 case 2: RegClassID = R600::R600_Reg64RegClassID; break; 2202 case 4: 2203 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR) 2204 RegClassID = R600::R600_Reg128VerticalRegClassID; 2205 else 2206 RegClassID = R600::R600_Reg128RegClassID; 2207 break; 2208 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR"); 2209 } 2210 SelectBuildVector(N, RegClassID); 2211 return; 2212 } 2213 } 2214 2215 SelectCode(N); 2216 } 2217 2218 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base, 2219 SDValue &Offset) { 2220 ConstantSDNode *C; 2221 SDLoc DL(Addr); 2222 2223 if ((C = dyn_cast<ConstantSDNode>(Addr))) { 2224 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2225 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2226 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) && 2227 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) { 2228 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32); 2229 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2230 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) && 2231 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) { 2232 Base = Addr.getOperand(0); 2233 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32); 2234 } else { 2235 Base = Addr; 2236 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32); 2237 } 2238 2239 return true; 2240 } 2241 2242 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 2243 SDValue &Offset) { 2244 ConstantSDNode *IMMOffset; 2245 2246 if (Addr.getOpcode() == ISD::ADD 2247 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 2248 && isInt<16>(IMMOffset->getZExtValue())) { 2249 2250 Base = Addr.getOperand(0); 2251 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2252 MVT::i32); 2253 return true; 2254 // If the pointer address is constant, we can move it to the offset field. 2255 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 2256 && isInt<16>(IMMOffset->getZExtValue())) { 2257 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 2258 SDLoc(CurDAG->getEntryNode()), 2259 R600::ZERO, MVT::i32); 2260 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr), 2261 MVT::i32); 2262 return true; 2263 } 2264 2265 // Default case, no offset 2266 Base = Addr; 2267 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32); 2268 return true; 2269 } 2270
