1 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #include "SIMachineFunctionInfo.h" 11 #include "AMDGPUArgumentUsageInfo.h" 12 #include "AMDGPUSubtarget.h" 13 #include "SIRegisterInfo.h" 14 #include "MCTargetDesc/AMDGPUMCTargetDesc.h" 15 #include "Utils/AMDGPUBaseInfo.h" 16 #include "llvm/ADT/Optional.h" 17 #include "llvm/CodeGen/MachineBasicBlock.h" 18 #include "llvm/CodeGen/MachineFrameInfo.h" 19 #include "llvm/CodeGen/MachineFunction.h" 20 #include "llvm/CodeGen/MachineRegisterInfo.h" 21 #include "llvm/IR/CallingConv.h" 22 #include "llvm/IR/Function.h" 23 #include <cassert> 24 #include <vector> 25 26 #define MAX_LANES 64 27 28 using namespace llvm; 29 30 SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF) 31 : AMDGPUMachineFunction(MF), 32 PrivateSegmentBuffer(false), 33 DispatchPtr(false), 34 QueuePtr(false), 35 KernargSegmentPtr(false), 36 DispatchID(false), 37 FlatScratchInit(false), 38 WorkGroupIDX(false), 39 WorkGroupIDY(false), 40 WorkGroupIDZ(false), 41 WorkGroupInfo(false), 42 PrivateSegmentWaveByteOffset(false), 43 WorkItemIDX(false), 44 WorkItemIDY(false), 45 WorkItemIDZ(false), 46 ImplicitBufferPtr(false), 47 ImplicitArgPtr(false), 48 GITPtrHigh(0xffffffff), 49 HighBitsOf32BitAddress(0) { 50 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 51 const Function &F = MF.getFunction(); 52 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F); 53 WavesPerEU = ST.getWavesPerEU(F); 54 55 Occupancy = getMaxWavesPerEU(); 56 limitOccupancy(MF); 57 CallingConv::ID CC = F.getCallingConv(); 58 59 if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL) { 60 if (!F.arg_empty()) 61 KernargSegmentPtr = true; 62 WorkGroupIDX = true; 63 WorkItemIDX = true; 64 } else if (CC == CallingConv::AMDGPU_PS) { 65 PSInputAddr = AMDGPU::getInitialPSInputAddr(F); 66 } 67 68 if (!isEntryFunction()) { 69 // Non-entry functions have no special inputs for now, other registers 70 // required for scratch access. 71 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3; 72 ScratchWaveOffsetReg = AMDGPU::SGPR4; 73 FrameOffsetReg = AMDGPU::SGPR5; 74 StackPtrOffsetReg = AMDGPU::SGPR32; 75 76 ArgInfo.PrivateSegmentBuffer = 77 ArgDescriptor::createRegister(ScratchRSrcReg); 78 ArgInfo.PrivateSegmentWaveByteOffset = 79 ArgDescriptor::createRegister(ScratchWaveOffsetReg); 80 81 if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) 82 ImplicitArgPtr = true; 83 } else { 84 if (F.hasFnAttribute("amdgpu-implicitarg-ptr")) { 85 KernargSegmentPtr = true; 86 MaxKernArgAlign = std::max(ST.getAlignmentForImplicitArgPtr(), 87 MaxKernArgAlign); 88 } 89 } 90 91 if (ST.debuggerEmitPrologue()) { 92 // Enable everything. 93 WorkGroupIDX = true; 94 WorkGroupIDY = true; 95 WorkGroupIDZ = true; 96 WorkItemIDX = true; 97 WorkItemIDY = true; 98 WorkItemIDZ = true; 99 } else { 100 if (F.hasFnAttribute("amdgpu-work-group-id-x")) 101 WorkGroupIDX = true; 102 103 if (F.hasFnAttribute("amdgpu-work-group-id-y")) 104 WorkGroupIDY = true; 105 106 if (F.hasFnAttribute("amdgpu-work-group-id-z")) 107 WorkGroupIDZ = true; 108 109 if (F.hasFnAttribute("amdgpu-work-item-id-x")) 110 WorkItemIDX = true; 111 112 if (F.hasFnAttribute("amdgpu-work-item-id-y")) 113 WorkItemIDY = true; 114 115 if (F.hasFnAttribute("amdgpu-work-item-id-z")) 116 WorkItemIDZ = true; 117 } 118 119 const MachineFrameInfo &FrameInfo = MF.getFrameInfo(); 120 bool MaySpill = ST.isVGPRSpillingEnabled(F); 121 bool HasStackObjects = FrameInfo.hasStackObjects(); 122 123 if (isEntryFunction()) { 124 // X, XY, and XYZ are the only supported combinations, so make sure Y is 125 // enabled if Z is. 126 if (WorkItemIDZ) 127 WorkItemIDY = true; 128 129 if (HasStackObjects || MaySpill) { 130 PrivateSegmentWaveByteOffset = true; 131 132 // HS and GS always have the scratch wave offset in SGPR5 on GFX9. 133 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 && 134 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS)) 135 ArgInfo.PrivateSegmentWaveByteOffset 136 = ArgDescriptor::createRegister(AMDGPU::SGPR5); 137 } 138 } 139 140 bool IsCOV2 = ST.isAmdCodeObjectV2(F); 141 if (IsCOV2) { 142 if (HasStackObjects || MaySpill) 143 PrivateSegmentBuffer = true; 144 145 if (F.hasFnAttribute("amdgpu-dispatch-ptr")) 146 DispatchPtr = true; 147 148 if (F.hasFnAttribute("amdgpu-queue-ptr")) 149 QueuePtr = true; 150 151 if (F.hasFnAttribute("amdgpu-dispatch-id")) 152 DispatchID = true; 153 } else if (ST.isMesaGfxShader(F)) { 154 if (HasStackObjects || MaySpill) 155 ImplicitBufferPtr = true; 156 } 157 158 if (F.hasFnAttribute("amdgpu-kernarg-segment-ptr")) 159 KernargSegmentPtr = true; 160 161 if (ST.hasFlatAddressSpace() && isEntryFunction() && IsCOV2) { 162 // TODO: This could be refined a lot. The attribute is a poor way of 163 // detecting calls that may require it before argument lowering. 164 if (HasStackObjects || F.hasFnAttribute("amdgpu-flat-scratch")) 165 FlatScratchInit = true; 166 } 167 168 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high"); 169 StringRef S = A.getValueAsString(); 170 if (!S.empty()) 171 S.consumeInteger(0, GITPtrHigh); 172 173 A = F.getFnAttribute("amdgpu-32bit-address-high-bits"); 174 S = A.getValueAsString(); 175 if (!S.empty()) 176 S.consumeInteger(0, HighBitsOf32BitAddress); 177 } 178 179 void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) { 180 limitOccupancy(getMaxWavesPerEU()); 181 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>(); 182 limitOccupancy(ST.getOccupancyWithLocalMemSize(getLDSSize(), 183 MF.getFunction())); 184 } 185 186 unsigned SIMachineFunctionInfo::addPrivateSegmentBuffer( 187 const SIRegisterInfo &TRI) { 188 ArgInfo.PrivateSegmentBuffer = 189 ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 190 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_128RegClass)); 191 NumUserSGPRs += 4; 192 return ArgInfo.PrivateSegmentBuffer.getRegister(); 193 } 194 195 unsigned SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) { 196 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 197 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 198 NumUserSGPRs += 2; 199 return ArgInfo.DispatchPtr.getRegister(); 200 } 201 202 unsigned SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) { 203 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 204 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 205 NumUserSGPRs += 2; 206 return ArgInfo.QueuePtr.getRegister(); 207 } 208 209 unsigned SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) { 210 ArgInfo.KernargSegmentPtr 211 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 212 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 213 NumUserSGPRs += 2; 214 return ArgInfo.KernargSegmentPtr.getRegister(); 215 } 216 217 unsigned SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) { 218 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 219 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 220 NumUserSGPRs += 2; 221 return ArgInfo.DispatchID.getRegister(); 222 } 223 224 unsigned SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) { 225 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 226 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 227 NumUserSGPRs += 2; 228 return ArgInfo.FlatScratchInit.getRegister(); 229 } 230 231 unsigned SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) { 232 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg( 233 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass)); 234 NumUserSGPRs += 2; 235 return ArgInfo.ImplicitBufferPtr.getRegister(); 236 } 237 238 static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) { 239 for (unsigned I = 0; CSRegs[I]; ++I) { 240 if (CSRegs[I] == Reg) 241 return true; 242 } 243 244 return false; 245 } 246 247 /// Reserve a slice of a VGPR to support spilling for FrameIndex \p FI. 248 bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF, 249 int FI) { 250 std::vector<SpilledReg> &SpillLanes = SGPRToVGPRSpills[FI]; 251 252 // This has already been allocated. 253 if (!SpillLanes.empty()) 254 return true; 255 256 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); 257 const SIRegisterInfo *TRI = ST.getRegisterInfo(); 258 MachineFrameInfo &FrameInfo = MF.getFrameInfo(); 259 MachineRegisterInfo &MRI = MF.getRegInfo(); 260 unsigned WaveSize = ST.getWavefrontSize(); 261 262 unsigned Size = FrameInfo.getObjectSize(FI); 263 assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size"); 264 assert(TRI->spillSGPRToVGPR() && "not spilling SGPRs to VGPRs"); 265 266 int NumLanes = Size / 4; 267 268 const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF); 269 270 // Make sure to handle the case where a wide SGPR spill may span between two 271 // VGPRs. 272 for (int I = 0; I < NumLanes; ++I, ++NumVGPRSpillLanes) { 273 unsigned LaneVGPR; 274 unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize); 275 276 if (VGPRIndex == 0) { 277 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF); 278 if (LaneVGPR == AMDGPU::NoRegister) { 279 // We have no VGPRs left for spilling SGPRs. Reset because we will not 280 // partially spill the SGPR to VGPRs. 281 SGPRToVGPRSpills.erase(FI); 282 NumVGPRSpillLanes -= I; 283 return false; 284 } 285 286 Optional<int> CSRSpillFI; 287 if ((FrameInfo.hasCalls() || !isEntryFunction()) && CSRegs && 288 isCalleeSavedReg(CSRegs, LaneVGPR)) { 289 CSRSpillFI = FrameInfo.CreateSpillStackObject(4, 4); 290 } 291 292 SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, CSRSpillFI)); 293 294 // Add this register as live-in to all blocks to avoid machine verifer 295 // complaining about use of an undefined physical register. 296 for (MachineBasicBlock &BB : MF) 297 BB.addLiveIn(LaneVGPR); 298 } else { 299 LaneVGPR = SpillVGPRs.back().VGPR; 300 } 301 302 SpillLanes.push_back(SpilledReg(LaneVGPR, VGPRIndex)); 303 } 304 305 return true; 306 } 307 308 void SIMachineFunctionInfo::removeSGPRToVGPRFrameIndices(MachineFrameInfo &MFI) { 309 for (auto &R : SGPRToVGPRSpills) 310 MFI.RemoveStackObject(R.first); 311 } 312 313 314 /// \returns VGPR used for \p Dim' work item ID. 315 unsigned SIMachineFunctionInfo::getWorkItemIDVGPR(unsigned Dim) const { 316 switch (Dim) { 317 case 0: 318 assert(hasWorkItemIDX()); 319 return AMDGPU::VGPR0; 320 case 1: 321 assert(hasWorkItemIDY()); 322 return AMDGPU::VGPR1; 323 case 2: 324 assert(hasWorkItemIDZ()); 325 return AMDGPU::VGPR2; 326 } 327 llvm_unreachable("unexpected dimension"); 328 } 329 330 MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const { 331 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs"); 332 return AMDGPU::SGPR0 + NumUserSGPRs; 333 } 334 335 MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const { 336 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs; 337 } 338