1 //===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // On Fermi, image handles are not supported. To work around this, we traverse 11 // the machine code and replace image handles with concrete symbols. For this 12 // to work reliably, inlining of all function call must be performed. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #include "NVPTX.h" 17 #include "NVPTXMachineFunctionInfo.h" 18 #include "NVPTXSubtarget.h" 19 #include "NVPTXTargetMachine.h" 20 #include "llvm/ADT/DenseSet.h" 21 #include "llvm/CodeGen/MachineFunction.h" 22 #include "llvm/CodeGen/MachineFunctionPass.h" 23 #include "llvm/CodeGen/MachineRegisterInfo.h" 24 #include "llvm/Support/raw_ostream.h" 25 26 using namespace llvm; 27 28 namespace { 29 class NVPTXReplaceImageHandles : public MachineFunctionPass { 30 private: 31 static char ID; 32 DenseSet<MachineInstr *> InstrsToRemove; 33 34 public: 35 NVPTXReplaceImageHandles(); 36 37 bool runOnMachineFunction(MachineFunction &MF) override; 38 39 const char *getPassName() const override { 40 return "NVPTX Replace Image Handles"; 41 } 42 private: 43 bool processInstr(MachineInstr &MI); 44 void replaceImageHandle(MachineOperand &Op, MachineFunction &MF); 45 bool findIndexForHandle(MachineOperand &Op, MachineFunction &MF, 46 unsigned &Idx); 47 }; 48 } 49 50 char NVPTXReplaceImageHandles::ID = 0; 51 52 NVPTXReplaceImageHandles::NVPTXReplaceImageHandles() 53 : MachineFunctionPass(ID) {} 54 55 bool NVPTXReplaceImageHandles::runOnMachineFunction(MachineFunction &MF) { 56 bool Changed = false; 57 InstrsToRemove.clear(); 58 59 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE; 60 ++BI) { 61 for (MachineBasicBlock::iterator I = (*BI).begin(), E = (*BI).end(); 62 I != E; ++I) { 63 MachineInstr &MI = *I; 64 Changed |= processInstr(MI); 65 } 66 } 67 68 // Now clean up any handle-access instructions 69 // This is needed in debug mode when code cleanup passes are not executed, 70 // but we need the handle access to be eliminated because they are not 71 // valid instructions when image handles are disabled. 72 for (DenseSet<MachineInstr *>::iterator I = InstrsToRemove.begin(), 73 E = InstrsToRemove.end(); I != E; ++I) { 74 (*I)->eraseFromParent(); 75 } 76 return Changed; 77 } 78 79 bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) { 80 MachineFunction &MF = *MI.getParent()->getParent(); 81 const MCInstrDesc &MCID = MI.getDesc(); 82 83 if (MCID.TSFlags & NVPTXII::IsTexFlag) { 84 // This is a texture fetch, so operand 4 is a texref and operand 5 is 85 // a samplerref 86 MachineOperand &TexHandle = MI.getOperand(4); 87 replaceImageHandle(TexHandle, MF); 88 89 if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) { 90 MachineOperand &SampHandle = MI.getOperand(5); 91 replaceImageHandle(SampHandle, MF); 92 } 93 94 return true; 95 } else if (MCID.TSFlags & NVPTXII::IsSuldMask) { 96 unsigned VecSize = 97 1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) - 1); 98 99 // For a surface load of vector size N, the Nth operand will be the surfref 100 MachineOperand &SurfHandle = MI.getOperand(VecSize); 101 102 replaceImageHandle(SurfHandle, MF); 103 104 return true; 105 } else if (MCID.TSFlags & NVPTXII::IsSustFlag) { 106 // This is a surface store, so operand 0 is a surfref 107 MachineOperand &SurfHandle = MI.getOperand(0); 108 109 replaceImageHandle(SurfHandle, MF); 110 111 return true; 112 } else if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) { 113 // This is a query, so operand 1 is a surfref/texref 114 MachineOperand &Handle = MI.getOperand(1); 115 116 replaceImageHandle(Handle, MF); 117 118 return true; 119 } 120 121 return false; 122 } 123 124 void NVPTXReplaceImageHandles:: 125 replaceImageHandle(MachineOperand &Op, MachineFunction &MF) { 126 unsigned Idx; 127 if (findIndexForHandle(Op, MF, Idx)) { 128 Op.ChangeToImmediate(Idx); 129 } 130 } 131 132 bool NVPTXReplaceImageHandles:: 133 findIndexForHandle(MachineOperand &Op, MachineFunction &MF, unsigned &Idx) { 134 const MachineRegisterInfo &MRI = MF.getRegInfo(); 135 NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>(); 136 137 assert(Op.isReg() && "Handle is not in a reg?"); 138 139 // Which instruction defines the handle? 140 MachineInstr &TexHandleDef = *MRI.getVRegDef(Op.getReg()); 141 142 switch (TexHandleDef.getOpcode()) { 143 case NVPTX::LD_i64_avar: { 144 // The handle is a parameter value being loaded, replace with the 145 // parameter symbol 146 const NVPTXTargetMachine &TM = 147 static_cast<const NVPTXTargetMachine &>(MF.getTarget()); 148 if (TM.getDrvInterface() == NVPTX::CUDA) { 149 // For CUDA, we preserve the param loads coming from function arguments 150 return false; 151 } 152 153 assert(TexHandleDef.getOperand(6).isSymbol() && "Load is not a symbol!"); 154 StringRef Sym = TexHandleDef.getOperand(6).getSymbolName(); 155 std::string ParamBaseName = MF.getName(); 156 ParamBaseName += "_param_"; 157 assert(Sym.startswith(ParamBaseName) && "Invalid symbol reference"); 158 unsigned Param = atoi(Sym.data()+ParamBaseName.size()); 159 std::string NewSym; 160 raw_string_ostream NewSymStr(NewSym); 161 NewSymStr << MF.getFunction()->getName() << "_param_" << Param; 162 163 InstrsToRemove.insert(&TexHandleDef); 164 Idx = MFI->getImageHandleSymbolIndex(NewSymStr.str().c_str()); 165 return true; 166 } 167 case NVPTX::texsurf_handles: { 168 // The handle is a global variable, replace with the global variable name 169 assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!"); 170 const GlobalValue *GV = TexHandleDef.getOperand(1).getGlobal(); 171 assert(GV->hasName() && "Global sampler must be named!"); 172 InstrsToRemove.insert(&TexHandleDef); 173 Idx = MFI->getImageHandleSymbolIndex(GV->getName().data()); 174 return true; 175 } 176 case NVPTX::nvvm_move_i64: 177 case TargetOpcode::COPY: { 178 bool Res = findIndexForHandle(TexHandleDef.getOperand(1), MF, Idx); 179 if (Res) { 180 InstrsToRemove.insert(&TexHandleDef); 181 } 182 return Res; 183 } 184 default: 185 llvm_unreachable("Unknown instruction operating on handle"); 186 } 187 } 188 189 MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesPass() { 190 return new NVPTXReplaceImageHandles(); 191 } 192
