1 //===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines the interfaces that NVPTX uses to lower LLVM code into a 11 // selection DAG. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "NVPTXISelLowering.h" 16 #include "MCTargetDesc/NVPTXBaseInfo.h" 17 #include "NVPTX.h" 18 #include "NVPTXSubtarget.h" 19 #include "NVPTXTargetMachine.h" 20 #include "NVPTXTargetObjectFile.h" 21 #include "NVPTXUtilities.h" 22 #include "llvm/ADT/APInt.h" 23 #include "llvm/ADT/SmallVector.h" 24 #include "llvm/ADT/StringRef.h" 25 #include "llvm/CodeGen/Analysis.h" 26 #include "llvm/CodeGen/MachineFunction.h" 27 #include "llvm/CodeGen/MachineMemOperand.h" 28 #include "llvm/CodeGen/SelectionDAG.h" 29 #include "llvm/CodeGen/SelectionDAGNodes.h" 30 #include "llvm/CodeGen/TargetCallingConv.h" 31 #include "llvm/CodeGen/TargetLowering.h" 32 #include "llvm/CodeGen/ValueTypes.h" 33 #include "llvm/IR/Argument.h" 34 #include "llvm/IR/Attributes.h" 35 #include "llvm/IR/CallSite.h" 36 #include "llvm/IR/Constants.h" 37 #include "llvm/IR/DataLayout.h" 38 #include "llvm/IR/DerivedTypes.h" 39 #include "llvm/IR/Function.h" 40 #include "llvm/IR/GlobalValue.h" 41 #include "llvm/IR/Instruction.h" 42 #include "llvm/IR/Instructions.h" 43 #include "llvm/IR/Module.h" 44 #include "llvm/IR/Type.h" 45 #include "llvm/IR/Value.h" 46 #include "llvm/Support/Casting.h" 47 #include "llvm/Support/CodeGen.h" 48 #include "llvm/Support/CommandLine.h" 49 #include "llvm/Support/ErrorHandling.h" 50 #include "llvm/Support/MachineValueType.h" 51 #include "llvm/Support/MathExtras.h" 52 #include "llvm/Support/raw_ostream.h" 53 #include "llvm/Target/TargetMachine.h" 54 #include "llvm/Target/TargetOptions.h" 55 #include <algorithm> 56 #include <cassert> 57 #include <cstdint> 58 #include <iterator> 59 #include <sstream> 60 #include <string> 61 #include <utility> 62 #include <vector> 63 64 #define DEBUG_TYPE "nvptx-lower" 65 66 using namespace llvm; 67 68 static unsigned int uniqueCallSite = 0; 69 70 static cl::opt<bool> sched4reg( 71 "nvptx-sched4reg", 72 cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false)); 73 74 static cl::opt<unsigned> 75 FMAContractLevelOpt("nvptx-fma-level", cl::ZeroOrMore, cl::Hidden, 76 cl::desc("NVPTX Specific: FMA contraction (0: don't do it" 77 " 1: do it 2: do it aggressively"), 78 cl::init(2)); 79 80 static cl::opt<int> UsePrecDivF32( 81 "nvptx-prec-divf32", cl::ZeroOrMore, cl::Hidden, 82 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 83 " IEEE Compliant F32 div.rnd if available."), 84 cl::init(2)); 85 86 static cl::opt<bool> UsePrecSqrtF32( 87 "nvptx-prec-sqrtf32", cl::Hidden, 88 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 89 cl::init(true)); 90 91 static cl::opt<bool> FtzEnabled( 92 "nvptx-f32ftz", cl::ZeroOrMore, cl::Hidden, 93 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), 94 cl::init(false)); 95 96 int NVPTXTargetLowering::getDivF32Level() const { 97 if (UsePrecDivF32.getNumOccurrences() > 0) { 98 // If nvptx-prec-div32=N is used on the command-line, always honor it 99 return UsePrecDivF32; 100 } else { 101 // Otherwise, use div.approx if fast math is enabled 102 if (getTargetMachine().Options.UnsafeFPMath) 103 return 0; 104 else 105 return 2; 106 } 107 } 108 109 bool NVPTXTargetLowering::usePrecSqrtF32() const { 110 if (UsePrecSqrtF32.getNumOccurrences() > 0) { 111 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it 112 return UsePrecSqrtF32; 113 } else { 114 // Otherwise, use sqrt.approx if fast math is enabled 115 return !getTargetMachine().Options.UnsafeFPMath; 116 } 117 } 118 119 bool NVPTXTargetLowering::useF32FTZ(const MachineFunction &MF) const { 120 // TODO: Get rid of this flag; there can be only one way to do this. 121 if (FtzEnabled.getNumOccurrences() > 0) { 122 // If nvptx-f32ftz is used on the command-line, always honor it 123 return FtzEnabled; 124 } else { 125 const Function &F = MF.getFunction(); 126 // Otherwise, check for an nvptx-f32ftz attribute on the function 127 if (F.hasFnAttribute("nvptx-f32ftz")) 128 return F.getFnAttribute("nvptx-f32ftz").getValueAsString() == "true"; 129 else 130 return false; 131 } 132 } 133 134 static bool IsPTXVectorType(MVT VT) { 135 switch (VT.SimpleTy) { 136 default: 137 return false; 138 case MVT::v2i1: 139 case MVT::v4i1: 140 case MVT::v2i8: 141 case MVT::v4i8: 142 case MVT::v2i16: 143 case MVT::v4i16: 144 case MVT::v2i32: 145 case MVT::v4i32: 146 case MVT::v2i64: 147 case MVT::v2f16: 148 case MVT::v4f16: 149 case MVT::v8f16: // <4 x f16x2> 150 case MVT::v2f32: 151 case MVT::v4f32: 152 case MVT::v2f64: 153 return true; 154 } 155 } 156 157 /// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive 158 /// EVTs that compose it. Unlike ComputeValueVTs, this will break apart vectors 159 /// into their primitive components. 160 /// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the 161 /// same number of types as the Ins/Outs arrays in LowerFormalArguments, 162 /// LowerCall, and LowerReturn. 163 static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL, 164 Type *Ty, SmallVectorImpl<EVT> &ValueVTs, 165 SmallVectorImpl<uint64_t> *Offsets = nullptr, 166 uint64_t StartingOffset = 0) { 167 SmallVector<EVT, 16> TempVTs; 168 SmallVector<uint64_t, 16> TempOffsets; 169 170 // Special case for i128 - decompose to (i64, i64) 171 if (Ty->isIntegerTy(128)) { 172 ValueVTs.push_back(EVT(MVT::i64)); 173 ValueVTs.push_back(EVT(MVT::i64)); 174 175 if (Offsets) { 176 Offsets->push_back(StartingOffset + 0); 177 Offsets->push_back(StartingOffset + 8); 178 } 179 180 return; 181 } 182 183 ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset); 184 for (unsigned i = 0, e = TempVTs.size(); i != e; ++i) { 185 EVT VT = TempVTs[i]; 186 uint64_t Off = TempOffsets[i]; 187 // Split vectors into individual elements, except for v2f16, which 188 // we will pass as a single scalar. 189 if (VT.isVector()) { 190 unsigned NumElts = VT.getVectorNumElements(); 191 EVT EltVT = VT.getVectorElementType(); 192 // Vectors with an even number of f16 elements will be passed to 193 // us as an array of v2f16 elements. We must match this so we 194 // stay in sync with Ins/Outs. 195 if (EltVT == MVT::f16 && NumElts % 2 == 0) { 196 EltVT = MVT::v2f16; 197 NumElts /= 2; 198 } 199 for (unsigned j = 0; j != NumElts; ++j) { 200 ValueVTs.push_back(EltVT); 201 if (Offsets) 202 Offsets->push_back(Off + j * EltVT.getStoreSize()); 203 } 204 } else { 205 ValueVTs.push_back(VT); 206 if (Offsets) 207 Offsets->push_back(Off); 208 } 209 } 210 } 211 212 // Check whether we can merge loads/stores of some of the pieces of a 213 // flattened function parameter or return value into a single vector 214 // load/store. 215 // 216 // The flattened parameter is represented as a list of EVTs and 217 // offsets, and the whole structure is aligned to ParamAlignment. This 218 // function determines whether we can load/store pieces of the 219 // parameter starting at index Idx using a single vectorized op of 220 // size AccessSize. If so, it returns the number of param pieces 221 // covered by the vector op. Otherwise, it returns 1. 222 static unsigned CanMergeParamLoadStoresStartingAt( 223 unsigned Idx, uint32_t AccessSize, const SmallVectorImpl<EVT> &ValueVTs, 224 const SmallVectorImpl<uint64_t> &Offsets, unsigned ParamAlignment) { 225 assert(isPowerOf2_32(AccessSize) && "must be a power of 2!"); 226 227 // Can't vectorize if param alignment is not sufficient. 228 if (AccessSize > ParamAlignment) 229 return 1; 230 // Can't vectorize if offset is not aligned. 231 if (Offsets[Idx] & (AccessSize - 1)) 232 return 1; 233 234 EVT EltVT = ValueVTs[Idx]; 235 unsigned EltSize = EltVT.getStoreSize(); 236 237 // Element is too large to vectorize. 238 if (EltSize >= AccessSize) 239 return 1; 240 241 unsigned NumElts = AccessSize / EltSize; 242 // Can't vectorize if AccessBytes if not a multiple of EltSize. 243 if (AccessSize != EltSize * NumElts) 244 return 1; 245 246 // We don't have enough elements to vectorize. 247 if (Idx + NumElts > ValueVTs.size()) 248 return 1; 249 250 // PTX ISA can only deal with 2- and 4-element vector ops. 251 if (NumElts != 4 && NumElts != 2) 252 return 1; 253 254 for (unsigned j = Idx + 1; j < Idx + NumElts; ++j) { 255 // Types do not match. 256 if (ValueVTs[j] != EltVT) 257 return 1; 258 259 // Elements are not contiguous. 260 if (Offsets[j] - Offsets[j - 1] != EltSize) 261 return 1; 262 } 263 // OK. We can vectorize ValueVTs[i..i+NumElts) 264 return NumElts; 265 } 266 267 // Flags for tracking per-element vectorization state of loads/stores 268 // of a flattened function parameter or return value. 269 enum ParamVectorizationFlags { 270 PVF_INNER = 0x0, // Middle elements of a vector. 271 PVF_FIRST = 0x1, // First element of the vector. 272 PVF_LAST = 0x2, // Last element of the vector. 273 // Scalar is effectively a 1-element vector. 274 PVF_SCALAR = PVF_FIRST | PVF_LAST 275 }; 276 277 // Computes whether and how we can vectorize the loads/stores of a 278 // flattened function parameter or return value. 279 // 280 // The flattened parameter is represented as the list of ValueVTs and 281 // Offsets, and is aligned to ParamAlignment bytes. We return a vector 282 // of the same size as ValueVTs indicating how each piece should be 283 // loaded/stored (i.e. as a scalar, or as part of a vector 284 // load/store). 285 static SmallVector<ParamVectorizationFlags, 16> 286 VectorizePTXValueVTs(const SmallVectorImpl<EVT> &ValueVTs, 287 const SmallVectorImpl<uint64_t> &Offsets, 288 unsigned ParamAlignment) { 289 // Set vector size to match ValueVTs and mark all elements as 290 // scalars by default. 291 SmallVector<ParamVectorizationFlags, 16> VectorInfo; 292 VectorInfo.assign(ValueVTs.size(), PVF_SCALAR); 293 294 // Check what we can vectorize using 128/64/32-bit accesses. 295 for (int I = 0, E = ValueVTs.size(); I != E; ++I) { 296 // Skip elements we've already processed. 297 assert(VectorInfo[I] == PVF_SCALAR && "Unexpected vector info state."); 298 for (unsigned AccessSize : {16, 8, 4, 2}) { 299 unsigned NumElts = CanMergeParamLoadStoresStartingAt( 300 I, AccessSize, ValueVTs, Offsets, ParamAlignment); 301 // Mark vectorized elements. 302 switch (NumElts) { 303 default: 304 llvm_unreachable("Unexpected return value"); 305 case 1: 306 // Can't vectorize using this size, try next smaller size. 307 continue; 308 case 2: 309 assert(I + 1 < E && "Not enough elements."); 310 VectorInfo[I] = PVF_FIRST; 311 VectorInfo[I + 1] = PVF_LAST; 312 I += 1; 313 break; 314 case 4: 315 assert(I + 3 < E && "Not enough elements."); 316 VectorInfo[I] = PVF_FIRST; 317 VectorInfo[I + 1] = PVF_INNER; 318 VectorInfo[I + 2] = PVF_INNER; 319 VectorInfo[I + 3] = PVF_LAST; 320 I += 3; 321 break; 322 } 323 // Break out of the inner loop because we've already succeeded 324 // using largest possible AccessSize. 325 break; 326 } 327 } 328 return VectorInfo; 329 } 330 331 // NVPTXTargetLowering Constructor. 332 NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM, 333 const NVPTXSubtarget &STI) 334 : TargetLowering(TM), nvTM(&TM), STI(STI) { 335 // always lower memset, memcpy, and memmove intrinsics to load/store 336 // instructions, rather 337 // then generating calls to memset, mempcy or memmove. 338 MaxStoresPerMemset = (unsigned) 0xFFFFFFFF; 339 MaxStoresPerMemcpy = (unsigned) 0xFFFFFFFF; 340 MaxStoresPerMemmove = (unsigned) 0xFFFFFFFF; 341 342 setBooleanContents(ZeroOrNegativeOneBooleanContent); 343 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); 344 345 // Jump is Expensive. Don't create extra control flow for 'and', 'or' 346 // condition branches. 347 setJumpIsExpensive(true); 348 349 // Wide divides are _very_ slow. Try to reduce the width of the divide if 350 // possible. 351 addBypassSlowDiv(64, 32); 352 353 // By default, use the Source scheduling 354 if (sched4reg) 355 setSchedulingPreference(Sched::RegPressure); 356 else 357 setSchedulingPreference(Sched::Source); 358 359 auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action, 360 LegalizeAction NoF16Action) { 361 setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action); 362 }; 363 364 addRegisterClass(MVT::i1, &NVPTX::Int1RegsRegClass); 365 addRegisterClass(MVT::i16, &NVPTX::Int16RegsRegClass); 366 addRegisterClass(MVT::i32, &NVPTX::Int32RegsRegClass); 367 addRegisterClass(MVT::i64, &NVPTX::Int64RegsRegClass); 368 addRegisterClass(MVT::f32, &NVPTX::Float32RegsRegClass); 369 addRegisterClass(MVT::f64, &NVPTX::Float64RegsRegClass); 370 addRegisterClass(MVT::f16, &NVPTX::Float16RegsRegClass); 371 addRegisterClass(MVT::v2f16, &NVPTX::Float16x2RegsRegClass); 372 373 // Conversion to/from FP16/FP16x2 is always legal. 374 setOperationAction(ISD::SINT_TO_FP, MVT::f16, Legal); 375 setOperationAction(ISD::FP_TO_SINT, MVT::f16, Legal); 376 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom); 377 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom); 378 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Expand); 379 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f16, Expand); 380 381 setFP16OperationAction(ISD::SETCC, MVT::f16, Legal, Promote); 382 setFP16OperationAction(ISD::SETCC, MVT::v2f16, Legal, Expand); 383 384 // Operations not directly supported by NVPTX. 385 for (MVT VT : {MVT::f16, MVT::v2f16, MVT::f32, MVT::f64, MVT::i1, MVT::i8, 386 MVT::i16, MVT::i32, MVT::i64}) { 387 setOperationAction(ISD::SELECT_CC, VT, Expand); 388 setOperationAction(ISD::BR_CC, VT, Expand); 389 } 390 391 // Some SIGN_EXTEND_INREG can be done using cvt instruction. 392 // For others we will expand to a SHL/SRA pair. 393 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i64, Legal); 394 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); 395 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal); 396 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); 397 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); 398 399 setOperationAction(ISD::SHL_PARTS, MVT::i32 , Custom); 400 setOperationAction(ISD::SRA_PARTS, MVT::i32 , Custom); 401 setOperationAction(ISD::SRL_PARTS, MVT::i32 , Custom); 402 setOperationAction(ISD::SHL_PARTS, MVT::i64 , Custom); 403 setOperationAction(ISD::SRA_PARTS, MVT::i64 , Custom); 404 setOperationAction(ISD::SRL_PARTS, MVT::i64 , Custom); 405 406 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); 407 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); 408 409 // TODO: we may consider expanding ROTL/ROTR on older GPUs. Currently on GPUs 410 // that don't have h/w rotation we lower them to multi-instruction assembly. 411 // See ROT*_sw in NVPTXIntrInfo.td 412 setOperationAction(ISD::ROTL, MVT::i64, Legal); 413 setOperationAction(ISD::ROTR, MVT::i64, Legal); 414 setOperationAction(ISD::ROTL, MVT::i32, Legal); 415 setOperationAction(ISD::ROTR, MVT::i32, Legal); 416 417 setOperationAction(ISD::ROTL, MVT::i16, Expand); 418 setOperationAction(ISD::ROTR, MVT::i16, Expand); 419 setOperationAction(ISD::ROTL, MVT::i8, Expand); 420 setOperationAction(ISD::ROTR, MVT::i8, Expand); 421 setOperationAction(ISD::BSWAP, MVT::i16, Expand); 422 setOperationAction(ISD::BSWAP, MVT::i32, Expand); 423 setOperationAction(ISD::BSWAP, MVT::i64, Expand); 424 425 // Indirect branch is not supported. 426 // This also disables Jump Table creation. 427 setOperationAction(ISD::BR_JT, MVT::Other, Expand); 428 setOperationAction(ISD::BRIND, MVT::Other, Expand); 429 430 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); 431 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); 432 433 // We want to legalize constant related memmove and memcopy 434 // intrinsics. 435 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 436 437 // Turn FP extload into load/fpextend 438 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); 439 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); 440 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); 441 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); 442 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); 443 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand); 444 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); 445 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); 446 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand); 447 // Turn FP truncstore into trunc + store. 448 // FIXME: vector types should also be expanded 449 setTruncStoreAction(MVT::f32, MVT::f16, Expand); 450 setTruncStoreAction(MVT::f64, MVT::f16, Expand); 451 setTruncStoreAction(MVT::f64, MVT::f32, Expand); 452 453 // PTX does not support load / store predicate registers 454 setOperationAction(ISD::LOAD, MVT::i1, Custom); 455 setOperationAction(ISD::STORE, MVT::i1, Custom); 456 457 for (MVT VT : MVT::integer_valuetypes()) { 458 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); 459 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); 460 setTruncStoreAction(VT, MVT::i1, Expand); 461 } 462 463 // This is legal in NVPTX 464 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); 465 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); 466 setOperationAction(ISD::ConstantFP, MVT::f16, Legal); 467 468 // TRAP can be lowered to PTX trap 469 setOperationAction(ISD::TRAP, MVT::Other, Legal); 470 471 // Register custom handling for vector loads/stores 472 for (MVT VT : MVT::vector_valuetypes()) { 473 if (IsPTXVectorType(VT)) { 474 setOperationAction(ISD::LOAD, VT, Custom); 475 setOperationAction(ISD::STORE, VT, Custom); 476 setOperationAction(ISD::INTRINSIC_W_CHAIN, VT, Custom); 477 } 478 } 479 480 // Custom handling for i8 intrinsics 481 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom); 482 483 for (const auto& Ty : {MVT::i16, MVT::i32, MVT::i64}) { 484 setOperationAction(ISD::ABS, Ty, Legal); 485 setOperationAction(ISD::SMIN, Ty, Legal); 486 setOperationAction(ISD::SMAX, Ty, Legal); 487 setOperationAction(ISD::UMIN, Ty, Legal); 488 setOperationAction(ISD::UMAX, Ty, Legal); 489 490 setOperationAction(ISD::CTPOP, Ty, Legal); 491 setOperationAction(ISD::CTLZ, Ty, Legal); 492 } 493 494 setOperationAction(ISD::CTTZ, MVT::i16, Expand); 495 setOperationAction(ISD::CTTZ, MVT::i32, Expand); 496 setOperationAction(ISD::CTTZ, MVT::i64, Expand); 497 498 // PTX does not directly support SELP of i1, so promote to i32 first 499 setOperationAction(ISD::SELECT, MVT::i1, Custom); 500 501 // PTX cannot multiply two i64s in a single instruction. 502 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); 503 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); 504 505 // We have some custom DAG combine patterns for these nodes 506 setTargetDAGCombine(ISD::ADD); 507 setTargetDAGCombine(ISD::AND); 508 setTargetDAGCombine(ISD::FADD); 509 setTargetDAGCombine(ISD::MUL); 510 setTargetDAGCombine(ISD::SHL); 511 setTargetDAGCombine(ISD::SREM); 512 setTargetDAGCombine(ISD::UREM); 513 514 // setcc for f16x2 needs special handling to prevent legalizer's 515 // attempt to scalarize it due to v2i1 not being legal. 516 if (STI.allowFP16Math()) 517 setTargetDAGCombine(ISD::SETCC); 518 519 // Promote fp16 arithmetic if fp16 hardware isn't available or the 520 // user passed --nvptx-no-fp16-math. The flag is useful because, 521 // although sm_53+ GPUs have some sort of FP16 support in 522 // hardware, only sm_53 and sm_60 have full implementation. Others 523 // only have token amount of hardware and are likely to run faster 524 // by using fp32 units instead. 525 for (const auto &Op : {ISD::FADD, ISD::FMUL, ISD::FSUB, ISD::FMA}) { 526 setFP16OperationAction(Op, MVT::f16, Legal, Promote); 527 setFP16OperationAction(Op, MVT::v2f16, Legal, Expand); 528 } 529 530 // There's no neg.f16 instruction. Expand to (0-x). 531 setOperationAction(ISD::FNEG, MVT::f16, Expand); 532 setOperationAction(ISD::FNEG, MVT::v2f16, Expand); 533 534 // (would be) Library functions. 535 536 // These map to conversion instructions for scalar FP types. 537 for (const auto &Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT, 538 ISD::FROUND, ISD::FTRUNC}) { 539 setOperationAction(Op, MVT::f16, Legal); 540 setOperationAction(Op, MVT::f32, Legal); 541 setOperationAction(Op, MVT::f64, Legal); 542 setOperationAction(Op, MVT::v2f16, Expand); 543 } 544 545 // 'Expand' implements FCOPYSIGN without calling an external library. 546 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand); 547 setOperationAction(ISD::FCOPYSIGN, MVT::v2f16, Expand); 548 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); 549 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); 550 551 // These map to corresponding instructions for f32/f64. f16 must be 552 // promoted to f32. v2f16 is expanded to f16, which is then promoted 553 // to f32. 554 for (const auto &Op : {ISD::FDIV, ISD::FREM, ISD::FSQRT, ISD::FSIN, ISD::FCOS, 555 ISD::FABS, ISD::FMINNUM, ISD::FMAXNUM}) { 556 setOperationAction(Op, MVT::f16, Promote); 557 setOperationAction(Op, MVT::f32, Legal); 558 setOperationAction(Op, MVT::f64, Legal); 559 setOperationAction(Op, MVT::v2f16, Expand); 560 } 561 setOperationAction(ISD::FMINNUM, MVT::f16, Promote); 562 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); 563 setOperationAction(ISD::FMINNAN, MVT::f16, Promote); 564 setOperationAction(ISD::FMAXNAN, MVT::f16, Promote); 565 566 // No FEXP2, FLOG2. The PTX ex2 and log2 functions are always approximate. 567 // No FPOW or FREM in PTX. 568 569 // Now deduce the information based on the above mentioned 570 // actions 571 computeRegisterProperties(STI.getRegisterInfo()); 572 } 573 574 const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const { 575 switch ((NVPTXISD::NodeType)Opcode) { 576 case NVPTXISD::FIRST_NUMBER: 577 break; 578 case NVPTXISD::CALL: 579 return "NVPTXISD::CALL"; 580 case NVPTXISD::RET_FLAG: 581 return "NVPTXISD::RET_FLAG"; 582 case NVPTXISD::LOAD_PARAM: 583 return "NVPTXISD::LOAD_PARAM"; 584 case NVPTXISD::Wrapper: 585 return "NVPTXISD::Wrapper"; 586 case NVPTXISD::DeclareParam: 587 return "NVPTXISD::DeclareParam"; 588 case NVPTXISD::DeclareScalarParam: 589 return "NVPTXISD::DeclareScalarParam"; 590 case NVPTXISD::DeclareRet: 591 return "NVPTXISD::DeclareRet"; 592 case NVPTXISD::DeclareScalarRet: 593 return "NVPTXISD::DeclareScalarRet"; 594 case NVPTXISD::DeclareRetParam: 595 return "NVPTXISD::DeclareRetParam"; 596 case NVPTXISD::PrintCall: 597 return "NVPTXISD::PrintCall"; 598 case NVPTXISD::PrintConvergentCall: 599 return "NVPTXISD::PrintConvergentCall"; 600 case NVPTXISD::PrintCallUni: 601 return "NVPTXISD::PrintCallUni"; 602 case NVPTXISD::PrintConvergentCallUni: 603 return "NVPTXISD::PrintConvergentCallUni"; 604 case NVPTXISD::LoadParam: 605 return "NVPTXISD::LoadParam"; 606 case NVPTXISD::LoadParamV2: 607 return "NVPTXISD::LoadParamV2"; 608 case NVPTXISD::LoadParamV4: 609 return "NVPTXISD::LoadParamV4"; 610 case NVPTXISD::StoreParam: 611 return "NVPTXISD::StoreParam"; 612 case NVPTXISD::StoreParamV2: 613 return "NVPTXISD::StoreParamV2"; 614 case NVPTXISD::StoreParamV4: 615 return "NVPTXISD::StoreParamV4"; 616 case NVPTXISD::StoreParamS32: 617 return "NVPTXISD::StoreParamS32"; 618 case NVPTXISD::StoreParamU32: 619 return "NVPTXISD::StoreParamU32"; 620 case NVPTXISD::CallArgBegin: 621 return "NVPTXISD::CallArgBegin"; 622 case NVPTXISD::CallArg: 623 return "NVPTXISD::CallArg"; 624 case NVPTXISD::LastCallArg: 625 return "NVPTXISD::LastCallArg"; 626 case NVPTXISD::CallArgEnd: 627 return "NVPTXISD::CallArgEnd"; 628 case NVPTXISD::CallVoid: 629 return "NVPTXISD::CallVoid"; 630 case NVPTXISD::CallVal: 631 return "NVPTXISD::CallVal"; 632 case NVPTXISD::CallSymbol: 633 return "NVPTXISD::CallSymbol"; 634 case NVPTXISD::Prototype: 635 return "NVPTXISD::Prototype"; 636 case NVPTXISD::MoveParam: 637 return "NVPTXISD::MoveParam"; 638 case NVPTXISD::StoreRetval: 639 return "NVPTXISD::StoreRetval"; 640 case NVPTXISD::StoreRetvalV2: 641 return "NVPTXISD::StoreRetvalV2"; 642 case NVPTXISD::StoreRetvalV4: 643 return "NVPTXISD::StoreRetvalV4"; 644 case NVPTXISD::PseudoUseParam: 645 return "NVPTXISD::PseudoUseParam"; 646 case NVPTXISD::RETURN: 647 return "NVPTXISD::RETURN"; 648 case NVPTXISD::CallSeqBegin: 649 return "NVPTXISD::CallSeqBegin"; 650 case NVPTXISD::CallSeqEnd: 651 return "NVPTXISD::CallSeqEnd"; 652 case NVPTXISD::CallPrototype: 653 return "NVPTXISD::CallPrototype"; 654 case NVPTXISD::LoadV2: 655 return "NVPTXISD::LoadV2"; 656 case NVPTXISD::LoadV4: 657 return "NVPTXISD::LoadV4"; 658 case NVPTXISD::LDGV2: 659 return "NVPTXISD::LDGV2"; 660 case NVPTXISD::LDGV4: 661 return "NVPTXISD::LDGV4"; 662 case NVPTXISD::LDUV2: 663 return "NVPTXISD::LDUV2"; 664 case NVPTXISD::LDUV4: 665 return "NVPTXISD::LDUV4"; 666 case NVPTXISD::StoreV2: 667 return "NVPTXISD::StoreV2"; 668 case NVPTXISD::StoreV4: 669 return "NVPTXISD::StoreV4"; 670 case NVPTXISD::FUN_SHFL_CLAMP: 671 return "NVPTXISD::FUN_SHFL_CLAMP"; 672 case NVPTXISD::FUN_SHFR_CLAMP: 673 return "NVPTXISD::FUN_SHFR_CLAMP"; 674 case NVPTXISD::IMAD: 675 return "NVPTXISD::IMAD"; 676 case NVPTXISD::SETP_F16X2: 677 return "NVPTXISD::SETP_F16X2"; 678 case NVPTXISD::Dummy: 679 return "NVPTXISD::Dummy"; 680 case NVPTXISD::MUL_WIDE_SIGNED: 681 return "NVPTXISD::MUL_WIDE_SIGNED"; 682 case NVPTXISD::MUL_WIDE_UNSIGNED: 683 return "NVPTXISD::MUL_WIDE_UNSIGNED"; 684 case NVPTXISD::Tex1DFloatS32: return "NVPTXISD::Tex1DFloatS32"; 685 case NVPTXISD::Tex1DFloatFloat: return "NVPTXISD::Tex1DFloatFloat"; 686 case NVPTXISD::Tex1DFloatFloatLevel: 687 return "NVPTXISD::Tex1DFloatFloatLevel"; 688 case NVPTXISD::Tex1DFloatFloatGrad: 689 return "NVPTXISD::Tex1DFloatFloatGrad"; 690 case NVPTXISD::Tex1DS32S32: return "NVPTXISD::Tex1DS32S32"; 691 case NVPTXISD::Tex1DS32Float: return "NVPTXISD::Tex1DS32Float"; 692 case NVPTXISD::Tex1DS32FloatLevel: 693 return "NVPTXISD::Tex1DS32FloatLevel"; 694 case NVPTXISD::Tex1DS32FloatGrad: 695 return "NVPTXISD::Tex1DS32FloatGrad"; 696 case NVPTXISD::Tex1DU32S32: return "NVPTXISD::Tex1DU32S32"; 697 case NVPTXISD::Tex1DU32Float: return "NVPTXISD::Tex1DU32Float"; 698 case NVPTXISD::Tex1DU32FloatLevel: 699 return "NVPTXISD::Tex1DU32FloatLevel"; 700 case NVPTXISD::Tex1DU32FloatGrad: 701 return "NVPTXISD::Tex1DU32FloatGrad"; 702 case NVPTXISD::Tex1DArrayFloatS32: return "NVPTXISD::Tex1DArrayFloatS32"; 703 case NVPTXISD::Tex1DArrayFloatFloat: return "NVPTXISD::Tex1DArrayFloatFloat"; 704 case NVPTXISD::Tex1DArrayFloatFloatLevel: 705 return "NVPTXISD::Tex1DArrayFloatFloatLevel"; 706 case NVPTXISD::Tex1DArrayFloatFloatGrad: 707 return "NVPTXISD::Tex1DArrayFloatFloatGrad"; 708 case NVPTXISD::Tex1DArrayS32S32: return "NVPTXISD::Tex1DArrayS32S32"; 709 case NVPTXISD::Tex1DArrayS32Float: return "NVPTXISD::Tex1DArrayS32Float"; 710 case NVPTXISD::Tex1DArrayS32FloatLevel: 711 return "NVPTXISD::Tex1DArrayS32FloatLevel"; 712 case NVPTXISD::Tex1DArrayS32FloatGrad: 713 return "NVPTXISD::Tex1DArrayS32FloatGrad"; 714 case NVPTXISD::Tex1DArrayU32S32: return "NVPTXISD::Tex1DArrayU32S32"; 715 case NVPTXISD::Tex1DArrayU32Float: return "NVPTXISD::Tex1DArrayU32Float"; 716 case NVPTXISD::Tex1DArrayU32FloatLevel: 717 return "NVPTXISD::Tex1DArrayU32FloatLevel"; 718 case NVPTXISD::Tex1DArrayU32FloatGrad: 719 return "NVPTXISD::Tex1DArrayU32FloatGrad"; 720 case NVPTXISD::Tex2DFloatS32: return "NVPTXISD::Tex2DFloatS32"; 721 case NVPTXISD::Tex2DFloatFloat: return "NVPTXISD::Tex2DFloatFloat"; 722 case NVPTXISD::Tex2DFloatFloatLevel: 723 return "NVPTXISD::Tex2DFloatFloatLevel"; 724 case NVPTXISD::Tex2DFloatFloatGrad: 725 return "NVPTXISD::Tex2DFloatFloatGrad"; 726 case NVPTXISD::Tex2DS32S32: return "NVPTXISD::Tex2DS32S32"; 727 case NVPTXISD::Tex2DS32Float: return "NVPTXISD::Tex2DS32Float"; 728 case NVPTXISD::Tex2DS32FloatLevel: 729 return "NVPTXISD::Tex2DS32FloatLevel"; 730 case NVPTXISD::Tex2DS32FloatGrad: 731 return "NVPTXISD::Tex2DS32FloatGrad"; 732 case NVPTXISD::Tex2DU32S32: return "NVPTXISD::Tex2DU32S32"; 733 case NVPTXISD::Tex2DU32Float: return "NVPTXISD::Tex2DU32Float"; 734 case NVPTXISD::Tex2DU32FloatLevel: 735 return "NVPTXISD::Tex2DU32FloatLevel"; 736 case NVPTXISD::Tex2DU32FloatGrad: 737 return "NVPTXISD::Tex2DU32FloatGrad"; 738 case NVPTXISD::Tex2DArrayFloatS32: return "NVPTXISD::Tex2DArrayFloatS32"; 739 case NVPTXISD::Tex2DArrayFloatFloat: return "NVPTXISD::Tex2DArrayFloatFloat"; 740 case NVPTXISD::Tex2DArrayFloatFloatLevel: 741 return "NVPTXISD::Tex2DArrayFloatFloatLevel"; 742 case NVPTXISD::Tex2DArrayFloatFloatGrad: 743 return "NVPTXISD::Tex2DArrayFloatFloatGrad"; 744 case NVPTXISD::Tex2DArrayS32S32: return "NVPTXISD::Tex2DArrayS32S32"; 745 case NVPTXISD::Tex2DArrayS32Float: return "NVPTXISD::Tex2DArrayS32Float"; 746 case NVPTXISD::Tex2DArrayS32FloatLevel: 747 return "NVPTXISD::Tex2DArrayS32FloatLevel"; 748 case NVPTXISD::Tex2DArrayS32FloatGrad: 749 return "NVPTXISD::Tex2DArrayS32FloatGrad"; 750 case NVPTXISD::Tex2DArrayU32S32: return "NVPTXISD::Tex2DArrayU32S32"; 751 case NVPTXISD::Tex2DArrayU32Float: return "NVPTXISD::Tex2DArrayU32Float"; 752 case NVPTXISD::Tex2DArrayU32FloatLevel: 753 return "NVPTXISD::Tex2DArrayU32FloatLevel"; 754 case NVPTXISD::Tex2DArrayU32FloatGrad: 755 return "NVPTXISD::Tex2DArrayU32FloatGrad"; 756 case NVPTXISD::Tex3DFloatS32: return "NVPTXISD::Tex3DFloatS32"; 757 case NVPTXISD::Tex3DFloatFloat: return "NVPTXISD::Tex3DFloatFloat"; 758 case NVPTXISD::Tex3DFloatFloatLevel: 759 return "NVPTXISD::Tex3DFloatFloatLevel"; 760 case NVPTXISD::Tex3DFloatFloatGrad: 761 return "NVPTXISD::Tex3DFloatFloatGrad"; 762 case NVPTXISD::Tex3DS32S32: return "NVPTXISD::Tex3DS32S32"; 763 case NVPTXISD::Tex3DS32Float: return "NVPTXISD::Tex3DS32Float"; 764 case NVPTXISD::Tex3DS32FloatLevel: 765 return "NVPTXISD::Tex3DS32FloatLevel"; 766 case NVPTXISD::Tex3DS32FloatGrad: 767 return "NVPTXISD::Tex3DS32FloatGrad"; 768 case NVPTXISD::Tex3DU32S32: return "NVPTXISD::Tex3DU32S32"; 769 case NVPTXISD::Tex3DU32Float: return "NVPTXISD::Tex3DU32Float"; 770 case NVPTXISD::Tex3DU32FloatLevel: 771 return "NVPTXISD::Tex3DU32FloatLevel"; 772 case NVPTXISD::Tex3DU32FloatGrad: 773 return "NVPTXISD::Tex3DU32FloatGrad"; 774 case NVPTXISD::TexCubeFloatFloat: return "NVPTXISD::TexCubeFloatFloat"; 775 case NVPTXISD::TexCubeFloatFloatLevel: 776 return "NVPTXISD::TexCubeFloatFloatLevel"; 777 case NVPTXISD::TexCubeS32Float: return "NVPTXISD::TexCubeS32Float"; 778 case NVPTXISD::TexCubeS32FloatLevel: 779 return "NVPTXISD::TexCubeS32FloatLevel"; 780 case NVPTXISD::TexCubeU32Float: return "NVPTXISD::TexCubeU32Float"; 781 case NVPTXISD::TexCubeU32FloatLevel: 782 return "NVPTXISD::TexCubeU32FloatLevel"; 783 case NVPTXISD::TexCubeArrayFloatFloat: 784 return "NVPTXISD::TexCubeArrayFloatFloat"; 785 case NVPTXISD::TexCubeArrayFloatFloatLevel: 786 return "NVPTXISD::TexCubeArrayFloatFloatLevel"; 787 case NVPTXISD::TexCubeArrayS32Float: 788 return "NVPTXISD::TexCubeArrayS32Float"; 789 case NVPTXISD::TexCubeArrayS32FloatLevel: 790 return "NVPTXISD::TexCubeArrayS32FloatLevel"; 791 case NVPTXISD::TexCubeArrayU32Float: 792 return "NVPTXISD::TexCubeArrayU32Float"; 793 case NVPTXISD::TexCubeArrayU32FloatLevel: 794 return "NVPTXISD::TexCubeArrayU32FloatLevel"; 795 case NVPTXISD::Tld4R2DFloatFloat: 796 return "NVPTXISD::Tld4R2DFloatFloat"; 797 case NVPTXISD::Tld4G2DFloatFloat: 798 return "NVPTXISD::Tld4G2DFloatFloat"; 799 case NVPTXISD::Tld4B2DFloatFloat: 800 return "NVPTXISD::Tld4B2DFloatFloat"; 801 case NVPTXISD::Tld4A2DFloatFloat: 802 return "NVPTXISD::Tld4A2DFloatFloat"; 803 case NVPTXISD::Tld4R2DS64Float: 804 return "NVPTXISD::Tld4R2DS64Float"; 805 case NVPTXISD::Tld4G2DS64Float: 806 return "NVPTXISD::Tld4G2DS64Float"; 807 case NVPTXISD::Tld4B2DS64Float: 808 return "NVPTXISD::Tld4B2DS64Float"; 809 case NVPTXISD::Tld4A2DS64Float: 810 return "NVPTXISD::Tld4A2DS64Float"; 811 case NVPTXISD::Tld4R2DU64Float: 812 return "NVPTXISD::Tld4R2DU64Float"; 813 case NVPTXISD::Tld4G2DU64Float: 814 return "NVPTXISD::Tld4G2DU64Float"; 815 case NVPTXISD::Tld4B2DU64Float: 816 return "NVPTXISD::Tld4B2DU64Float"; 817 case NVPTXISD::Tld4A2DU64Float: 818 return "NVPTXISD::Tld4A2DU64Float"; 819 820 case NVPTXISD::TexUnified1DFloatS32: 821 return "NVPTXISD::TexUnified1DFloatS32"; 822 case NVPTXISD::TexUnified1DFloatFloat: 823 return "NVPTXISD::TexUnified1DFloatFloat"; 824 case NVPTXISD::TexUnified1DFloatFloatLevel: 825 return "NVPTXISD::TexUnified1DFloatFloatLevel"; 826 case NVPTXISD::TexUnified1DFloatFloatGrad: 827 return "NVPTXISD::TexUnified1DFloatFloatGrad"; 828 case NVPTXISD::TexUnified1DS32S32: 829 return "NVPTXISD::TexUnified1DS32S32"; 830 case NVPTXISD::TexUnified1DS32Float: 831 return "NVPTXISD::TexUnified1DS32Float"; 832 case NVPTXISD::TexUnified1DS32FloatLevel: 833 return "NVPTXISD::TexUnified1DS32FloatLevel"; 834 case NVPTXISD::TexUnified1DS32FloatGrad: 835 return "NVPTXISD::TexUnified1DS32FloatGrad"; 836 case NVPTXISD::TexUnified1DU32S32: 837 return "NVPTXISD::TexUnified1DU32S32"; 838 case NVPTXISD::TexUnified1DU32Float: 839 return "NVPTXISD::TexUnified1DU32Float"; 840 case NVPTXISD::TexUnified1DU32FloatLevel: 841 return "NVPTXISD::TexUnified1DU32FloatLevel"; 842 case NVPTXISD::TexUnified1DU32FloatGrad: 843 return "NVPTXISD::TexUnified1DU32FloatGrad"; 844 case NVPTXISD::TexUnified1DArrayFloatS32: 845 return "NVPTXISD::TexUnified1DArrayFloatS32"; 846 case NVPTXISD::TexUnified1DArrayFloatFloat: 847 return "NVPTXISD::TexUnified1DArrayFloatFloat"; 848 case NVPTXISD::TexUnified1DArrayFloatFloatLevel: 849 return "NVPTXISD::TexUnified1DArrayFloatFloatLevel"; 850 case NVPTXISD::TexUnified1DArrayFloatFloatGrad: 851 return "NVPTXISD::TexUnified1DArrayFloatFloatGrad"; 852 case NVPTXISD::TexUnified1DArrayS32S32: 853 return "NVPTXISD::TexUnified1DArrayS32S32"; 854 case NVPTXISD::TexUnified1DArrayS32Float: 855 return "NVPTXISD::TexUnified1DArrayS32Float"; 856 case NVPTXISD::TexUnified1DArrayS32FloatLevel: 857 return "NVPTXISD::TexUnified1DArrayS32FloatLevel"; 858 case NVPTXISD::TexUnified1DArrayS32FloatGrad: 859 return "NVPTXISD::TexUnified1DArrayS32FloatGrad"; 860 case NVPTXISD::TexUnified1DArrayU32S32: 861 return "NVPTXISD::TexUnified1DArrayU32S32"; 862 case NVPTXISD::TexUnified1DArrayU32Float: 863 return "NVPTXISD::TexUnified1DArrayU32Float"; 864 case NVPTXISD::TexUnified1DArrayU32FloatLevel: 865 return "NVPTXISD::TexUnified1DArrayU32FloatLevel"; 866 case NVPTXISD::TexUnified1DArrayU32FloatGrad: 867 return "NVPTXISD::TexUnified1DArrayU32FloatGrad"; 868 case NVPTXISD::TexUnified2DFloatS32: 869 return "NVPTXISD::TexUnified2DFloatS32"; 870 case NVPTXISD::TexUnified2DFloatFloat: 871 return "NVPTXISD::TexUnified2DFloatFloat"; 872 case NVPTXISD::TexUnified2DFloatFloatLevel: 873 return "NVPTXISD::TexUnified2DFloatFloatLevel"; 874 case NVPTXISD::TexUnified2DFloatFloatGrad: 875 return "NVPTXISD::TexUnified2DFloatFloatGrad"; 876 case NVPTXISD::TexUnified2DS32S32: 877 return "NVPTXISD::TexUnified2DS32S32"; 878 case NVPTXISD::TexUnified2DS32Float: 879 return "NVPTXISD::TexUnified2DS32Float"; 880 case NVPTXISD::TexUnified2DS32FloatLevel: 881 return "NVPTXISD::TexUnified2DS32FloatLevel"; 882 case NVPTXISD::TexUnified2DS32FloatGrad: 883 return "NVPTXISD::TexUnified2DS32FloatGrad"; 884 case NVPTXISD::TexUnified2DU32S32: 885 return "NVPTXISD::TexUnified2DU32S32"; 886 case NVPTXISD::TexUnified2DU32Float: 887 return "NVPTXISD::TexUnified2DU32Float"; 888 case NVPTXISD::TexUnified2DU32FloatLevel: 889 return "NVPTXISD::TexUnified2DU32FloatLevel"; 890 case NVPTXISD::TexUnified2DU32FloatGrad: 891 return "NVPTXISD::TexUnified2DU32FloatGrad"; 892 case NVPTXISD::TexUnified2DArrayFloatS32: 893 return "NVPTXISD::TexUnified2DArrayFloatS32"; 894 case NVPTXISD::TexUnified2DArrayFloatFloat: 895 return "NVPTXISD::TexUnified2DArrayFloatFloat"; 896 case NVPTXISD::TexUnified2DArrayFloatFloatLevel: 897 return "NVPTXISD::TexUnified2DArrayFloatFloatLevel"; 898 case NVPTXISD::TexUnified2DArrayFloatFloatGrad: 899 return "NVPTXISD::TexUnified2DArrayFloatFloatGrad"; 900 case NVPTXISD::TexUnified2DArrayS32S32: 901 return "NVPTXISD::TexUnified2DArrayS32S32"; 902 case NVPTXISD::TexUnified2DArrayS32Float: 903 return "NVPTXISD::TexUnified2DArrayS32Float"; 904 case NVPTXISD::TexUnified2DArrayS32FloatLevel: 905 return "NVPTXISD::TexUnified2DArrayS32FloatLevel"; 906 case NVPTXISD::TexUnified2DArrayS32FloatGrad: 907 return "NVPTXISD::TexUnified2DArrayS32FloatGrad"; 908 case NVPTXISD::TexUnified2DArrayU32S32: 909 return "NVPTXISD::TexUnified2DArrayU32S32"; 910 case NVPTXISD::TexUnified2DArrayU32Float: 911 return "NVPTXISD::TexUnified2DArrayU32Float"; 912 case NVPTXISD::TexUnified2DArrayU32FloatLevel: 913 return "NVPTXISD::TexUnified2DArrayU32FloatLevel"; 914 case NVPTXISD::TexUnified2DArrayU32FloatGrad: 915 return "NVPTXISD::TexUnified2DArrayU32FloatGrad"; 916 case NVPTXISD::TexUnified3DFloatS32: 917 return "NVPTXISD::TexUnified3DFloatS32"; 918 case NVPTXISD::TexUnified3DFloatFloat: 919 return "NVPTXISD::TexUnified3DFloatFloat"; 920 case NVPTXISD::TexUnified3DFloatFloatLevel: 921 return "NVPTXISD::TexUnified3DFloatFloatLevel"; 922 case NVPTXISD::TexUnified3DFloatFloatGrad: 923 return "NVPTXISD::TexUnified3DFloatFloatGrad"; 924 case NVPTXISD::TexUnified3DS32S32: 925 return "NVPTXISD::TexUnified3DS32S32"; 926 case NVPTXISD::TexUnified3DS32Float: 927 return "NVPTXISD::TexUnified3DS32Float"; 928 case NVPTXISD::TexUnified3DS32FloatLevel: 929 return "NVPTXISD::TexUnified3DS32FloatLevel"; 930 case NVPTXISD::TexUnified3DS32FloatGrad: 931 return "NVPTXISD::TexUnified3DS32FloatGrad"; 932 case NVPTXISD::TexUnified3DU32S32: 933 return "NVPTXISD::TexUnified3DU32S32"; 934 case NVPTXISD::TexUnified3DU32Float: 935 return "NVPTXISD::TexUnified3DU32Float"; 936 case NVPTXISD::TexUnified3DU32FloatLevel: 937 return "NVPTXISD::TexUnified3DU32FloatLevel"; 938 case NVPTXISD::TexUnified3DU32FloatGrad: 939 return "NVPTXISD::TexUnified3DU32FloatGrad"; 940 case NVPTXISD::TexUnifiedCubeFloatFloat: 941 return "NVPTXISD::TexUnifiedCubeFloatFloat"; 942 case NVPTXISD::TexUnifiedCubeFloatFloatLevel: 943 return "NVPTXISD::TexUnifiedCubeFloatFloatLevel"; 944 case NVPTXISD::TexUnifiedCubeS32Float: 945 return "NVPTXISD::TexUnifiedCubeS32Float"; 946 case NVPTXISD::TexUnifiedCubeS32FloatLevel: 947 return "NVPTXISD::TexUnifiedCubeS32FloatLevel"; 948 case NVPTXISD::TexUnifiedCubeU32Float: 949 return "NVPTXISD::TexUnifiedCubeU32Float"; 950 case NVPTXISD::TexUnifiedCubeU32FloatLevel: 951 return "NVPTXISD::TexUnifiedCubeU32FloatLevel"; 952 case NVPTXISD::TexUnifiedCubeArrayFloatFloat: 953 return "NVPTXISD::TexUnifiedCubeArrayFloatFloat"; 954 case NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel: 955 return "NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel"; 956 case NVPTXISD::TexUnifiedCubeArrayS32Float: 957 return "NVPTXISD::TexUnifiedCubeArrayS32Float"; 958 case NVPTXISD::TexUnifiedCubeArrayS32FloatLevel: 959 return "NVPTXISD::TexUnifiedCubeArrayS32FloatLevel"; 960 case NVPTXISD::TexUnifiedCubeArrayU32Float: 961 return "NVPTXISD::TexUnifiedCubeArrayU32Float"; 962 case NVPTXISD::TexUnifiedCubeArrayU32FloatLevel: 963 return "NVPTXISD::TexUnifiedCubeArrayU32FloatLevel"; 964 case NVPTXISD::Tld4UnifiedR2DFloatFloat: 965 return "NVPTXISD::Tld4UnifiedR2DFloatFloat"; 966 case NVPTXISD::Tld4UnifiedG2DFloatFloat: 967 return "NVPTXISD::Tld4UnifiedG2DFloatFloat"; 968 case NVPTXISD::Tld4UnifiedB2DFloatFloat: 969 return "NVPTXISD::Tld4UnifiedB2DFloatFloat"; 970 case NVPTXISD::Tld4UnifiedA2DFloatFloat: 971 return "NVPTXISD::Tld4UnifiedA2DFloatFloat"; 972 case NVPTXISD::Tld4UnifiedR2DS64Float: 973 return "NVPTXISD::Tld4UnifiedR2DS64Float"; 974 case NVPTXISD::Tld4UnifiedG2DS64Float: 975 return "NVPTXISD::Tld4UnifiedG2DS64Float"; 976 case NVPTXISD::Tld4UnifiedB2DS64Float: 977 return "NVPTXISD::Tld4UnifiedB2DS64Float"; 978 case NVPTXISD::Tld4UnifiedA2DS64Float: 979 return "NVPTXISD::Tld4UnifiedA2DS64Float"; 980 case NVPTXISD::Tld4UnifiedR2DU64Float: 981 return "NVPTXISD::Tld4UnifiedR2DU64Float"; 982 case NVPTXISD::Tld4UnifiedG2DU64Float: 983 return "NVPTXISD::Tld4UnifiedG2DU64Float"; 984 case NVPTXISD::Tld4UnifiedB2DU64Float: 985 return "NVPTXISD::Tld4UnifiedB2DU64Float"; 986 case NVPTXISD::Tld4UnifiedA2DU64Float: 987 return "NVPTXISD::Tld4UnifiedA2DU64Float"; 988 989 case NVPTXISD::Suld1DI8Clamp: return "NVPTXISD::Suld1DI8Clamp"; 990 case NVPTXISD::Suld1DI16Clamp: return "NVPTXISD::Suld1DI16Clamp"; 991 case NVPTXISD::Suld1DI32Clamp: return "NVPTXISD::Suld1DI32Clamp"; 992 case NVPTXISD::Suld1DI64Clamp: return "NVPTXISD::Suld1DI64Clamp"; 993 case NVPTXISD::Suld1DV2I8Clamp: return "NVPTXISD::Suld1DV2I8Clamp"; 994 case NVPTXISD::Suld1DV2I16Clamp: return "NVPTXISD::Suld1DV2I16Clamp"; 995 case NVPTXISD::Suld1DV2I32Clamp: return "NVPTXISD::Suld1DV2I32Clamp"; 996 case NVPTXISD::Suld1DV2I64Clamp: return "NVPTXISD::Suld1DV2I64Clamp"; 997 case NVPTXISD::Suld1DV4I8Clamp: return "NVPTXISD::Suld1DV4I8Clamp"; 998 case NVPTXISD::Suld1DV4I16Clamp: return "NVPTXISD::Suld1DV4I16Clamp"; 999 case NVPTXISD::Suld1DV4I32Clamp: return "NVPTXISD::Suld1DV4I32Clamp"; 1000 1001 case NVPTXISD::Suld1DArrayI8Clamp: return "NVPTXISD::Suld1DArrayI8Clamp"; 1002 case NVPTXISD::Suld1DArrayI16Clamp: return "NVPTXISD::Suld1DArrayI16Clamp"; 1003 case NVPTXISD::Suld1DArrayI32Clamp: return "NVPTXISD::Suld1DArrayI32Clamp"; 1004 case NVPTXISD::Suld1DArrayI64Clamp: return "NVPTXISD::Suld1DArrayI64Clamp"; 1005 case NVPTXISD::Suld1DArrayV2I8Clamp: return "NVPTXISD::Suld1DArrayV2I8Clamp"; 1006 case NVPTXISD::Suld1DArrayV2I16Clamp:return "NVPTXISD::Suld1DArrayV2I16Clamp"; 1007 case NVPTXISD::Suld1DArrayV2I32Clamp:return "NVPTXISD::Suld1DArrayV2I32Clamp"; 1008 case NVPTXISD::Suld1DArrayV2I64Clamp:return "NVPTXISD::Suld1DArrayV2I64Clamp"; 1009 case NVPTXISD::Suld1DArrayV4I8Clamp: return "NVPTXISD::Suld1DArrayV4I8Clamp"; 1010 case NVPTXISD::Suld1DArrayV4I16Clamp:return "NVPTXISD::Suld1DArrayV4I16Clamp"; 1011 case NVPTXISD::Suld1DArrayV4I32Clamp:return "NVPTXISD::Suld1DArrayV4I32Clamp"; 1012 1013 case NVPTXISD::Suld2DI8Clamp: return "NVPTXISD::Suld2DI8Clamp"; 1014 case NVPTXISD::Suld2DI16Clamp: return "NVPTXISD::Suld2DI16Clamp"; 1015 case NVPTXISD::Suld2DI32Clamp: return "NVPTXISD::Suld2DI32Clamp"; 1016 case NVPTXISD::Suld2DI64Clamp: return "NVPTXISD::Suld2DI64Clamp"; 1017 case NVPTXISD::Suld2DV2I8Clamp: return "NVPTXISD::Suld2DV2I8Clamp"; 1018 case NVPTXISD::Suld2DV2I16Clamp: return "NVPTXISD::Suld2DV2I16Clamp"; 1019 case NVPTXISD::Suld2DV2I32Clamp: return "NVPTXISD::Suld2DV2I32Clamp"; 1020 case NVPTXISD::Suld2DV2I64Clamp: return "NVPTXISD::Suld2DV2I64Clamp"; 1021 case NVPTXISD::Suld2DV4I8Clamp: return "NVPTXISD::Suld2DV4I8Clamp"; 1022 case NVPTXISD::Suld2DV4I16Clamp: return "NVPTXISD::Suld2DV4I16Clamp"; 1023 case NVPTXISD::Suld2DV4I32Clamp: return "NVPTXISD::Suld2DV4I32Clamp"; 1024 1025 case NVPTXISD::Suld2DArrayI8Clamp: return "NVPTXISD::Suld2DArrayI8Clamp"; 1026 case NVPTXISD::Suld2DArrayI16Clamp: return "NVPTXISD::Suld2DArrayI16Clamp"; 1027 case NVPTXISD::Suld2DArrayI32Clamp: return "NVPTXISD::Suld2DArrayI32Clamp"; 1028 case NVPTXISD::Suld2DArrayI64Clamp: return "NVPTXISD::Suld2DArrayI64Clamp"; 1029 case NVPTXISD::Suld2DArrayV2I8Clamp: return "NVPTXISD::Suld2DArrayV2I8Clamp"; 1030 case NVPTXISD::Suld2DArrayV2I16Clamp:return "NVPTXISD::Suld2DArrayV2I16Clamp"; 1031 case NVPTXISD::Suld2DArrayV2I32Clamp:return "NVPTXISD::Suld2DArrayV2I32Clamp"; 1032 case NVPTXISD::Suld2DArrayV2I64Clamp:return "NVPTXISD::Suld2DArrayV2I64Clamp"; 1033 case NVPTXISD::Suld2DArrayV4I8Clamp: return "NVPTXISD::Suld2DArrayV4I8Clamp"; 1034 case NVPTXISD::Suld2DArrayV4I16Clamp:return "NVPTXISD::Suld2DArrayV4I16Clamp"; 1035 case NVPTXISD::Suld2DArrayV4I32Clamp:return "NVPTXISD::Suld2DArrayV4I32Clamp"; 1036 1037 case NVPTXISD::Suld3DI8Clamp: return "NVPTXISD::Suld3DI8Clamp"; 1038 case NVPTXISD::Suld3DI16Clamp: return "NVPTXISD::Suld3DI16Clamp"; 1039 case NVPTXISD::Suld3DI32Clamp: return "NVPTXISD::Suld3DI32Clamp"; 1040 case NVPTXISD::Suld3DI64Clamp: return "NVPTXISD::Suld3DI64Clamp"; 1041 case NVPTXISD::Suld3DV2I8Clamp: return "NVPTXISD::Suld3DV2I8Clamp"; 1042 case NVPTXISD::Suld3DV2I16Clamp: return "NVPTXISD::Suld3DV2I16Clamp"; 1043 case NVPTXISD::Suld3DV2I32Clamp: return "NVPTXISD::Suld3DV2I32Clamp"; 1044 case NVPTXISD::Suld3DV2I64Clamp: return "NVPTXISD::Suld3DV2I64Clamp"; 1045 case NVPTXISD::Suld3DV4I8Clamp: return "NVPTXISD::Suld3DV4I8Clamp"; 1046 case NVPTXISD::Suld3DV4I16Clamp: return "NVPTXISD::Suld3DV4I16Clamp"; 1047 case NVPTXISD::Suld3DV4I32Clamp: return "NVPTXISD::Suld3DV4I32Clamp"; 1048 1049 case NVPTXISD::Suld1DI8Trap: return "NVPTXISD::Suld1DI8Trap"; 1050 case NVPTXISD::Suld1DI16Trap: return "NVPTXISD::Suld1DI16Trap"; 1051 case NVPTXISD::Suld1DI32Trap: return "NVPTXISD::Suld1DI32Trap"; 1052 case NVPTXISD::Suld1DI64Trap: return "NVPTXISD::Suld1DI64Trap"; 1053 case NVPTXISD::Suld1DV2I8Trap: return "NVPTXISD::Suld1DV2I8Trap"; 1054 case NVPTXISD::Suld1DV2I16Trap: return "NVPTXISD::Suld1DV2I16Trap"; 1055 case NVPTXISD::Suld1DV2I32Trap: return "NVPTXISD::Suld1DV2I32Trap"; 1056 case NVPTXISD::Suld1DV2I64Trap: return "NVPTXISD::Suld1DV2I64Trap"; 1057 case NVPTXISD::Suld1DV4I8Trap: return "NVPTXISD::Suld1DV4I8Trap"; 1058 case NVPTXISD::Suld1DV4I16Trap: return "NVPTXISD::Suld1DV4I16Trap"; 1059 case NVPTXISD::Suld1DV4I32Trap: return "NVPTXISD::Suld1DV4I32Trap"; 1060 1061 case NVPTXISD::Suld1DArrayI8Trap: return "NVPTXISD::Suld1DArrayI8Trap"; 1062 case NVPTXISD::Suld1DArrayI16Trap: return "NVPTXISD::Suld1DArrayI16Trap"; 1063 case NVPTXISD::Suld1DArrayI32Trap: return "NVPTXISD::Suld1DArrayI32Trap"; 1064 case NVPTXISD::Suld1DArrayI64Trap: return "NVPTXISD::Suld1DArrayI64Trap"; 1065 case NVPTXISD::Suld1DArrayV2I8Trap: return "NVPTXISD::Suld1DArrayV2I8Trap"; 1066 case NVPTXISD::Suld1DArrayV2I16Trap: return "NVPTXISD::Suld1DArrayV2I16Trap"; 1067 case NVPTXISD::Suld1DArrayV2I32Trap: return "NVPTXISD::Suld1DArrayV2I32Trap"; 1068 case NVPTXISD::Suld1DArrayV2I64Trap: return "NVPTXISD::Suld1DArrayV2I64Trap"; 1069 case NVPTXISD::Suld1DArrayV4I8Trap: return "NVPTXISD::Suld1DArrayV4I8Trap"; 1070 case NVPTXISD::Suld1DArrayV4I16Trap: return "NVPTXISD::Suld1DArrayV4I16Trap"; 1071 case NVPTXISD::Suld1DArrayV4I32Trap: return "NVPTXISD::Suld1DArrayV4I32Trap"; 1072 1073 case NVPTXISD::Suld2DI8Trap: return "NVPTXISD::Suld2DI8Trap"; 1074 case NVPTXISD::Suld2DI16Trap: return "NVPTXISD::Suld2DI16Trap"; 1075 case NVPTXISD::Suld2DI32Trap: return "NVPTXISD::Suld2DI32Trap"; 1076 case NVPTXISD::Suld2DI64Trap: return "NVPTXISD::Suld2DI64Trap"; 1077 case NVPTXISD::Suld2DV2I8Trap: return "NVPTXISD::Suld2DV2I8Trap"; 1078 case NVPTXISD::Suld2DV2I16Trap: return "NVPTXISD::Suld2DV2I16Trap"; 1079 case NVPTXISD::Suld2DV2I32Trap: return "NVPTXISD::Suld2DV2I32Trap"; 1080 case NVPTXISD::Suld2DV2I64Trap: return "NVPTXISD::Suld2DV2I64Trap"; 1081 case NVPTXISD::Suld2DV4I8Trap: return "NVPTXISD::Suld2DV4I8Trap"; 1082 case NVPTXISD::Suld2DV4I16Trap: return "NVPTXISD::Suld2DV4I16Trap"; 1083 case NVPTXISD::Suld2DV4I32Trap: return "NVPTXISD::Suld2DV4I32Trap"; 1084 1085 case NVPTXISD::Suld2DArrayI8Trap: return "NVPTXISD::Suld2DArrayI8Trap"; 1086 case NVPTXISD::Suld2DArrayI16Trap: return "NVPTXISD::Suld2DArrayI16Trap"; 1087 case NVPTXISD::Suld2DArrayI32Trap: return "NVPTXISD::Suld2DArrayI32Trap"; 1088 case NVPTXISD::Suld2DArrayI64Trap: return "NVPTXISD::Suld2DArrayI64Trap"; 1089 case NVPTXISD::Suld2DArrayV2I8Trap: return "NVPTXISD::Suld2DArrayV2I8Trap"; 1090 case NVPTXISD::Suld2DArrayV2I16Trap: return "NVPTXISD::Suld2DArrayV2I16Trap"; 1091 case NVPTXISD::Suld2DArrayV2I32Trap: return "NVPTXISD::Suld2DArrayV2I32Trap"; 1092 case NVPTXISD::Suld2DArrayV2I64Trap: return "NVPTXISD::Suld2DArrayV2I64Trap"; 1093 case NVPTXISD::Suld2DArrayV4I8Trap: return "NVPTXISD::Suld2DArrayV4I8Trap"; 1094 case NVPTXISD::Suld2DArrayV4I16Trap: return "NVPTXISD::Suld2DArrayV4I16Trap"; 1095 case NVPTXISD::Suld2DArrayV4I32Trap: return "NVPTXISD::Suld2DArrayV4I32Trap"; 1096 1097 case NVPTXISD::Suld3DI8Trap: return "NVPTXISD::Suld3DI8Trap"; 1098 case NVPTXISD::Suld3DI16Trap: return "NVPTXISD::Suld3DI16Trap"; 1099 case NVPTXISD::Suld3DI32Trap: return "NVPTXISD::Suld3DI32Trap"; 1100 case NVPTXISD::Suld3DI64Trap: return "NVPTXISD::Suld3DI64Trap"; 1101 case NVPTXISD::Suld3DV2I8Trap: return "NVPTXISD::Suld3DV2I8Trap"; 1102 case NVPTXISD::Suld3DV2I16Trap: return "NVPTXISD::Suld3DV2I16Trap"; 1103 case NVPTXISD::Suld3DV2I32Trap: return "NVPTXISD::Suld3DV2I32Trap"; 1104 case NVPTXISD::Suld3DV2I64Trap: return "NVPTXISD::Suld3DV2I64Trap"; 1105 case NVPTXISD::Suld3DV4I8Trap: return "NVPTXISD::Suld3DV4I8Trap"; 1106 case NVPTXISD::Suld3DV4I16Trap: return "NVPTXISD::Suld3DV4I16Trap"; 1107 case NVPTXISD::Suld3DV4I32Trap: return "NVPTXISD::Suld3DV4I32Trap"; 1108 1109 case NVPTXISD::Suld1DI8Zero: return "NVPTXISD::Suld1DI8Zero"; 1110 case NVPTXISD::Suld1DI16Zero: return "NVPTXISD::Suld1DI16Zero"; 1111 case NVPTXISD::Suld1DI32Zero: return "NVPTXISD::Suld1DI32Zero"; 1112 case NVPTXISD::Suld1DI64Zero: return "NVPTXISD::Suld1DI64Zero"; 1113 case NVPTXISD::Suld1DV2I8Zero: return "NVPTXISD::Suld1DV2I8Zero"; 1114 case NVPTXISD::Suld1DV2I16Zero: return "NVPTXISD::Suld1DV2I16Zero"; 1115 case NVPTXISD::Suld1DV2I32Zero: return "NVPTXISD::Suld1DV2I32Zero"; 1116 case NVPTXISD::Suld1DV2I64Zero: return "NVPTXISD::Suld1DV2I64Zero"; 1117 case NVPTXISD::Suld1DV4I8Zero: return "NVPTXISD::Suld1DV4I8Zero"; 1118 case NVPTXISD::Suld1DV4I16Zero: return "NVPTXISD::Suld1DV4I16Zero"; 1119 case NVPTXISD::Suld1DV4I32Zero: return "NVPTXISD::Suld1DV4I32Zero"; 1120 1121 case NVPTXISD::Suld1DArrayI8Zero: return "NVPTXISD::Suld1DArrayI8Zero"; 1122 case NVPTXISD::Suld1DArrayI16Zero: return "NVPTXISD::Suld1DArrayI16Zero"; 1123 case NVPTXISD::Suld1DArrayI32Zero: return "NVPTXISD::Suld1DArrayI32Zero"; 1124 case NVPTXISD::Suld1DArrayI64Zero: return "NVPTXISD::Suld1DArrayI64Zero"; 1125 case NVPTXISD::Suld1DArrayV2I8Zero: return "NVPTXISD::Suld1DArrayV2I8Zero"; 1126 case NVPTXISD::Suld1DArrayV2I16Zero: return "NVPTXISD::Suld1DArrayV2I16Zero"; 1127 case NVPTXISD::Suld1DArrayV2I32Zero: return "NVPTXISD::Suld1DArrayV2I32Zero"; 1128 case NVPTXISD::Suld1DArrayV2I64Zero: return "NVPTXISD::Suld1DArrayV2I64Zero"; 1129 case NVPTXISD::Suld1DArrayV4I8Zero: return "NVPTXISD::Suld1DArrayV4I8Zero"; 1130 case NVPTXISD::Suld1DArrayV4I16Zero: return "NVPTXISD::Suld1DArrayV4I16Zero"; 1131 case NVPTXISD::Suld1DArrayV4I32Zero: return "NVPTXISD::Suld1DArrayV4I32Zero"; 1132 1133 case NVPTXISD::Suld2DI8Zero: return "NVPTXISD::Suld2DI8Zero"; 1134 case NVPTXISD::Suld2DI16Zero: return "NVPTXISD::Suld2DI16Zero"; 1135 case NVPTXISD::Suld2DI32Zero: return "NVPTXISD::Suld2DI32Zero"; 1136 case NVPTXISD::Suld2DI64Zero: return "NVPTXISD::Suld2DI64Zero"; 1137 case NVPTXISD::Suld2DV2I8Zero: return "NVPTXISD::Suld2DV2I8Zero"; 1138 case NVPTXISD::Suld2DV2I16Zero: return "NVPTXISD::Suld2DV2I16Zero"; 1139 case NVPTXISD::Suld2DV2I32Zero: return "NVPTXISD::Suld2DV2I32Zero"; 1140 case NVPTXISD::Suld2DV2I64Zero: return "NVPTXISD::Suld2DV2I64Zero"; 1141 case NVPTXISD::Suld2DV4I8Zero: return "NVPTXISD::Suld2DV4I8Zero"; 1142 case NVPTXISD::Suld2DV4I16Zero: return "NVPTXISD::Suld2DV4I16Zero"; 1143 case NVPTXISD::Suld2DV4I32Zero: return "NVPTXISD::Suld2DV4I32Zero"; 1144 1145 case NVPTXISD::Suld2DArrayI8Zero: return "NVPTXISD::Suld2DArrayI8Zero"; 1146 case NVPTXISD::Suld2DArrayI16Zero: return "NVPTXISD::Suld2DArrayI16Zero"; 1147 case NVPTXISD::Suld2DArrayI32Zero: return "NVPTXISD::Suld2DArrayI32Zero"; 1148 case NVPTXISD::Suld2DArrayI64Zero: return "NVPTXISD::Suld2DArrayI64Zero"; 1149 case NVPTXISD::Suld2DArrayV2I8Zero: return "NVPTXISD::Suld2DArrayV2I8Zero"; 1150 case NVPTXISD::Suld2DArrayV2I16Zero: return "NVPTXISD::Suld2DArrayV2I16Zero"; 1151 case NVPTXISD::Suld2DArrayV2I32Zero: return "NVPTXISD::Suld2DArrayV2I32Zero"; 1152 case NVPTXISD::Suld2DArrayV2I64Zero: return "NVPTXISD::Suld2DArrayV2I64Zero"; 1153 case NVPTXISD::Suld2DArrayV4I8Zero: return "NVPTXISD::Suld2DArrayV4I8Zero"; 1154 case NVPTXISD::Suld2DArrayV4I16Zero: return "NVPTXISD::Suld2DArrayV4I16Zero"; 1155 case NVPTXISD::Suld2DArrayV4I32Zero: return "NVPTXISD::Suld2DArrayV4I32Zero"; 1156 1157 case NVPTXISD::Suld3DI8Zero: return "NVPTXISD::Suld3DI8Zero"; 1158 case NVPTXISD::Suld3DI16Zero: return "NVPTXISD::Suld3DI16Zero"; 1159 case NVPTXISD::Suld3DI32Zero: return "NVPTXISD::Suld3DI32Zero"; 1160 case NVPTXISD::Suld3DI64Zero: return "NVPTXISD::Suld3DI64Zero"; 1161 case NVPTXISD::Suld3DV2I8Zero: return "NVPTXISD::Suld3DV2I8Zero"; 1162 case NVPTXISD::Suld3DV2I16Zero: return "NVPTXISD::Suld3DV2I16Zero"; 1163 case NVPTXISD::Suld3DV2I32Zero: return "NVPTXISD::Suld3DV2I32Zero"; 1164 case NVPTXISD::Suld3DV2I64Zero: return "NVPTXISD::Suld3DV2I64Zero"; 1165 case NVPTXISD::Suld3DV4I8Zero: return "NVPTXISD::Suld3DV4I8Zero"; 1166 case NVPTXISD::Suld3DV4I16Zero: return "NVPTXISD::Suld3DV4I16Zero"; 1167 case NVPTXISD::Suld3DV4I32Zero: return "NVPTXISD::Suld3DV4I32Zero"; 1168 } 1169 return nullptr; 1170 } 1171 1172 TargetLoweringBase::LegalizeTypeAction 1173 NVPTXTargetLowering::getPreferredVectorAction(EVT VT) const { 1174 if (VT.getVectorNumElements() != 1 && VT.getScalarType() == MVT::i1) 1175 return TypeSplitVector; 1176 if (VT == MVT::v2f16) 1177 return TypeLegal; 1178 return TargetLoweringBase::getPreferredVectorAction(VT); 1179 } 1180 1181 SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, 1182 int Enabled, int &ExtraSteps, 1183 bool &UseOneConst, 1184 bool Reciprocal) const { 1185 if (!(Enabled == ReciprocalEstimate::Enabled || 1186 (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32()))) 1187 return SDValue(); 1188 1189 if (ExtraSteps == ReciprocalEstimate::Unspecified) 1190 ExtraSteps = 0; 1191 1192 SDLoc DL(Operand); 1193 EVT VT = Operand.getValueType(); 1194 bool Ftz = useF32FTZ(DAG.getMachineFunction()); 1195 1196 auto MakeIntrinsicCall = [&](Intrinsic::ID IID) { 1197 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, 1198 DAG.getConstant(IID, DL, MVT::i32), Operand); 1199 }; 1200 1201 // The sqrt and rsqrt refinement processes assume we always start out with an 1202 // approximation of the rsqrt. Therefore, if we're going to do any refinement 1203 // (i.e. ExtraSteps > 0), we must return an rsqrt. But if we're *not* doing 1204 // any refinement, we must return a regular sqrt. 1205 if (Reciprocal || ExtraSteps > 0) { 1206 if (VT == MVT::f32) 1207 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f 1208 : Intrinsic::nvvm_rsqrt_approx_f); 1209 else if (VT == MVT::f64) 1210 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d); 1211 else 1212 return SDValue(); 1213 } else { 1214 if (VT == MVT::f32) 1215 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f 1216 : Intrinsic::nvvm_sqrt_approx_f); 1217 else { 1218 // There's no sqrt.approx.f64 instruction, so we emit 1219 // reciprocal(rsqrt(x)). This is faster than 1220 // select(x == 0, 0, x * rsqrt(x)). (In fact, it's faster than plain 1221 // x * rsqrt(x).) 1222 return DAG.getNode( 1223 ISD::INTRINSIC_WO_CHAIN, DL, VT, 1224 DAG.getConstant(Intrinsic::nvvm_rcp_approx_ftz_d, DL, MVT::i32), 1225 MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d)); 1226 } 1227 } 1228 } 1229 1230 SDValue 1231 NVPTXTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { 1232 SDLoc dl(Op); 1233 const GlobalAddressSDNode *GAN = cast<GlobalAddressSDNode>(Op); 1234 auto PtrVT = getPointerTy(DAG.getDataLayout(), GAN->getAddressSpace()); 1235 Op = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, PtrVT); 1236 return DAG.getNode(NVPTXISD::Wrapper, dl, PtrVT, Op); 1237 } 1238 1239 std::string NVPTXTargetLowering::getPrototype( 1240 const DataLayout &DL, Type *retTy, const ArgListTy &Args, 1241 const SmallVectorImpl<ISD::OutputArg> &Outs, unsigned retAlignment, 1242 ImmutableCallSite CS) const { 1243 auto PtrVT = getPointerTy(DL); 1244 1245 bool isABI = (STI.getSmVersion() >= 20); 1246 assert(isABI && "Non-ABI compilation is not supported"); 1247 if (!isABI) 1248 return ""; 1249 1250 std::stringstream O; 1251 O << "prototype_" << uniqueCallSite << " : .callprototype "; 1252 1253 if (retTy->getTypeID() == Type::VoidTyID) { 1254 O << "()"; 1255 } else { 1256 O << "("; 1257 if (retTy->isFloatingPointTy() || (retTy->isIntegerTy() && !retTy->isIntegerTy(128))) { 1258 unsigned size = 0; 1259 if (auto *ITy = dyn_cast<IntegerType>(retTy)) { 1260 size = ITy->getBitWidth(); 1261 } else { 1262 assert(retTy->isFloatingPointTy() && 1263 "Floating point type expected here"); 1264 size = retTy->getPrimitiveSizeInBits(); 1265 } 1266 // PTX ABI requires all scalar return values to be at least 32 1267 // bits in size. fp16 normally uses .b16 as its storage type in 1268 // PTX, so its size must be adjusted here, too. 1269 if (size < 32) 1270 size = 32; 1271 1272 O << ".param .b" << size << " _"; 1273 } else if (isa<PointerType>(retTy)) { 1274 O << ".param .b" << PtrVT.getSizeInBits() << " _"; 1275 } else if (retTy->isAggregateType() || retTy->isVectorTy() || retTy->isIntegerTy(128)) { 1276 auto &DL = CS.getCalledFunction()->getParent()->getDataLayout(); 1277 O << ".param .align " << retAlignment << " .b8 _[" 1278 << DL.getTypeAllocSize(retTy) << "]"; 1279 } else { 1280 llvm_unreachable("Unknown return type"); 1281 } 1282 O << ") "; 1283 } 1284 O << "_ ("; 1285 1286 bool first = true; 1287 1288 unsigned OIdx = 0; 1289 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) { 1290 Type *Ty = Args[i].Ty; 1291 if (!first) { 1292 O << ", "; 1293 } 1294 first = false; 1295 1296 if (!Outs[OIdx].Flags.isByVal()) { 1297 if (Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128)) { 1298 unsigned align = 0; 1299 const CallInst *CallI = cast<CallInst>(CS.getInstruction()); 1300 // +1 because index 0 is reserved for return type alignment 1301 if (!getAlign(*CallI, i + 1, align)) 1302 align = DL.getABITypeAlignment(Ty); 1303 unsigned sz = DL.getTypeAllocSize(Ty); 1304 O << ".param .align " << align << " .b8 "; 1305 O << "_"; 1306 O << "[" << sz << "]"; 1307 // update the index for Outs 1308 SmallVector<EVT, 16> vtparts; 1309 ComputeValueVTs(*this, DL, Ty, vtparts); 1310 if (unsigned len = vtparts.size()) 1311 OIdx += len - 1; 1312 continue; 1313 } 1314 // i8 types in IR will be i16 types in SDAG 1315 assert((getValueType(DL, Ty) == Outs[OIdx].VT || 1316 (getValueType(DL, Ty) == MVT::i8 && Outs[OIdx].VT == MVT::i16)) && 1317 "type mismatch between callee prototype and arguments"); 1318 // scalar type 1319 unsigned sz = 0; 1320 if (isa<IntegerType>(Ty)) { 1321 sz = cast<IntegerType>(Ty)->getBitWidth(); 1322 if (sz < 32) 1323 sz = 32; 1324 } else if (isa<PointerType>(Ty)) { 1325 sz = PtrVT.getSizeInBits(); 1326 } else if (Ty->isHalfTy()) 1327 // PTX ABI requires all scalar parameters to be at least 32 1328 // bits in size. fp16 normally uses .b16 as its storage type 1329 // in PTX, so its size must be adjusted here, too. 1330 sz = 32; 1331 else 1332 sz = Ty->getPrimitiveSizeInBits(); 1333 O << ".param .b" << sz << " "; 1334 O << "_"; 1335 continue; 1336 } 1337 auto *PTy = dyn_cast<PointerType>(Ty); 1338 assert(PTy && "Param with byval attribute should be a pointer type"); 1339 Type *ETy = PTy->getElementType(); 1340 1341 unsigned align = Outs[OIdx].Flags.getByValAlign(); 1342 unsigned sz = DL.getTypeAllocSize(ETy); 1343 O << ".param .align " << align << " .b8 "; 1344 O << "_"; 1345 O << "[" << sz << "]"; 1346 } 1347 O << ");"; 1348 return O.str(); 1349 } 1350 1351 unsigned NVPTXTargetLowering::getArgumentAlignment(SDValue Callee, 1352 ImmutableCallSite CS, 1353 Type *Ty, unsigned Idx, 1354 const DataLayout &DL) const { 1355 if (!CS) { 1356 // CallSite is zero, fallback to ABI type alignment 1357 return DL.getABITypeAlignment(Ty); 1358 } 1359 1360 unsigned Align = 0; 1361 const Value *DirectCallee = CS.getCalledFunction(); 1362 1363 if (!DirectCallee) { 1364 // We don't have a direct function symbol, but that may be because of 1365 // constant cast instructions in the call. 1366 const Instruction *CalleeI = CS.getInstruction(); 1367 assert(CalleeI && "Call target is not a function or derived value?"); 1368 1369 // With bitcast'd call targets, the instruction will be the call 1370 if (isa<CallInst>(CalleeI)) { 1371 // Check if we have call alignment metadata 1372 if (getAlign(*cast<CallInst>(CalleeI), Idx, Align)) 1373 return Align; 1374 1375 const Value *CalleeV = cast<CallInst>(CalleeI)->getCalledValue(); 1376 // Ignore any bitcast instructions 1377 while (isa<ConstantExpr>(CalleeV)) { 1378 const ConstantExpr *CE = cast<ConstantExpr>(CalleeV); 1379 if (!CE->isCast()) 1380 break; 1381 // Look through the bitcast 1382 CalleeV = cast<ConstantExpr>(CalleeV)->getOperand(0); 1383 } 1384 1385 // We have now looked past all of the bitcasts. Do we finally have a 1386 // Function? 1387 if (isa<Function>(CalleeV)) 1388 DirectCallee = CalleeV; 1389 } 1390 } 1391 1392 // Check for function alignment information if we found that the 1393 // ultimate target is a Function 1394 if (DirectCallee) 1395 if (getAlign(*cast<Function>(DirectCallee), Idx, Align)) 1396 return Align; 1397 1398 // Call is indirect or alignment information is not available, fall back to 1399 // the ABI type alignment 1400 return DL.getABITypeAlignment(Ty); 1401 } 1402 1403 SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, 1404 SmallVectorImpl<SDValue> &InVals) const { 1405 SelectionDAG &DAG = CLI.DAG; 1406 SDLoc dl = CLI.DL; 1407 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; 1408 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; 1409 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; 1410 SDValue Chain = CLI.Chain; 1411 SDValue Callee = CLI.Callee; 1412 bool &isTailCall = CLI.IsTailCall; 1413 ArgListTy &Args = CLI.getArgs(); 1414 Type *RetTy = CLI.RetTy; 1415 ImmutableCallSite CS = CLI.CS; 1416 const DataLayout &DL = DAG.getDataLayout(); 1417 1418 bool isABI = (STI.getSmVersion() >= 20); 1419 assert(isABI && "Non-ABI compilation is not supported"); 1420 if (!isABI) 1421 return Chain; 1422 1423 SDValue tempChain = Chain; 1424 Chain = DAG.getCALLSEQ_START(Chain, uniqueCallSite, 0, dl); 1425 SDValue InFlag = Chain.getValue(1); 1426 1427 unsigned paramCount = 0; 1428 // Args.size() and Outs.size() need not match. 1429 // Outs.size() will be larger 1430 // * if there is an aggregate argument with multiple fields (each field 1431 // showing up separately in Outs) 1432 // * if there is a vector argument with more than typical vector-length 1433 // elements (generally if more than 4) where each vector element is 1434 // individually present in Outs. 1435 // So a different index should be used for indexing into Outs/OutVals. 1436 // See similar issue in LowerFormalArguments. 1437 unsigned OIdx = 0; 1438 // Declare the .params or .reg need to pass values 1439 // to the function 1440 for (unsigned i = 0, e = Args.size(); i != e; ++i, ++OIdx) { 1441 EVT VT = Outs[OIdx].VT; 1442 Type *Ty = Args[i].Ty; 1443 1444 if (!Outs[OIdx].Flags.isByVal()) { 1445 SmallVector<EVT, 16> VTs; 1446 SmallVector<uint64_t, 16> Offsets; 1447 ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets); 1448 unsigned ArgAlign = 1449 getArgumentAlignment(Callee, CS, Ty, paramCount + 1, DL); 1450 unsigned AllocSize = DL.getTypeAllocSize(Ty); 1451 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1452 bool NeedAlign; // Does argument declaration specify alignment? 1453 if (Ty->isAggregateType() || Ty->isVectorTy() || Ty->isIntegerTy(128)) { 1454 // declare .param .align <align> .b8 .param<n>[<size>]; 1455 SDValue DeclareParamOps[] = { 1456 Chain, DAG.getConstant(ArgAlign, dl, MVT::i32), 1457 DAG.getConstant(paramCount, dl, MVT::i32), 1458 DAG.getConstant(AllocSize, dl, MVT::i32), InFlag}; 1459 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs, 1460 DeclareParamOps); 1461 NeedAlign = true; 1462 } else { 1463 // declare .param .b<size> .param<n>; 1464 if ((VT.isInteger() || VT.isFloatingPoint()) && AllocSize < 4) { 1465 // PTX ABI requires integral types to be at least 32 bits in 1466 // size. FP16 is loaded/stored using i16, so it's handled 1467 // here as well. 1468 AllocSize = 4; 1469 } 1470 SDValue DeclareScalarParamOps[] = { 1471 Chain, DAG.getConstant(paramCount, dl, MVT::i32), 1472 DAG.getConstant(AllocSize * 8, dl, MVT::i32), 1473 DAG.getConstant(0, dl, MVT::i32), InFlag}; 1474 Chain = DAG.getNode(NVPTXISD::DeclareScalarParam, dl, DeclareParamVTs, 1475 DeclareScalarParamOps); 1476 NeedAlign = false; 1477 } 1478 InFlag = Chain.getValue(1); 1479 1480 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter 1481 // than 32-bits are sign extended or zero extended, depending on 1482 // whether they are signed or unsigned types. This case applies 1483 // only to scalar parameters and not to aggregate values. 1484 bool ExtendIntegerParam = 1485 Ty->isIntegerTy() && DL.getTypeAllocSizeInBits(Ty) < 32; 1486 1487 auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, ArgAlign); 1488 SmallVector<SDValue, 6> StoreOperands; 1489 for (unsigned j = 0, je = VTs.size(); j != je; ++j) { 1490 // New store. 1491 if (VectorInfo[j] & PVF_FIRST) { 1492 assert(StoreOperands.empty() && "Unfinished preceeding store."); 1493 StoreOperands.push_back(Chain); 1494 StoreOperands.push_back(DAG.getConstant(paramCount, dl, MVT::i32)); 1495 StoreOperands.push_back(DAG.getConstant(Offsets[j], dl, MVT::i32)); 1496 } 1497 1498 EVT EltVT = VTs[j]; 1499 SDValue StVal = OutVals[OIdx]; 1500 if (ExtendIntegerParam) { 1501 assert(VTs.size() == 1 && "Scalar can't have multiple parts."); 1502 // zext/sext to i32 1503 StVal = DAG.getNode(Outs[OIdx].Flags.isSExt() ? ISD::SIGN_EXTEND 1504 : ISD::ZERO_EXTEND, 1505 dl, MVT::i32, StVal); 1506 } else if (EltVT.getSizeInBits() < 16) { 1507 // Use 16-bit registers for small stores as it's the 1508 // smallest general purpose register size supported by NVPTX. 1509 StVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, StVal); 1510 } 1511 1512 // Record the value to store. 1513 StoreOperands.push_back(StVal); 1514 1515 if (VectorInfo[j] & PVF_LAST) { 1516 unsigned NumElts = StoreOperands.size() - 3; 1517 NVPTXISD::NodeType Op; 1518 switch (NumElts) { 1519 case 1: 1520 Op = NVPTXISD::StoreParam; 1521 break; 1522 case 2: 1523 Op = NVPTXISD::StoreParamV2; 1524 break; 1525 case 4: 1526 Op = NVPTXISD::StoreParamV4; 1527 break; 1528 default: 1529 llvm_unreachable("Invalid vector info."); 1530 } 1531 1532 StoreOperands.push_back(InFlag); 1533 1534 // Adjust type of the store op if we've extended the scalar 1535 // return value. 1536 EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : VTs[j]; 1537 unsigned EltAlign = 1538 NeedAlign ? GreatestCommonDivisor64(ArgAlign, Offsets[j]) : 0; 1539 1540 Chain = DAG.getMemIntrinsicNode( 1541 Op, dl, DAG.getVTList(MVT::Other, MVT::Glue), StoreOperands, 1542 TheStoreType, MachinePointerInfo(), EltAlign, 1543 MachineMemOperand::MOStore); 1544 InFlag = Chain.getValue(1); 1545 1546 // Cleanup. 1547 StoreOperands.clear(); 1548 } 1549 ++OIdx; 1550 } 1551 assert(StoreOperands.empty() && "Unfinished parameter store."); 1552 if (VTs.size() > 0) 1553 --OIdx; 1554 ++paramCount; 1555 continue; 1556 } 1557 1558 // ByVal arguments 1559 SmallVector<EVT, 16> VTs; 1560 SmallVector<uint64_t, 16> Offsets; 1561 auto *PTy = dyn_cast<PointerType>(Args[i].Ty); 1562 assert(PTy && "Type of a byval parameter should be pointer"); 1563 ComputePTXValueVTs(*this, DL, PTy->getElementType(), VTs, &Offsets, 0); 1564 1565 // declare .param .align <align> .b8 .param<n>[<size>]; 1566 unsigned sz = Outs[OIdx].Flags.getByValSize(); 1567 SDVTList DeclareParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1568 unsigned ArgAlign = Outs[OIdx].Flags.getByValAlign(); 1569 // The ByValAlign in the Outs[OIdx].Flags is alway set at this point, 1570 // so we don't need to worry about natural alignment or not. 1571 // See TargetLowering::LowerCallTo(). 1572 1573 // Enforce minumum alignment of 4 to work around ptxas miscompile 1574 // for sm_50+. See corresponding alignment adjustment in 1575 // emitFunctionParamList() for details. 1576 if (ArgAlign < 4) 1577 ArgAlign = 4; 1578 SDValue DeclareParamOps[] = {Chain, DAG.getConstant(ArgAlign, dl, MVT::i32), 1579 DAG.getConstant(paramCount, dl, MVT::i32), 1580 DAG.getConstant(sz, dl, MVT::i32), InFlag}; 1581 Chain = DAG.getNode(NVPTXISD::DeclareParam, dl, DeclareParamVTs, 1582 DeclareParamOps); 1583 InFlag = Chain.getValue(1); 1584 for (unsigned j = 0, je = VTs.size(); j != je; ++j) { 1585 EVT elemtype = VTs[j]; 1586 int curOffset = Offsets[j]; 1587 unsigned PartAlign = GreatestCommonDivisor64(ArgAlign, curOffset); 1588 auto PtrVT = getPointerTy(DL); 1589 SDValue srcAddr = DAG.getNode(ISD::ADD, dl, PtrVT, OutVals[OIdx], 1590 DAG.getConstant(curOffset, dl, PtrVT)); 1591 SDValue theVal = DAG.getLoad(elemtype, dl, tempChain, srcAddr, 1592 MachinePointerInfo(), PartAlign); 1593 if (elemtype.getSizeInBits() < 16) { 1594 theVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, theVal); 1595 } 1596 SDVTList CopyParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1597 SDValue CopyParamOps[] = { Chain, 1598 DAG.getConstant(paramCount, dl, MVT::i32), 1599 DAG.getConstant(curOffset, dl, MVT::i32), 1600 theVal, InFlag }; 1601 Chain = DAG.getMemIntrinsicNode(NVPTXISD::StoreParam, dl, CopyParamVTs, 1602 CopyParamOps, elemtype, 1603 MachinePointerInfo(), /* Align */ 0, 1604 MachineMemOperand::MOStore); 1605 1606 InFlag = Chain.getValue(1); 1607 } 1608 ++paramCount; 1609 } 1610 1611 GlobalAddressSDNode *Func = dyn_cast<GlobalAddressSDNode>(Callee.getNode()); 1612 unsigned retAlignment = 0; 1613 1614 // Handle Result 1615 if (Ins.size() > 0) { 1616 SmallVector<EVT, 16> resvtparts; 1617 ComputeValueVTs(*this, DL, RetTy, resvtparts); 1618 1619 // Declare 1620 // .param .align 16 .b8 retval0[<size-in-bytes>], or 1621 // .param .b<size-in-bits> retval0 1622 unsigned resultsz = DL.getTypeAllocSizeInBits(RetTy); 1623 // Emit ".param .b<size-in-bits> retval0" instead of byte arrays only for 1624 // these three types to match the logic in 1625 // NVPTXAsmPrinter::printReturnValStr and NVPTXTargetLowering::getPrototype. 1626 // Plus, this behavior is consistent with nvcc's. 1627 if (RetTy->isFloatingPointTy() || RetTy->isPointerTy() || 1628 (RetTy->isIntegerTy() && !RetTy->isIntegerTy(128))) { 1629 // Scalar needs to be at least 32bit wide 1630 if (resultsz < 32) 1631 resultsz = 32; 1632 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1633 SDValue DeclareRetOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32), 1634 DAG.getConstant(resultsz, dl, MVT::i32), 1635 DAG.getConstant(0, dl, MVT::i32), InFlag }; 1636 Chain = DAG.getNode(NVPTXISD::DeclareRet, dl, DeclareRetVTs, 1637 DeclareRetOps); 1638 InFlag = Chain.getValue(1); 1639 } else { 1640 retAlignment = getArgumentAlignment(Callee, CS, RetTy, 0, DL); 1641 SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1642 SDValue DeclareRetOps[] = { Chain, 1643 DAG.getConstant(retAlignment, dl, MVT::i32), 1644 DAG.getConstant(resultsz / 8, dl, MVT::i32), 1645 DAG.getConstant(0, dl, MVT::i32), InFlag }; 1646 Chain = DAG.getNode(NVPTXISD::DeclareRetParam, dl, DeclareRetVTs, 1647 DeclareRetOps); 1648 InFlag = Chain.getValue(1); 1649 } 1650 } 1651 1652 if (!Func) { 1653 // This is indirect function call case : PTX requires a prototype of the 1654 // form 1655 // proto_0 : .callprototype(.param .b32 _) _ (.param .b32 _); 1656 // to be emitted, and the label has to used as the last arg of call 1657 // instruction. 1658 // The prototype is embedded in a string and put as the operand for a 1659 // CallPrototype SDNode which will print out to the value of the string. 1660 SDVTList ProtoVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1661 std::string Proto = getPrototype(DL, RetTy, Args, Outs, retAlignment, CS); 1662 const char *ProtoStr = 1663 nvTM->getManagedStrPool()->getManagedString(Proto.c_str())->c_str(); 1664 SDValue ProtoOps[] = { 1665 Chain, DAG.getTargetExternalSymbol(ProtoStr, MVT::i32), InFlag, 1666 }; 1667 Chain = DAG.getNode(NVPTXISD::CallPrototype, dl, ProtoVTs, ProtoOps); 1668 InFlag = Chain.getValue(1); 1669 } 1670 // Op to just print "call" 1671 SDVTList PrintCallVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1672 SDValue PrintCallOps[] = { 1673 Chain, DAG.getConstant((Ins.size() == 0) ? 0 : 1, dl, MVT::i32), InFlag 1674 }; 1675 // We model convergent calls as separate opcodes. 1676 unsigned Opcode = Func ? NVPTXISD::PrintCallUni : NVPTXISD::PrintCall; 1677 if (CLI.IsConvergent) 1678 Opcode = Opcode == NVPTXISD::PrintCallUni ? NVPTXISD::PrintConvergentCallUni 1679 : NVPTXISD::PrintConvergentCall; 1680 Chain = DAG.getNode(Opcode, dl, PrintCallVTs, PrintCallOps); 1681 InFlag = Chain.getValue(1); 1682 1683 // Ops to print out the function name 1684 SDVTList CallVoidVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1685 SDValue CallVoidOps[] = { Chain, Callee, InFlag }; 1686 Chain = DAG.getNode(NVPTXISD::CallVoid, dl, CallVoidVTs, CallVoidOps); 1687 InFlag = Chain.getValue(1); 1688 1689 // Ops to print out the param list 1690 SDVTList CallArgBeginVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1691 SDValue CallArgBeginOps[] = { Chain, InFlag }; 1692 Chain = DAG.getNode(NVPTXISD::CallArgBegin, dl, CallArgBeginVTs, 1693 CallArgBeginOps); 1694 InFlag = Chain.getValue(1); 1695 1696 for (unsigned i = 0, e = paramCount; i != e; ++i) { 1697 unsigned opcode; 1698 if (i == (e - 1)) 1699 opcode = NVPTXISD::LastCallArg; 1700 else 1701 opcode = NVPTXISD::CallArg; 1702 SDVTList CallArgVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1703 SDValue CallArgOps[] = { Chain, DAG.getConstant(1, dl, MVT::i32), 1704 DAG.getConstant(i, dl, MVT::i32), InFlag }; 1705 Chain = DAG.getNode(opcode, dl, CallArgVTs, CallArgOps); 1706 InFlag = Chain.getValue(1); 1707 } 1708 SDVTList CallArgEndVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1709 SDValue CallArgEndOps[] = { Chain, 1710 DAG.getConstant(Func ? 1 : 0, dl, MVT::i32), 1711 InFlag }; 1712 Chain = DAG.getNode(NVPTXISD::CallArgEnd, dl, CallArgEndVTs, CallArgEndOps); 1713 InFlag = Chain.getValue(1); 1714 1715 if (!Func) { 1716 SDVTList PrototypeVTs = DAG.getVTList(MVT::Other, MVT::Glue); 1717 SDValue PrototypeOps[] = { Chain, 1718 DAG.getConstant(uniqueCallSite, dl, MVT::i32), 1719 InFlag }; 1720 Chain = DAG.getNode(NVPTXISD::Prototype, dl, PrototypeVTs, PrototypeOps); 1721 InFlag = Chain.getValue(1); 1722 } 1723 1724 // Generate loads from param memory/moves from registers for result 1725 if (Ins.size() > 0) { 1726 SmallVector<EVT, 16> VTs; 1727 SmallVector<uint64_t, 16> Offsets; 1728 ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets, 0); 1729 assert(VTs.size() == Ins.size() && "Bad value decomposition"); 1730 1731 unsigned RetAlign = getArgumentAlignment(Callee, CS, RetTy, 0, DL); 1732 auto VectorInfo = VectorizePTXValueVTs(VTs, Offsets, RetAlign); 1733 1734 SmallVector<EVT, 6> LoadVTs; 1735 int VecIdx = -1; // Index of the first element of the vector. 1736 1737 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than 1738 // 32-bits are sign extended or zero extended, depending on whether 1739 // they are signed or unsigned types. 1740 bool ExtendIntegerRetVal = 1741 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32; 1742 1743 for (unsigned i = 0, e = VTs.size(); i != e; ++i) { 1744 bool needTruncate = false; 1745 EVT TheLoadType = VTs[i]; 1746 EVT EltType = Ins[i].VT; 1747 unsigned EltAlign = GreatestCommonDivisor64(RetAlign, Offsets[i]); 1748 if (ExtendIntegerRetVal) { 1749 TheLoadType = MVT::i32; 1750 EltType = MVT::i32; 1751 needTruncate = true; 1752 } else if (TheLoadType.getSizeInBits() < 16) { 1753 if (VTs[i].isInteger()) 1754 needTruncate = true; 1755 EltType = MVT::i16; 1756 } 1757 1758 // Record index of the very first element of the vector. 1759 if (VectorInfo[i] & PVF_FIRST) { 1760 assert(VecIdx == -1 && LoadVTs.empty() && "Orphaned operand list."); 1761 VecIdx = i; 1762 } 1763 1764 LoadVTs.push_back(EltType); 1765 1766 if (VectorInfo[i] & PVF_LAST) { 1767 unsigned NumElts = LoadVTs.size(); 1768 LoadVTs.push_back(MVT::Other); 1769 LoadVTs.push_back(MVT::Glue); 1770 NVPTXISD::NodeType Op; 1771 switch (NumElts) { 1772 case 1: 1773 Op = NVPTXISD::LoadParam; 1774 break; 1775 case 2: 1776 Op = NVPTXISD::LoadParamV2; 1777 break; 1778 case 4: 1779 Op = NVPTXISD::LoadParamV4; 1780 break; 1781 default: 1782 llvm_unreachable("Invalid vector info."); 1783 } 1784 1785 SDValue LoadOperands[] = { 1786 Chain, DAG.getConstant(1, dl, MVT::i32), 1787 DAG.getConstant(Offsets[VecIdx], dl, MVT::i32), InFlag}; 1788 SDValue RetVal = DAG.getMemIntrinsicNode( 1789 Op, dl, DAG.getVTList(LoadVTs), LoadOperands, TheLoadType, 1790 MachinePointerInfo(), EltAlign, 1791 MachineMemOperand::MOLoad); 1792 1793 for (unsigned j = 0; j < NumElts; ++j) { 1794 SDValue Ret = RetVal.getValue(j); 1795 if (needTruncate) 1796 Ret = DAG.getNode(ISD::TRUNCATE, dl, Ins[VecIdx + j].VT, Ret); 1797 InVals.push_back(Ret); 1798 } 1799 Chain = RetVal.getValue(NumElts); 1800 InFlag = RetVal.getValue(NumElts + 1); 1801 1802 // Cleanup 1803 VecIdx = -1; 1804 LoadVTs.clear(); 1805 } 1806 } 1807 } 1808 1809 Chain = DAG.getCALLSEQ_END(Chain, 1810 DAG.getIntPtrConstant(uniqueCallSite, dl, true), 1811 DAG.getIntPtrConstant(uniqueCallSite + 1, dl, 1812 true), 1813 InFlag, dl); 1814 uniqueCallSite++; 1815 1816 // set isTailCall to false for now, until we figure out how to express 1817 // tail call optimization in PTX 1818 isTailCall = false; 1819 return Chain; 1820 } 1821 1822 // By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack() 1823 // (see LegalizeDAG.cpp). This is slow and uses local memory. 1824 // We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5 1825 SDValue 1826 NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const { 1827 SDNode *Node = Op.getNode(); 1828 SDLoc dl(Node); 1829 SmallVector<SDValue, 8> Ops; 1830 unsigned NumOperands = Node->getNumOperands(); 1831 for (unsigned i = 0; i < NumOperands; ++i) { 1832 SDValue SubOp = Node->getOperand(i); 1833 EVT VVT = SubOp.getNode()->getValueType(0); 1834 EVT EltVT = VVT.getVectorElementType(); 1835 unsigned NumSubElem = VVT.getVectorNumElements(); 1836 for (unsigned j = 0; j < NumSubElem; ++j) { 1837 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp, 1838 DAG.getIntPtrConstant(j, dl))); 1839 } 1840 } 1841 return DAG.getBuildVector(Node->getValueType(0), dl, Ops); 1842 } 1843 1844 // We can init constant f16x2 with a single .b32 move. Normally it 1845 // would get lowered as two constant loads and vector-packing move. 1846 // mov.b16 %h1, 0x4000; 1847 // mov.b16 %h2, 0x3C00; 1848 // mov.b32 %hh2, {%h2, %h1}; 1849 // Instead we want just a constant move: 1850 // mov.b32 %hh2, 0x40003C00 1851 // 1852 // This results in better SASS code with CUDA 7.x. Ptxas in CUDA 8.0 1853 // generates good SASS in both cases. 1854 SDValue NVPTXTargetLowering::LowerBUILD_VECTOR(SDValue Op, 1855 SelectionDAG &DAG) const { 1856 //return Op; 1857 if (!(Op->getValueType(0) == MVT::v2f16 && 1858 isa<ConstantFPSDNode>(Op->getOperand(0)) && 1859 isa<ConstantFPSDNode>(Op->getOperand(1)))) 1860 return Op; 1861 1862 APInt E0 = 1863 cast<ConstantFPSDNode>(Op->getOperand(0))->getValueAPF().bitcastToAPInt(); 1864 APInt E1 = 1865 cast<ConstantFPSDNode>(Op->getOperand(1))->getValueAPF().bitcastToAPInt(); 1866 SDValue Const = 1867 DAG.getConstant(E1.zext(32).shl(16) | E0.zext(32), SDLoc(Op), MVT::i32); 1868 return DAG.getNode(ISD::BITCAST, SDLoc(Op), MVT::v2f16, Const); 1869 } 1870 1871 SDValue NVPTXTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, 1872 SelectionDAG &DAG) const { 1873 SDValue Index = Op->getOperand(1); 1874 // Constant index will be matched by tablegen. 1875 if (isa<ConstantSDNode>(Index.getNode())) 1876 return Op; 1877 1878 // Extract individual elements and select one of them. 1879 SDValue Vector = Op->getOperand(0); 1880 EVT VectorVT = Vector.getValueType(); 1881 assert(VectorVT == MVT::v2f16 && "Unexpected vector type."); 1882 EVT EltVT = VectorVT.getVectorElementType(); 1883 1884 SDLoc dl(Op.getNode()); 1885 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector, 1886 DAG.getIntPtrConstant(0, dl)); 1887 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Vector, 1888 DAG.getIntPtrConstant(1, dl)); 1889 return DAG.getSelectCC(dl, Index, DAG.getIntPtrConstant(0, dl), E0, E1, 1890 ISD::CondCode::SETEQ); 1891 } 1892 1893 /// LowerShiftRightParts - Lower SRL_PARTS, SRA_PARTS, which 1894 /// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift 1895 /// amount, or 1896 /// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift 1897 /// amount. 1898 SDValue NVPTXTargetLowering::LowerShiftRightParts(SDValue Op, 1899 SelectionDAG &DAG) const { 1900 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 1901 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); 1902 1903 EVT VT = Op.getValueType(); 1904 unsigned VTBits = VT.getSizeInBits(); 1905 SDLoc dl(Op); 1906 SDValue ShOpLo = Op.getOperand(0); 1907 SDValue ShOpHi = Op.getOperand(1); 1908 SDValue ShAmt = Op.getOperand(2); 1909 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; 1910 1911 if (VTBits == 32 && STI.getSmVersion() >= 35) { 1912 // For 32bit and sm35, we can use the funnel shift 'shf' instruction. 1913 // {dHi, dLo} = {aHi, aLo} >> Amt 1914 // dHi = aHi >> Amt 1915 // dLo = shf.r.clamp aLo, aHi, Amt 1916 1917 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); 1918 SDValue Lo = DAG.getNode(NVPTXISD::FUN_SHFR_CLAMP, dl, VT, ShOpLo, ShOpHi, 1919 ShAmt); 1920 1921 SDValue Ops[2] = { Lo, Hi }; 1922 return DAG.getMergeValues(Ops, dl); 1923 } 1924 else { 1925 // {dHi, dLo} = {aHi, aLo} >> Amt 1926 // - if (Amt>=size) then 1927 // dLo = aHi >> (Amt-size) 1928 // dHi = aHi >> Amt (this is either all 0 or all 1) 1929 // else 1930 // dLo = (aLo >>logic Amt) | (aHi << (size-Amt)) 1931 // dHi = aHi >> Amt 1932 1933 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, 1934 DAG.getConstant(VTBits, dl, MVT::i32), 1935 ShAmt); 1936 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); 1937 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, 1938 DAG.getConstant(VTBits, dl, MVT::i32)); 1939 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); 1940 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 1941 SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt); 1942 1943 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt, 1944 DAG.getConstant(VTBits, dl, MVT::i32), 1945 ISD::SETGE); 1946 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt); 1947 SDValue Lo = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal); 1948 1949 SDValue Ops[2] = { Lo, Hi }; 1950 return DAG.getMergeValues(Ops, dl); 1951 } 1952 } 1953 1954 /// LowerShiftLeftParts - Lower SHL_PARTS, which 1955 /// 1) returns two i32 values and take a 2 x i32 value to shift plus a shift 1956 /// amount, or 1957 /// 2) returns two i64 values and take a 2 x i64 value to shift plus a shift 1958 /// amount. 1959 SDValue NVPTXTargetLowering::LowerShiftLeftParts(SDValue Op, 1960 SelectionDAG &DAG) const { 1961 assert(Op.getNumOperands() == 3 && "Not a double-shift!"); 1962 assert(Op.getOpcode() == ISD::SHL_PARTS); 1963 1964 EVT VT = Op.getValueType(); 1965 unsigned VTBits = VT.getSizeInBits(); 1966 SDLoc dl(Op); 1967 SDValue ShOpLo = Op.getOperand(0); 1968 SDValue ShOpHi = Op.getOperand(1); 1969 SDValue ShAmt = Op.getOperand(2); 1970 1971 if (VTBits == 32 && STI.getSmVersion() >= 35) { 1972 // For 32bit and sm35, we can use the funnel shift 'shf' instruction. 1973 // {dHi, dLo} = {aHi, aLo} << Amt 1974 // dHi = shf.l.clamp aLo, aHi, Amt 1975 // dLo = aLo << Amt 1976 1977 SDValue Hi = DAG.getNode(NVPTXISD::FUN_SHFL_CLAMP, dl, VT, ShOpLo, ShOpHi, 1978 ShAmt); 1979 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 1980 1981 SDValue Ops[2] = { Lo, Hi }; 1982 return DAG.getMergeValues(Ops, dl); 1983 } 1984 else { 1985 // {dHi, dLo} = {aHi, aLo} << Amt 1986 // - if (Amt>=size) then 1987 // dLo = aLo << Amt (all 0) 1988 // dLo = aLo << (Amt-size) 1989 // else 1990 // dLo = aLo << Amt 1991 // dHi = (aHi << Amt) | (aLo >> (size-Amt)) 1992 1993 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, 1994 DAG.getConstant(VTBits, dl, MVT::i32), 1995 ShAmt); 1996 SDValue Tmp1 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); 1997 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt, 1998 DAG.getConstant(VTBits, dl, MVT::i32)); 1999 SDValue Tmp2 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); 2000 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); 2001 SDValue TrueVal = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); 2002 2003 SDValue Cmp = DAG.getSetCC(dl, MVT::i1, ShAmt, 2004 DAG.getConstant(VTBits, dl, MVT::i32), 2005 ISD::SETGE); 2006 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); 2007 SDValue Hi = DAG.getNode(ISD::SELECT, dl, VT, Cmp, TrueVal, FalseVal); 2008 2009 SDValue Ops[2] = { Lo, Hi }; 2010 return DAG.getMergeValues(Ops, dl); 2011 } 2012 } 2013 2014 SDValue 2015 NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 2016 switch (Op.getOpcode()) { 2017 case ISD::RETURNADDR: 2018 return SDValue(); 2019 case ISD::FRAMEADDR: 2020 return SDValue(); 2021 case ISD::GlobalAddress: 2022 return LowerGlobalAddress(Op, DAG); 2023 case ISD::INTRINSIC_W_CHAIN: 2024 return Op; 2025 case ISD::BUILD_VECTOR: 2026 return LowerBUILD_VECTOR(Op, DAG); 2027 case ISD::EXTRACT_SUBVECTOR: 2028 return Op; 2029 case ISD::EXTRACT_VECTOR_ELT: 2030 return LowerEXTRACT_VECTOR_ELT(Op, DAG); 2031 case ISD::CONCAT_VECTORS: 2032 return LowerCONCAT_VECTORS(Op, DAG); 2033 case ISD::STORE: 2034 return LowerSTORE(Op, DAG); 2035 case ISD::LOAD: 2036 return LowerLOAD(Op, DAG); 2037 case ISD::SHL_PARTS: 2038 return LowerShiftLeftParts(Op, DAG); 2039 case ISD::SRA_PARTS: 2040 case ISD::SRL_PARTS: 2041 return LowerShiftRightParts(Op, DAG); 2042 case ISD::SELECT: 2043 return LowerSelect(Op, DAG); 2044 default: 2045 llvm_unreachable("Custom lowering not defined for operation"); 2046 } 2047 } 2048 2049 SDValue NVPTXTargetLowering::LowerSelect(SDValue Op, SelectionDAG &DAG) const { 2050 SDValue Op0 = Op->getOperand(0); 2051 SDValue Op1 = Op->getOperand(1); 2052 SDValue Op2 = Op->getOperand(2); 2053 SDLoc DL(Op.getNode()); 2054 2055 assert(Op.getValueType() == MVT::i1 && "Custom lowering enabled only for i1"); 2056 2057 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1); 2058 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2); 2059 SDValue Select = DAG.getNode(ISD::SELECT, DL, MVT::i32, Op0, Op1, Op2); 2060 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Select); 2061 2062 return Trunc; 2063 } 2064 2065 SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 2066 if (Op.getValueType() == MVT::i1) 2067 return LowerLOADi1(Op, DAG); 2068 2069 // v2f16 is legal, so we can't rely on legalizer to handle unaligned 2070 // loads and have to handle it here. 2071 if (Op.getValueType() == MVT::v2f16) { 2072 LoadSDNode *Load = cast<LoadSDNode>(Op); 2073 EVT MemVT = Load->getMemoryVT(); 2074 if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT, 2075 Load->getAddressSpace(), Load->getAlignment())) { 2076 SDValue Ops[2]; 2077 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG); 2078 return DAG.getMergeValues(Ops, SDLoc(Op)); 2079 } 2080 } 2081 2082 return SDValue(); 2083 } 2084 2085 // v = ld i1* addr 2086 // => 2087 // v1 = ld i8* addr (-> i16) 2088 // v = trunc i16 to i1 2089 SDValue NVPTXTargetLowering::LowerLOADi1(SDValue Op, SelectionDAG &DAG) const { 2090 SDNode *Node = Op.getNode(); 2091 LoadSDNode *LD = cast<LoadSDNode>(Node); 2092 SDLoc dl(Node); 2093 assert(LD->getExtensionType() == ISD::NON_EXTLOAD); 2094 assert(Node->getValueType(0) == MVT::i1 && 2095 "Custom lowering for i1 load only"); 2096 SDValue newLD = DAG.getLoad(MVT::i16, dl, LD->getChain(), LD->getBasePtr(), 2097 LD->getPointerInfo(), LD->getAlignment(), 2098 LD->getMemOperand()->getFlags()); 2099 SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD); 2100 // The legalizer (the caller) is expecting two values from the legalized 2101 // load, so we build a MergeValues node for it. See ExpandUnalignedLoad() 2102 // in LegalizeDAG.cpp which also uses MergeValues. 2103 SDValue Ops[] = { result, LD->getChain() }; 2104 return DAG.getMergeValues(Ops, dl); 2105 } 2106 2107 SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { 2108 StoreSDNode *Store = cast<StoreSDNode>(Op); 2109 EVT VT = Store->getMemoryVT(); 2110 2111 if (VT == MVT::i1) 2112 return LowerSTOREi1(Op, DAG); 2113 2114 // v2f16 is legal, so we can't rely on legalizer to handle unaligned 2115 // stores and have to handle it here. 2116 if (VT == MVT::v2f16 && 2117 !allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, 2118 Store->getAddressSpace(), Store->getAlignment())) 2119 return expandUnalignedStore(Store, DAG); 2120 2121 if (VT.isVector()) 2122 return LowerSTOREVector(Op, DAG); 2123 2124 return SDValue(); 2125 } 2126 2127 SDValue 2128 NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const { 2129 SDNode *N = Op.getNode(); 2130 SDValue Val = N->getOperand(1); 2131 SDLoc DL(N); 2132 EVT ValVT = Val.getValueType(); 2133 2134 if (ValVT.isVector()) { 2135 // We only handle "native" vector sizes for now, e.g. <4 x double> is not 2136 // legal. We can (and should) split that into 2 stores of <2 x double> here 2137 // but I'm leaving that as a TODO for now. 2138 if (!ValVT.isSimple()) 2139 return SDValue(); 2140 switch (ValVT.getSimpleVT().SimpleTy) { 2141 default: 2142 return SDValue(); 2143 case MVT::v2i8: 2144 case MVT::v2i16: 2145 case MVT::v2i32: 2146 case MVT::v2i64: 2147 case MVT::v2f16: 2148 case MVT::v2f32: 2149 case MVT::v2f64: 2150 case MVT::v4i8: 2151 case MVT::v4i16: 2152 case MVT::v4i32: 2153 case MVT::v4f16: 2154 case MVT::v4f32: 2155 case MVT::v8f16: // <4 x f16x2> 2156 // This is a "native" vector type 2157 break; 2158 } 2159 2160 MemSDNode *MemSD = cast<MemSDNode>(N); 2161 const DataLayout &TD = DAG.getDataLayout(); 2162 2163 unsigned Align = MemSD->getAlignment(); 2164 unsigned PrefAlign = 2165 TD.getPrefTypeAlignment(ValVT.getTypeForEVT(*DAG.getContext())); 2166 if (Align < PrefAlign) { 2167 // This store is not sufficiently aligned, so bail out and let this vector 2168 // store be scalarized. Note that we may still be able to emit smaller 2169 // vector stores. For example, if we are storing a <4 x float> with an 2170 // alignment of 8, this check will fail but the legalizer will try again 2171 // with 2 x <2 x float>, which will succeed with an alignment of 8. 2172 return SDValue(); 2173 } 2174 2175 unsigned Opcode = 0; 2176 EVT EltVT = ValVT.getVectorElementType(); 2177 unsigned NumElts = ValVT.getVectorNumElements(); 2178 2179 // Since StoreV2 is a target node, we cannot rely on DAG type legalization. 2180 // Therefore, we must ensure the type is legal. For i1 and i8, we set the 2181 // stored type to i16 and propagate the "real" type as the memory type. 2182 bool NeedExt = false; 2183 if (EltVT.getSizeInBits() < 16) 2184 NeedExt = true; 2185 2186 bool StoreF16x2 = false; 2187 switch (NumElts) { 2188 default: 2189 return SDValue(); 2190 case 2: 2191 Opcode = NVPTXISD::StoreV2; 2192 break; 2193 case 4: 2194 Opcode = NVPTXISD::StoreV4; 2195 break; 2196 case 8: 2197 // v8f16 is a special case. PTX doesn't have st.v8.f16 2198 // instruction. Instead, we split the vector into v2f16 chunks and 2199 // store them with st.v4.b32. 2200 assert(EltVT == MVT::f16 && "Wrong type for the vector."); 2201 Opcode = NVPTXISD::StoreV4; 2202 StoreF16x2 = true; 2203 break; 2204 } 2205 2206 SmallVector<SDValue, 8> Ops; 2207 2208 // First is the chain 2209 Ops.push_back(N->getOperand(0)); 2210 2211 if (StoreF16x2) { 2212 // Combine f16,f16 -> v2f16 2213 NumElts /= 2; 2214 for (unsigned i = 0; i < NumElts; ++i) { 2215 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Val, 2216 DAG.getIntPtrConstant(i * 2, DL)); 2217 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f16, Val, 2218 DAG.getIntPtrConstant(i * 2 + 1, DL)); 2219 SDValue V2 = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f16, E0, E1); 2220 Ops.push_back(V2); 2221 } 2222 } else { 2223 // Then the split values 2224 for (unsigned i = 0; i < NumElts; ++i) { 2225 SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val, 2226 DAG.getIntPtrConstant(i, DL)); 2227 if (NeedExt) 2228 ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i16, ExtVal); 2229 Ops.push_back(ExtVal); 2230 } 2231 } 2232 2233 // Then any remaining arguments 2234 Ops.append(N->op_begin() + 2, N->op_end()); 2235 2236 SDValue NewSt = 2237 DAG.getMemIntrinsicNode(Opcode, DL, DAG.getVTList(MVT::Other), Ops, 2238 MemSD->getMemoryVT(), MemSD->getMemOperand()); 2239 2240 // return DCI.CombineTo(N, NewSt, true); 2241 return NewSt; 2242 } 2243 2244 return SDValue(); 2245 } 2246 2247 // st i1 v, addr 2248 // => 2249 // v1 = zxt v to i16 2250 // st.u8 i16, addr 2251 SDValue NVPTXTargetLowering::LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const { 2252 SDNode *Node = Op.getNode(); 2253 SDLoc dl(Node); 2254 StoreSDNode *ST = cast<StoreSDNode>(Node); 2255 SDValue Tmp1 = ST->getChain(); 2256 SDValue Tmp2 = ST->getBasePtr(); 2257 SDValue Tmp3 = ST->getValue(); 2258 assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only"); 2259 Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Tmp3); 2260 SDValue Result = 2261 DAG.getTruncStore(Tmp1, dl, Tmp3, Tmp2, ST->getPointerInfo(), MVT::i8, 2262 ST->getAlignment(), ST->getMemOperand()->getFlags()); 2263 return Result; 2264 } 2265 2266 SDValue 2267 NVPTXTargetLowering::getParamSymbol(SelectionDAG &DAG, int idx, EVT v) const { 2268 std::string ParamSym; 2269 raw_string_ostream ParamStr(ParamSym); 2270 2271 ParamStr << DAG.getMachineFunction().getName() << "_param_" << idx; 2272 ParamStr.flush(); 2273 2274 std::string *SavedStr = 2275 nvTM->getManagedStrPool()->getManagedString(ParamSym.c_str()); 2276 return DAG.getTargetExternalSymbol(SavedStr->c_str(), v); 2277 } 2278 2279 // Check to see if the kernel argument is image*_t or sampler_t 2280 2281 static bool isImageOrSamplerVal(const Value *arg, const Module *context) { 2282 static const char *const specialTypes[] = { "struct._image2d_t", 2283 "struct._image3d_t", 2284 "struct._sampler_t" }; 2285 2286 Type *Ty = arg->getType(); 2287 auto *PTy = dyn_cast<PointerType>(Ty); 2288 2289 if (!PTy) 2290 return false; 2291 2292 if (!context) 2293 return false; 2294 2295 auto *STy = dyn_cast<StructType>(PTy->getElementType()); 2296 if (!STy || STy->isLiteral()) 2297 return false; 2298 2299 return std::find(std::begin(specialTypes), std::end(specialTypes), 2300 STy->getName()) != std::end(specialTypes); 2301 } 2302 2303 SDValue NVPTXTargetLowering::LowerFormalArguments( 2304 SDValue Chain, CallingConv::ID CallConv, bool isVarArg, 2305 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl, 2306 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { 2307 MachineFunction &MF = DAG.getMachineFunction(); 2308 const DataLayout &DL = DAG.getDataLayout(); 2309 auto PtrVT = getPointerTy(DAG.getDataLayout()); 2310 2311 const Function *F = &MF.getFunction(); 2312 const AttributeList &PAL = F->getAttributes(); 2313 const TargetLowering *TLI = STI.getTargetLowering(); 2314 2315 SDValue Root = DAG.getRoot(); 2316 std::vector<SDValue> OutChains; 2317 2318 bool isABI = (STI.getSmVersion() >= 20); 2319 assert(isABI && "Non-ABI compilation is not supported"); 2320 if (!isABI) 2321 return Chain; 2322 2323 std::vector<Type *> argTypes; 2324 std::vector<const Argument *> theArgs; 2325 for (const Argument &I : F->args()) { 2326 theArgs.push_back(&I); 2327 argTypes.push_back(I.getType()); 2328 } 2329 // argTypes.size() (or theArgs.size()) and Ins.size() need not match. 2330 // Ins.size() will be larger 2331 // * if there is an aggregate argument with multiple fields (each field 2332 // showing up separately in Ins) 2333 // * if there is a vector argument with more than typical vector-length 2334 // elements (generally if more than 4) where each vector element is 2335 // individually present in Ins. 2336 // So a different index should be used for indexing into Ins. 2337 // See similar issue in LowerCall. 2338 unsigned InsIdx = 0; 2339 2340 int idx = 0; 2341 for (unsigned i = 0, e = theArgs.size(); i != e; ++i, ++idx, ++InsIdx) { 2342 Type *Ty = argTypes[i]; 2343 2344 // If the kernel argument is image*_t or sampler_t, convert it to 2345 // a i32 constant holding the parameter position. This can later 2346 // matched in the AsmPrinter to output the correct mangled name. 2347 if (isImageOrSamplerVal( 2348 theArgs[i], 2349 (theArgs[i]->getParent() ? theArgs[i]->getParent()->getParent() 2350 : nullptr))) { 2351 assert(isKernelFunction(*F) && 2352 "Only kernels can have image/sampler params"); 2353 InVals.push_back(DAG.getConstant(i + 1, dl, MVT::i32)); 2354 continue; 2355 } 2356 2357 if (theArgs[i]->use_empty()) { 2358 // argument is dead 2359 if (Ty->isAggregateType() || Ty->isIntegerTy(128)) { 2360 SmallVector<EVT, 16> vtparts; 2361 2362 ComputePTXValueVTs(*this, DAG.getDataLayout(), Ty, vtparts); 2363 assert(vtparts.size() > 0 && "empty aggregate type not expected"); 2364 for (unsigned parti = 0, parte = vtparts.size(); parti != parte; 2365 ++parti) { 2366 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT)); 2367 ++InsIdx; 2368 } 2369 if (vtparts.size() > 0) 2370 --InsIdx; 2371 continue; 2372 } 2373 if (Ty->isVectorTy()) { 2374 EVT ObjectVT = getValueType(DL, Ty); 2375 unsigned NumRegs = TLI->getNumRegisters(F->getContext(), ObjectVT); 2376 for (unsigned parti = 0; parti < NumRegs; ++parti) { 2377 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT)); 2378 ++InsIdx; 2379 } 2380 if (NumRegs > 0) 2381 --InsIdx; 2382 continue; 2383 } 2384 InVals.push_back(DAG.getNode(ISD::UNDEF, dl, Ins[InsIdx].VT)); 2385 continue; 2386 } 2387 2388 // In the following cases, assign a node order of "idx+1" 2389 // to newly created nodes. The SDNodes for params have to 2390 // appear in the same order as their order of appearance 2391 // in the original function. "idx+1" holds that order. 2392 if (!PAL.hasParamAttribute(i, Attribute::ByVal)) { 2393 bool aggregateIsPacked = false; 2394 if (StructType *STy = dyn_cast<StructType>(Ty)) 2395 aggregateIsPacked = STy->isPacked(); 2396 2397 SmallVector<EVT, 16> VTs; 2398 SmallVector<uint64_t, 16> Offsets; 2399 ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets, 0); 2400 assert(VTs.size() > 0 && "Unexpected empty type."); 2401 auto VectorInfo = 2402 VectorizePTXValueVTs(VTs, Offsets, DL.getABITypeAlignment(Ty)); 2403 2404 SDValue Arg = getParamSymbol(DAG, idx, PtrVT); 2405 int VecIdx = -1; // Index of the first element of the current vector. 2406 for (unsigned parti = 0, parte = VTs.size(); parti != parte; ++parti) { 2407 if (VectorInfo[parti] & PVF_FIRST) { 2408 assert(VecIdx == -1 && "Orphaned vector."); 2409 VecIdx = parti; 2410 } 2411 2412 // That's the last element of this store op. 2413 if (VectorInfo[parti] & PVF_LAST) { 2414 unsigned NumElts = parti - VecIdx + 1; 2415 EVT EltVT = VTs[parti]; 2416 // i1 is loaded/stored as i8. 2417 EVT LoadVT = EltVT; 2418 if (EltVT == MVT::i1) 2419 LoadVT = MVT::i8; 2420 else if (EltVT == MVT::v2f16) 2421 // getLoad needs a vector type, but it can't handle 2422 // vectors which contain v2f16 elements. So we must load 2423 // using i32 here and then bitcast back. 2424 LoadVT = MVT::i32; 2425 2426 EVT VecVT = EVT::getVectorVT(F->getContext(), LoadVT, NumElts); 2427 SDValue VecAddr = 2428 DAG.getNode(ISD::ADD, dl, PtrVT, Arg, 2429 DAG.getConstant(Offsets[VecIdx], dl, PtrVT)); 2430 Value *srcValue = Constant::getNullValue(PointerType::get( 2431 EltVT.getTypeForEVT(F->getContext()), ADDRESS_SPACE_PARAM)); 2432 SDValue P = 2433 DAG.getLoad(VecVT, dl, Root, VecAddr, 2434 MachinePointerInfo(srcValue), aggregateIsPacked, 2435 MachineMemOperand::MODereferenceable | 2436 MachineMemOperand::MOInvariant); 2437 if (P.getNode()) 2438 P.getNode()->setIROrder(idx + 1); 2439 for (unsigned j = 0; j < NumElts; ++j) { 2440 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, LoadVT, P, 2441 DAG.getIntPtrConstant(j, dl)); 2442 // We've loaded i1 as an i8 and now must truncate it back to i1 2443 if (EltVT == MVT::i1) 2444 Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Elt); 2445 // v2f16 was loaded as an i32. Now we must bitcast it back. 2446 else if (EltVT == MVT::v2f16) 2447 Elt = DAG.getNode(ISD::BITCAST, dl, MVT::v2f16, Elt); 2448 // Extend the element if necessary (e.g. an i8 is loaded 2449 // into an i16 register) 2450 if (Ins[InsIdx].VT.isInteger() && 2451 Ins[InsIdx].VT.getSizeInBits() > LoadVT.getSizeInBits()) { 2452 unsigned Extend = Ins[InsIdx].Flags.isSExt() ? ISD::SIGN_EXTEND 2453 : ISD::ZERO_EXTEND; 2454 Elt = DAG.getNode(Extend, dl, Ins[InsIdx].VT, Elt); 2455 } 2456 InVals.push_back(Elt); 2457 } 2458 2459 // Reset vector tracking state. 2460 VecIdx = -1; 2461 } 2462 ++InsIdx; 2463 } 2464 if (VTs.size() > 0) 2465 --InsIdx; 2466 continue; 2467 } 2468 2469 // Param has ByVal attribute 2470 // Return MoveParam(param symbol). 2471 // Ideally, the param symbol can be returned directly, 2472 // but when SDNode builder decides to use it in a CopyToReg(), 2473 // machine instruction fails because TargetExternalSymbol 2474 // (not lowered) is target dependent, and CopyToReg assumes 2475 // the source is lowered. 2476 EVT ObjectVT = getValueType(DL, Ty); 2477 assert(ObjectVT == Ins[InsIdx].VT && 2478 "Ins type did not match function type"); 2479 SDValue Arg = getParamSymbol(DAG, idx, PtrVT); 2480 SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg); 2481 if (p.getNode()) 2482 p.getNode()->setIROrder(idx + 1); 2483 InVals.push_back(p); 2484 } 2485 2486 // Clang will check explicit VarArg and issue error if any. However, Clang 2487 // will let code with 2488 // implicit var arg like f() pass. See bug 617733. 2489 // We treat this case as if the arg list is empty. 2490 // if (F.isVarArg()) { 2491 // assert(0 && "VarArg not supported yet!"); 2492 //} 2493 2494 if (!OutChains.empty()) 2495 DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains)); 2496 2497 return Chain; 2498 } 2499 2500 SDValue 2501 NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, 2502 bool isVarArg, 2503 const SmallVectorImpl<ISD::OutputArg> &Outs, 2504 const SmallVectorImpl<SDValue> &OutVals, 2505 const SDLoc &dl, SelectionDAG &DAG) const { 2506 MachineFunction &MF = DAG.getMachineFunction(); 2507 Type *RetTy = MF.getFunction().getReturnType(); 2508 2509 bool isABI = (STI.getSmVersion() >= 20); 2510 assert(isABI && "Non-ABI compilation is not supported"); 2511 if (!isABI) 2512 return Chain; 2513 2514 const DataLayout DL = DAG.getDataLayout(); 2515 SmallVector<EVT, 16> VTs; 2516 SmallVector<uint64_t, 16> Offsets; 2517 ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets); 2518 assert(VTs.size() == OutVals.size() && "Bad return value decomposition"); 2519 2520 auto VectorInfo = VectorizePTXValueVTs( 2521 VTs, Offsets, RetTy->isSized() ? DL.getABITypeAlignment(RetTy) : 1); 2522 2523 // PTX Interoperability Guide 3.3(A): [Integer] Values shorter than 2524 // 32-bits are sign extended or zero extended, depending on whether 2525 // they are signed or unsigned types. 2526 bool ExtendIntegerRetVal = 2527 RetTy->isIntegerTy() && DL.getTypeAllocSizeInBits(RetTy) < 32; 2528 2529 SmallVector<SDValue, 6> StoreOperands; 2530 for (unsigned i = 0, e = VTs.size(); i != e; ++i) { 2531 // New load/store. Record chain and offset operands. 2532 if (VectorInfo[i] & PVF_FIRST) { 2533 assert(StoreOperands.empty() && "Orphaned operand list."); 2534 StoreOperands.push_back(Chain); 2535 StoreOperands.push_back(DAG.getConstant(Offsets[i], dl, MVT::i32)); 2536 } 2537 2538 SDValue RetVal = OutVals[i]; 2539 if (ExtendIntegerRetVal) { 2540 RetVal = DAG.getNode(Outs[i].Flags.isSExt() ? ISD::SIGN_EXTEND 2541 : ISD::ZERO_EXTEND, 2542 dl, MVT::i32, RetVal); 2543 } else if (RetVal.getValueSizeInBits() < 16) { 2544 // Use 16-bit registers for small load-stores as it's the 2545 // smallest general purpose register size supported by NVPTX. 2546 RetVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, RetVal); 2547 } 2548 2549 // Record the value to return. 2550 StoreOperands.push_back(RetVal); 2551 2552 // That's the last element of this store op. 2553 if (VectorInfo[i] & PVF_LAST) { 2554 NVPTXISD::NodeType Op; 2555 unsigned NumElts = StoreOperands.size() - 2; 2556 switch (NumElts) { 2557 case 1: 2558 Op = NVPTXISD::StoreRetval; 2559 break; 2560 case 2: 2561 Op = NVPTXISD::StoreRetvalV2; 2562 break; 2563 case 4: 2564 Op = NVPTXISD::StoreRetvalV4; 2565 break; 2566 default: 2567 llvm_unreachable("Invalid vector info."); 2568 } 2569 2570 // Adjust type of load/store op if we've extended the scalar 2571 // return value. 2572 EVT TheStoreType = ExtendIntegerRetVal ? MVT::i32 : VTs[i]; 2573 Chain = DAG.getMemIntrinsicNode(Op, dl, DAG.getVTList(MVT::Other), 2574 StoreOperands, TheStoreType, 2575 MachinePointerInfo(), /* Align */ 1, 2576 MachineMemOperand::MOStore); 2577 // Cleanup vector state. 2578 StoreOperands.clear(); 2579 } 2580 } 2581 2582 return DAG.getNode(NVPTXISD::RET_FLAG, dl, MVT::Other, Chain); 2583 } 2584 2585 void NVPTXTargetLowering::LowerAsmOperandForConstraint( 2586 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops, 2587 SelectionDAG &DAG) const { 2588 if (Constraint.length() > 1) 2589 return; 2590 else 2591 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); 2592 } 2593 2594 static unsigned getOpcForTextureInstr(unsigned Intrinsic) { 2595 switch (Intrinsic) { 2596 default: 2597 return 0; 2598 2599 case Intrinsic::nvvm_tex_1d_v4f32_s32: 2600 return NVPTXISD::Tex1DFloatS32; 2601 case Intrinsic::nvvm_tex_1d_v4f32_f32: 2602 return NVPTXISD::Tex1DFloatFloat; 2603 case Intrinsic::nvvm_tex_1d_level_v4f32_f32: 2604 return NVPTXISD::Tex1DFloatFloatLevel; 2605 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32: 2606 return NVPTXISD::Tex1DFloatFloatGrad; 2607 case Intrinsic::nvvm_tex_1d_v4s32_s32: 2608 return NVPTXISD::Tex1DS32S32; 2609 case Intrinsic::nvvm_tex_1d_v4s32_f32: 2610 return NVPTXISD::Tex1DS32Float; 2611 case Intrinsic::nvvm_tex_1d_level_v4s32_f32: 2612 return NVPTXISD::Tex1DS32FloatLevel; 2613 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32: 2614 return NVPTXISD::Tex1DS32FloatGrad; 2615 case Intrinsic::nvvm_tex_1d_v4u32_s32: 2616 return NVPTXISD::Tex1DU32S32; 2617 case Intrinsic::nvvm_tex_1d_v4u32_f32: 2618 return NVPTXISD::Tex1DU32Float; 2619 case Intrinsic::nvvm_tex_1d_level_v4u32_f32: 2620 return NVPTXISD::Tex1DU32FloatLevel; 2621 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32: 2622 return NVPTXISD::Tex1DU32FloatGrad; 2623 2624 case Intrinsic::nvvm_tex_1d_array_v4f32_s32: 2625 return NVPTXISD::Tex1DArrayFloatS32; 2626 case Intrinsic::nvvm_tex_1d_array_v4f32_f32: 2627 return NVPTXISD::Tex1DArrayFloatFloat; 2628 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32: 2629 return NVPTXISD::Tex1DArrayFloatFloatLevel; 2630 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32: 2631 return NVPTXISD::Tex1DArrayFloatFloatGrad; 2632 case Intrinsic::nvvm_tex_1d_array_v4s32_s32: 2633 return NVPTXISD::Tex1DArrayS32S32; 2634 case Intrinsic::nvvm_tex_1d_array_v4s32_f32: 2635 return NVPTXISD::Tex1DArrayS32Float; 2636 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32: 2637 return NVPTXISD::Tex1DArrayS32FloatLevel; 2638 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32: 2639 return NVPTXISD::Tex1DArrayS32FloatGrad; 2640 case Intrinsic::nvvm_tex_1d_array_v4u32_s32: 2641 return NVPTXISD::Tex1DArrayU32S32; 2642 case Intrinsic::nvvm_tex_1d_array_v4u32_f32: 2643 return NVPTXISD::Tex1DArrayU32Float; 2644 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32: 2645 return NVPTXISD::Tex1DArrayU32FloatLevel; 2646 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32: 2647 return NVPTXISD::Tex1DArrayU32FloatGrad; 2648 2649 case Intrinsic::nvvm_tex_2d_v4f32_s32: 2650 return NVPTXISD::Tex2DFloatS32; 2651 case Intrinsic::nvvm_tex_2d_v4f32_f32: 2652 return NVPTXISD::Tex2DFloatFloat; 2653 case Intrinsic::nvvm_tex_2d_level_v4f32_f32: 2654 return NVPTXISD::Tex2DFloatFloatLevel; 2655 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32: 2656 return NVPTXISD::Tex2DFloatFloatGrad; 2657 case Intrinsic::nvvm_tex_2d_v4s32_s32: 2658 return NVPTXISD::Tex2DS32S32; 2659 case Intrinsic::nvvm_tex_2d_v4s32_f32: 2660 return NVPTXISD::Tex2DS32Float; 2661 case Intrinsic::nvvm_tex_2d_level_v4s32_f32: 2662 return NVPTXISD::Tex2DS32FloatLevel; 2663 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32: 2664 return NVPTXISD::Tex2DS32FloatGrad; 2665 case Intrinsic::nvvm_tex_2d_v4u32_s32: 2666 return NVPTXISD::Tex2DU32S32; 2667 case Intrinsic::nvvm_tex_2d_v4u32_f32: 2668 return NVPTXISD::Tex2DU32Float; 2669 case Intrinsic::nvvm_tex_2d_level_v4u32_f32: 2670 return NVPTXISD::Tex2DU32FloatLevel; 2671 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32: 2672 return NVPTXISD::Tex2DU32FloatGrad; 2673 2674 case Intrinsic::nvvm_tex_2d_array_v4f32_s32: 2675 return NVPTXISD::Tex2DArrayFloatS32; 2676 case Intrinsic::nvvm_tex_2d_array_v4f32_f32: 2677 return NVPTXISD::Tex2DArrayFloatFloat; 2678 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32: 2679 return NVPTXISD::Tex2DArrayFloatFloatLevel; 2680 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32: 2681 return NVPTXISD::Tex2DArrayFloatFloatGrad; 2682 case Intrinsic::nvvm_tex_2d_array_v4s32_s32: 2683 return NVPTXISD::Tex2DArrayS32S32; 2684 case Intrinsic::nvvm_tex_2d_array_v4s32_f32: 2685 return NVPTXISD::Tex2DArrayS32Float; 2686 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32: 2687 return NVPTXISD::Tex2DArrayS32FloatLevel; 2688 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32: 2689 return NVPTXISD::Tex2DArrayS32FloatGrad; 2690 case Intrinsic::nvvm_tex_2d_array_v4u32_s32: 2691 return NVPTXISD::Tex2DArrayU32S32; 2692 case Intrinsic::nvvm_tex_2d_array_v4u32_f32: 2693 return NVPTXISD::Tex2DArrayU32Float; 2694 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32: 2695 return NVPTXISD::Tex2DArrayU32FloatLevel; 2696 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32: 2697 return NVPTXISD::Tex2DArrayU32FloatGrad; 2698 2699 case Intrinsic::nvvm_tex_3d_v4f32_s32: 2700 return NVPTXISD::Tex3DFloatS32; 2701 case Intrinsic::nvvm_tex_3d_v4f32_f32: 2702 return NVPTXISD::Tex3DFloatFloat; 2703 case Intrinsic::nvvm_tex_3d_level_v4f32_f32: 2704 return NVPTXISD::Tex3DFloatFloatLevel; 2705 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32: 2706 return NVPTXISD::Tex3DFloatFloatGrad; 2707 case Intrinsic::nvvm_tex_3d_v4s32_s32: 2708 return NVPTXISD::Tex3DS32S32; 2709 case Intrinsic::nvvm_tex_3d_v4s32_f32: 2710 return NVPTXISD::Tex3DS32Float; 2711 case Intrinsic::nvvm_tex_3d_level_v4s32_f32: 2712 return NVPTXISD::Tex3DS32FloatLevel; 2713 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32: 2714 return NVPTXISD::Tex3DS32FloatGrad; 2715 case Intrinsic::nvvm_tex_3d_v4u32_s32: 2716 return NVPTXISD::Tex3DU32S32; 2717 case Intrinsic::nvvm_tex_3d_v4u32_f32: 2718 return NVPTXISD::Tex3DU32Float; 2719 case Intrinsic::nvvm_tex_3d_level_v4u32_f32: 2720 return NVPTXISD::Tex3DU32FloatLevel; 2721 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32: 2722 return NVPTXISD::Tex3DU32FloatGrad; 2723 2724 case Intrinsic::nvvm_tex_cube_v4f32_f32: 2725 return NVPTXISD::TexCubeFloatFloat; 2726 case Intrinsic::nvvm_tex_cube_level_v4f32_f32: 2727 return NVPTXISD::TexCubeFloatFloatLevel; 2728 case Intrinsic::nvvm_tex_cube_v4s32_f32: 2729 return NVPTXISD::TexCubeS32Float; 2730 case Intrinsic::nvvm_tex_cube_level_v4s32_f32: 2731 return NVPTXISD::TexCubeS32FloatLevel; 2732 case Intrinsic::nvvm_tex_cube_v4u32_f32: 2733 return NVPTXISD::TexCubeU32Float; 2734 case Intrinsic::nvvm_tex_cube_level_v4u32_f32: 2735 return NVPTXISD::TexCubeU32FloatLevel; 2736 2737 case Intrinsic::nvvm_tex_cube_array_v4f32_f32: 2738 return NVPTXISD::TexCubeArrayFloatFloat; 2739 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32: 2740 return NVPTXISD::TexCubeArrayFloatFloatLevel; 2741 case Intrinsic::nvvm_tex_cube_array_v4s32_f32: 2742 return NVPTXISD::TexCubeArrayS32Float; 2743 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32: 2744 return NVPTXISD::TexCubeArrayS32FloatLevel; 2745 case Intrinsic::nvvm_tex_cube_array_v4u32_f32: 2746 return NVPTXISD::TexCubeArrayU32Float; 2747 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32: 2748 return NVPTXISD::TexCubeArrayU32FloatLevel; 2749 2750 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32: 2751 return NVPTXISD::Tld4R2DFloatFloat; 2752 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32: 2753 return NVPTXISD::Tld4G2DFloatFloat; 2754 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32: 2755 return NVPTXISD::Tld4B2DFloatFloat; 2756 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32: 2757 return NVPTXISD::Tld4A2DFloatFloat; 2758 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32: 2759 return NVPTXISD::Tld4R2DS64Float; 2760 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32: 2761 return NVPTXISD::Tld4G2DS64Float; 2762 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32: 2763 return NVPTXISD::Tld4B2DS64Float; 2764 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32: 2765 return NVPTXISD::Tld4A2DS64Float; 2766 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32: 2767 return NVPTXISD::Tld4R2DU64Float; 2768 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32: 2769 return NVPTXISD::Tld4G2DU64Float; 2770 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32: 2771 return NVPTXISD::Tld4B2DU64Float; 2772 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32: 2773 return NVPTXISD::Tld4A2DU64Float; 2774 2775 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32: 2776 return NVPTXISD::TexUnified1DFloatS32; 2777 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32: 2778 return NVPTXISD::TexUnified1DFloatFloat; 2779 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32: 2780 return NVPTXISD::TexUnified1DFloatFloatLevel; 2781 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32: 2782 return NVPTXISD::TexUnified1DFloatFloatGrad; 2783 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32: 2784 return NVPTXISD::TexUnified1DS32S32; 2785 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32: 2786 return NVPTXISD::TexUnified1DS32Float; 2787 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32: 2788 return NVPTXISD::TexUnified1DS32FloatLevel; 2789 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32: 2790 return NVPTXISD::TexUnified1DS32FloatGrad; 2791 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32: 2792 return NVPTXISD::TexUnified1DU32S32; 2793 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32: 2794 return NVPTXISD::TexUnified1DU32Float; 2795 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32: 2796 return NVPTXISD::TexUnified1DU32FloatLevel; 2797 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32: 2798 return NVPTXISD::TexUnified1DU32FloatGrad; 2799 2800 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32: 2801 return NVPTXISD::TexUnified1DArrayFloatS32; 2802 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32: 2803 return NVPTXISD::TexUnified1DArrayFloatFloat; 2804 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32: 2805 return NVPTXISD::TexUnified1DArrayFloatFloatLevel; 2806 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32: 2807 return NVPTXISD::TexUnified1DArrayFloatFloatGrad; 2808 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32: 2809 return NVPTXISD::TexUnified1DArrayS32S32; 2810 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32: 2811 return NVPTXISD::TexUnified1DArrayS32Float; 2812 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32: 2813 return NVPTXISD::TexUnified1DArrayS32FloatLevel; 2814 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32: 2815 return NVPTXISD::TexUnified1DArrayS32FloatGrad; 2816 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32: 2817 return NVPTXISD::TexUnified1DArrayU32S32; 2818 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32: 2819 return NVPTXISD::TexUnified1DArrayU32Float; 2820 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32: 2821 return NVPTXISD::TexUnified1DArrayU32FloatLevel; 2822 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32: 2823 return NVPTXISD::TexUnified1DArrayU32FloatGrad; 2824 2825 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32: 2826 return NVPTXISD::TexUnified2DFloatS32; 2827 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32: 2828 return NVPTXISD::TexUnified2DFloatFloat; 2829 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32: 2830 return NVPTXISD::TexUnified2DFloatFloatLevel; 2831 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32: 2832 return NVPTXISD::TexUnified2DFloatFloatGrad; 2833 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32: 2834 return NVPTXISD::TexUnified2DS32S32; 2835 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32: 2836 return NVPTXISD::TexUnified2DS32Float; 2837 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32: 2838 return NVPTXISD::TexUnified2DS32FloatLevel; 2839 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32: 2840 return NVPTXISD::TexUnified2DS32FloatGrad; 2841 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32: 2842 return NVPTXISD::TexUnified2DU32S32; 2843 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32: 2844 return NVPTXISD::TexUnified2DU32Float; 2845 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32: 2846 return NVPTXISD::TexUnified2DU32FloatLevel; 2847 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32: 2848 return NVPTXISD::TexUnified2DU32FloatGrad; 2849 2850 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32: 2851 return NVPTXISD::TexUnified2DArrayFloatS32; 2852 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32: 2853 return NVPTXISD::TexUnified2DArrayFloatFloat; 2854 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32: 2855 return NVPTXISD::TexUnified2DArrayFloatFloatLevel; 2856 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32: 2857 return NVPTXISD::TexUnified2DArrayFloatFloatGrad; 2858 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32: 2859 return NVPTXISD::TexUnified2DArrayS32S32; 2860 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32: 2861 return NVPTXISD::TexUnified2DArrayS32Float; 2862 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32: 2863 return NVPTXISD::TexUnified2DArrayS32FloatLevel; 2864 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32: 2865 return NVPTXISD::TexUnified2DArrayS32FloatGrad; 2866 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32: 2867 return NVPTXISD::TexUnified2DArrayU32S32; 2868 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32: 2869 return NVPTXISD::TexUnified2DArrayU32Float; 2870 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32: 2871 return NVPTXISD::TexUnified2DArrayU32FloatLevel; 2872 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32: 2873 return NVPTXISD::TexUnified2DArrayU32FloatGrad; 2874 2875 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32: 2876 return NVPTXISD::TexUnified3DFloatS32; 2877 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32: 2878 return NVPTXISD::TexUnified3DFloatFloat; 2879 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32: 2880 return NVPTXISD::TexUnified3DFloatFloatLevel; 2881 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32: 2882 return NVPTXISD::TexUnified3DFloatFloatGrad; 2883 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32: 2884 return NVPTXISD::TexUnified3DS32S32; 2885 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32: 2886 return NVPTXISD::TexUnified3DS32Float; 2887 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32: 2888 return NVPTXISD::TexUnified3DS32FloatLevel; 2889 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32: 2890 return NVPTXISD::TexUnified3DS32FloatGrad; 2891 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32: 2892 return NVPTXISD::TexUnified3DU32S32; 2893 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32: 2894 return NVPTXISD::TexUnified3DU32Float; 2895 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32: 2896 return NVPTXISD::TexUnified3DU32FloatLevel; 2897 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32: 2898 return NVPTXISD::TexUnified3DU32FloatGrad; 2899 2900 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32: 2901 return NVPTXISD::TexUnifiedCubeFloatFloat; 2902 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32: 2903 return NVPTXISD::TexUnifiedCubeFloatFloatLevel; 2904 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32: 2905 return NVPTXISD::TexUnifiedCubeS32Float; 2906 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32: 2907 return NVPTXISD::TexUnifiedCubeS32FloatLevel; 2908 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32: 2909 return NVPTXISD::TexUnifiedCubeU32Float; 2910 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32: 2911 return NVPTXISD::TexUnifiedCubeU32FloatLevel; 2912 2913 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32: 2914 return NVPTXISD::TexUnifiedCubeArrayFloatFloat; 2915 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32: 2916 return NVPTXISD::TexUnifiedCubeArrayFloatFloatLevel; 2917 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32: 2918 return NVPTXISD::TexUnifiedCubeArrayS32Float; 2919 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32: 2920 return NVPTXISD::TexUnifiedCubeArrayS32FloatLevel; 2921 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32: 2922 return NVPTXISD::TexUnifiedCubeArrayU32Float; 2923 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32: 2924 return NVPTXISD::TexUnifiedCubeArrayU32FloatLevel; 2925 2926 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32: 2927 return NVPTXISD::Tld4UnifiedR2DFloatFloat; 2928 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32: 2929 return NVPTXISD::Tld4UnifiedG2DFloatFloat; 2930 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32: 2931 return NVPTXISD::Tld4UnifiedB2DFloatFloat; 2932 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32: 2933 return NVPTXISD::Tld4UnifiedA2DFloatFloat; 2934 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32: 2935 return NVPTXISD::Tld4UnifiedR2DS64Float; 2936 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32: 2937 return NVPTXISD::Tld4UnifiedG2DS64Float; 2938 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32: 2939 return NVPTXISD::Tld4UnifiedB2DS64Float; 2940 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32: 2941 return NVPTXISD::Tld4UnifiedA2DS64Float; 2942 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32: 2943 return NVPTXISD::Tld4UnifiedR2DU64Float; 2944 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32: 2945 return NVPTXISD::Tld4UnifiedG2DU64Float; 2946 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32: 2947 return NVPTXISD::Tld4UnifiedB2DU64Float; 2948 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32: 2949 return NVPTXISD::Tld4UnifiedA2DU64Float; 2950 } 2951 } 2952 2953 static unsigned getOpcForSurfaceInstr(unsigned Intrinsic) { 2954 switch (Intrinsic) { 2955 default: 2956 return 0; 2957 case Intrinsic::nvvm_suld_1d_i8_clamp: 2958 return NVPTXISD::Suld1DI8Clamp; 2959 case Intrinsic::nvvm_suld_1d_i16_clamp: 2960 return NVPTXISD::Suld1DI16Clamp; 2961 case Intrinsic::nvvm_suld_1d_i32_clamp: 2962 return NVPTXISD::Suld1DI32Clamp; 2963 case Intrinsic::nvvm_suld_1d_i64_clamp: 2964 return NVPTXISD::Suld1DI64Clamp; 2965 case Intrinsic::nvvm_suld_1d_v2i8_clamp: 2966 return NVPTXISD::Suld1DV2I8Clamp; 2967 case Intrinsic::nvvm_suld_1d_v2i16_clamp: 2968 return NVPTXISD::Suld1DV2I16Clamp; 2969 case Intrinsic::nvvm_suld_1d_v2i32_clamp: 2970 return NVPTXISD::Suld1DV2I32Clamp; 2971 case Intrinsic::nvvm_suld_1d_v2i64_clamp: 2972 return NVPTXISD::Suld1DV2I64Clamp; 2973 case Intrinsic::nvvm_suld_1d_v4i8_clamp: 2974 return NVPTXISD::Suld1DV4I8Clamp; 2975 case Intrinsic::nvvm_suld_1d_v4i16_clamp: 2976 return NVPTXISD::Suld1DV4I16Clamp; 2977 case Intrinsic::nvvm_suld_1d_v4i32_clamp: 2978 return NVPTXISD::Suld1DV4I32Clamp; 2979 case Intrinsic::nvvm_suld_1d_array_i8_clamp: 2980 return NVPTXISD::Suld1DArrayI8Clamp; 2981 case Intrinsic::nvvm_suld_1d_array_i16_clamp: 2982 return NVPTXISD::Suld1DArrayI16Clamp; 2983 case Intrinsic::nvvm_suld_1d_array_i32_clamp: 2984 return NVPTXISD::Suld1DArrayI32Clamp; 2985 case Intrinsic::nvvm_suld_1d_array_i64_clamp: 2986 return NVPTXISD::Suld1DArrayI64Clamp; 2987 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp: 2988 return NVPTXISD::Suld1DArrayV2I8Clamp; 2989 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp: 2990 return NVPTXISD::Suld1DArrayV2I16Clamp; 2991 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp: 2992 return NVPTXISD::Suld1DArrayV2I32Clamp; 2993 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp: 2994 return NVPTXISD::Suld1DArrayV2I64Clamp; 2995 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp: 2996 return NVPTXISD::Suld1DArrayV4I8Clamp; 2997 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp: 2998 return NVPTXISD::Suld1DArrayV4I16Clamp; 2999 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp: 3000 return NVPTXISD::Suld1DArrayV4I32Clamp; 3001 case Intrinsic::nvvm_suld_2d_i8_clamp: 3002 return NVPTXISD::Suld2DI8Clamp; 3003 case Intrinsic::nvvm_suld_2d_i16_clamp: 3004 return NVPTXISD::Suld2DI16Clamp; 3005 case Intrinsic::nvvm_suld_2d_i32_clamp: 3006 return NVPTXISD::Suld2DI32Clamp; 3007 case Intrinsic::nvvm_suld_2d_i64_clamp: 3008 return NVPTXISD::Suld2DI64Clamp; 3009 case Intrinsic::nvvm_suld_2d_v2i8_clamp: 3010 return NVPTXISD::Suld2DV2I8Clamp; 3011 case Intrinsic::nvvm_suld_2d_v2i16_clamp: 3012 return NVPTXISD::Suld2DV2I16Clamp; 3013 case Intrinsic::nvvm_suld_2d_v2i32_clamp: 3014 return NVPTXISD::Suld2DV2I32Clamp; 3015 case Intrinsic::nvvm_suld_2d_v2i64_clamp: 3016 return NVPTXISD::Suld2DV2I64Clamp; 3017 case Intrinsic::nvvm_suld_2d_v4i8_clamp: 3018 return NVPTXISD::Suld2DV4I8Clamp; 3019 case Intrinsic::nvvm_suld_2d_v4i16_clamp: 3020 return NVPTXISD::Suld2DV4I16Clamp; 3021 case Intrinsic::nvvm_suld_2d_v4i32_clamp: 3022 return NVPTXISD::Suld2DV4I32Clamp; 3023 case Intrinsic::nvvm_suld_2d_array_i8_clamp: 3024 return NVPTXISD::Suld2DArrayI8Clamp; 3025 case Intrinsic::nvvm_suld_2d_array_i16_clamp: 3026 return NVPTXISD::Suld2DArrayI16Clamp; 3027 case Intrinsic::nvvm_suld_2d_array_i32_clamp: 3028 return NVPTXISD::Suld2DArrayI32Clamp; 3029 case Intrinsic::nvvm_suld_2d_array_i64_clamp: 3030 return NVPTXISD::Suld2DArrayI64Clamp; 3031 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp: 3032 return NVPTXISD::Suld2DArrayV2I8Clamp; 3033 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp: 3034 return NVPTXISD::Suld2DArrayV2I16Clamp; 3035 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp: 3036 return NVPTXISD::Suld2DArrayV2I32Clamp; 3037 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp: 3038 return NVPTXISD::Suld2DArrayV2I64Clamp; 3039 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp: 3040 return NVPTXISD::Suld2DArrayV4I8Clamp; 3041 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp: 3042 return NVPTXISD::Suld2DArrayV4I16Clamp; 3043 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp: 3044 return NVPTXISD::Suld2DArrayV4I32Clamp; 3045 case Intrinsic::nvvm_suld_3d_i8_clamp: 3046 return NVPTXISD::Suld3DI8Clamp; 3047 case Intrinsic::nvvm_suld_3d_i16_clamp: 3048 return NVPTXISD::Suld3DI16Clamp; 3049 case Intrinsic::nvvm_suld_3d_i32_clamp: 3050 return NVPTXISD::Suld3DI32Clamp; 3051 case Intrinsic::nvvm_suld_3d_i64_clamp: 3052 return NVPTXISD::Suld3DI64Clamp; 3053 case Intrinsic::nvvm_suld_3d_v2i8_clamp: 3054 return NVPTXISD::Suld3DV2I8Clamp; 3055 case Intrinsic::nvvm_suld_3d_v2i16_clamp: 3056 return NVPTXISD::Suld3DV2I16Clamp; 3057 case Intrinsic::nvvm_suld_3d_v2i32_clamp: 3058 return NVPTXISD::Suld3DV2I32Clamp; 3059 case Intrinsic::nvvm_suld_3d_v2i64_clamp: 3060 return NVPTXISD::Suld3DV2I64Clamp; 3061 case Intrinsic::nvvm_suld_3d_v4i8_clamp: 3062 return NVPTXISD::Suld3DV4I8Clamp; 3063 case Intrinsic::nvvm_suld_3d_v4i16_clamp: 3064 return NVPTXISD::Suld3DV4I16Clamp; 3065 case Intrinsic::nvvm_suld_3d_v4i32_clamp: 3066 return NVPTXISD::Suld3DV4I32Clamp; 3067 case Intrinsic::nvvm_suld_1d_i8_trap: 3068 return NVPTXISD::Suld1DI8Trap; 3069 case Intrinsic::nvvm_suld_1d_i16_trap: 3070 return NVPTXISD::Suld1DI16Trap; 3071 case Intrinsic::nvvm_suld_1d_i32_trap: 3072 return NVPTXISD::Suld1DI32Trap; 3073 case Intrinsic::nvvm_suld_1d_i64_trap: 3074 return NVPTXISD::Suld1DI64Trap; 3075 case Intrinsic::nvvm_suld_1d_v2i8_trap: 3076 return NVPTXISD::Suld1DV2I8Trap; 3077 case Intrinsic::nvvm_suld_1d_v2i16_trap: 3078 return NVPTXISD::Suld1DV2I16Trap; 3079 case Intrinsic::nvvm_suld_1d_v2i32_trap: 3080 return NVPTXISD::Suld1DV2I32Trap; 3081 case Intrinsic::nvvm_suld_1d_v2i64_trap: 3082 return NVPTXISD::Suld1DV2I64Trap; 3083 case Intrinsic::nvvm_suld_1d_v4i8_trap: 3084 return NVPTXISD::Suld1DV4I8Trap; 3085 case Intrinsic::nvvm_suld_1d_v4i16_trap: 3086 return NVPTXISD::Suld1DV4I16Trap; 3087 case Intrinsic::nvvm_suld_1d_v4i32_trap: 3088 return NVPTXISD::Suld1DV4I32Trap; 3089 case Intrinsic::nvvm_suld_1d_array_i8_trap: 3090 return NVPTXISD::Suld1DArrayI8Trap; 3091 case Intrinsic::nvvm_suld_1d_array_i16_trap: 3092 return NVPTXISD::Suld1DArrayI16Trap; 3093 case Intrinsic::nvvm_suld_1d_array_i32_trap: 3094 return NVPTXISD::Suld1DArrayI32Trap; 3095 case Intrinsic::nvvm_suld_1d_array_i64_trap: 3096 return NVPTXISD::Suld1DArrayI64Trap; 3097 case Intrinsic::nvvm_suld_1d_array_v2i8_trap: 3098 return NVPTXISD::Suld1DArrayV2I8Trap; 3099 case Intrinsic::nvvm_suld_1d_array_v2i16_trap: 3100 return NVPTXISD::Suld1DArrayV2I16Trap; 3101 case Intrinsic::nvvm_suld_1d_array_v2i32_trap: 3102 return NVPTXISD::Suld1DArrayV2I32Trap; 3103 case Intrinsic::nvvm_suld_1d_array_v2i64_trap: 3104 return NVPTXISD::Suld1DArrayV2I64Trap; 3105 case Intrinsic::nvvm_suld_1d_array_v4i8_trap: 3106 return NVPTXISD::Suld1DArrayV4I8Trap; 3107 case Intrinsic::nvvm_suld_1d_array_v4i16_trap: 3108 return NVPTXISD::Suld1DArrayV4I16Trap; 3109 case Intrinsic::nvvm_suld_1d_array_v4i32_trap: 3110 return NVPTXISD::Suld1DArrayV4I32Trap; 3111 case Intrinsic::nvvm_suld_2d_i8_trap: 3112 return NVPTXISD::Suld2DI8Trap; 3113 case Intrinsic::nvvm_suld_2d_i16_trap: 3114 return NVPTXISD::Suld2DI16Trap; 3115 case Intrinsic::nvvm_suld_2d_i32_trap: 3116 return NVPTXISD::Suld2DI32Trap; 3117 case Intrinsic::nvvm_suld_2d_i64_trap: 3118 return NVPTXISD::Suld2DI64Trap; 3119 case Intrinsic::nvvm_suld_2d_v2i8_trap: 3120 return NVPTXISD::Suld2DV2I8Trap; 3121 case Intrinsic::nvvm_suld_2d_v2i16_trap: 3122 return NVPTXISD::Suld2DV2I16Trap; 3123 case Intrinsic::nvvm_suld_2d_v2i32_trap: 3124 return NVPTXISD::Suld2DV2I32Trap; 3125 case Intrinsic::nvvm_suld_2d_v2i64_trap: 3126 return NVPTXISD::Suld2DV2I64Trap; 3127 case Intrinsic::nvvm_suld_2d_v4i8_trap: 3128 return NVPTXISD::Suld2DV4I8Trap; 3129 case Intrinsic::nvvm_suld_2d_v4i16_trap: 3130 return NVPTXISD::Suld2DV4I16Trap; 3131 case Intrinsic::nvvm_suld_2d_v4i32_trap: 3132 return NVPTXISD::Suld2DV4I32Trap; 3133 case Intrinsic::nvvm_suld_2d_array_i8_trap: 3134 return NVPTXISD::Suld2DArrayI8Trap; 3135 case Intrinsic::nvvm_suld_2d_array_i16_trap: 3136 return NVPTXISD::Suld2DArrayI16Trap; 3137 case Intrinsic::nvvm_suld_2d_array_i32_trap: 3138 return NVPTXISD::Suld2DArrayI32Trap; 3139 case Intrinsic::nvvm_suld_2d_array_i64_trap: 3140 return NVPTXISD::Suld2DArrayI64Trap; 3141 case Intrinsic::nvvm_suld_2d_array_v2i8_trap: 3142 return NVPTXISD::Suld2DArrayV2I8Trap; 3143 case Intrinsic::nvvm_suld_2d_array_v2i16_trap: 3144 return NVPTXISD::Suld2DArrayV2I16Trap; 3145 case Intrinsic::nvvm_suld_2d_array_v2i32_trap: 3146 return NVPTXISD::Suld2DArrayV2I32Trap; 3147 case Intrinsic::nvvm_suld_2d_array_v2i64_trap: 3148 return NVPTXISD::Suld2DArrayV2I64Trap; 3149 case Intrinsic::nvvm_suld_2d_array_v4i8_trap: 3150 return NVPTXISD::Suld2DArrayV4I8Trap; 3151 case Intrinsic::nvvm_suld_2d_array_v4i16_trap: 3152 return NVPTXISD::Suld2DArrayV4I16Trap; 3153 case Intrinsic::nvvm_suld_2d_array_v4i32_trap: 3154 return NVPTXISD::Suld2DArrayV4I32Trap; 3155 case Intrinsic::nvvm_suld_3d_i8_trap: 3156 return NVPTXISD::Suld3DI8Trap; 3157 case Intrinsic::nvvm_suld_3d_i16_trap: 3158 return NVPTXISD::Suld3DI16Trap; 3159 case Intrinsic::nvvm_suld_3d_i32_trap: 3160 return NVPTXISD::Suld3DI32Trap; 3161 case Intrinsic::nvvm_suld_3d_i64_trap: 3162 return NVPTXISD::Suld3DI64Trap; 3163 case Intrinsic::nvvm_suld_3d_v2i8_trap: 3164 return NVPTXISD::Suld3DV2I8Trap; 3165 case Intrinsic::nvvm_suld_3d_v2i16_trap: 3166 return NVPTXISD::Suld3DV2I16Trap; 3167 case Intrinsic::nvvm_suld_3d_v2i32_trap: 3168 return NVPTXISD::Suld3DV2I32Trap; 3169 case Intrinsic::nvvm_suld_3d_v2i64_trap: 3170 return NVPTXISD::Suld3DV2I64Trap; 3171 case Intrinsic::nvvm_suld_3d_v4i8_trap: 3172 return NVPTXISD::Suld3DV4I8Trap; 3173 case Intrinsic::nvvm_suld_3d_v4i16_trap: 3174 return NVPTXISD::Suld3DV4I16Trap; 3175 case Intrinsic::nvvm_suld_3d_v4i32_trap: 3176 return NVPTXISD::Suld3DV4I32Trap; 3177 case Intrinsic::nvvm_suld_1d_i8_zero: 3178 return NVPTXISD::Suld1DI8Zero; 3179 case Intrinsic::nvvm_suld_1d_i16_zero: 3180 return NVPTXISD::Suld1DI16Zero; 3181 case Intrinsic::nvvm_suld_1d_i32_zero: 3182 return NVPTXISD::Suld1DI32Zero; 3183 case Intrinsic::nvvm_suld_1d_i64_zero: 3184 return NVPTXISD::Suld1DI64Zero; 3185 case Intrinsic::nvvm_suld_1d_v2i8_zero: 3186 return NVPTXISD::Suld1DV2I8Zero; 3187 case Intrinsic::nvvm_suld_1d_v2i16_zero: 3188 return NVPTXISD::Suld1DV2I16Zero; 3189 case Intrinsic::nvvm_suld_1d_v2i32_zero: 3190 return NVPTXISD::Suld1DV2I32Zero; 3191 case Intrinsic::nvvm_suld_1d_v2i64_zero: 3192 return NVPTXISD::Suld1DV2I64Zero; 3193 case Intrinsic::nvvm_suld_1d_v4i8_zero: 3194 return NVPTXISD::Suld1DV4I8Zero; 3195 case Intrinsic::nvvm_suld_1d_v4i16_zero: 3196 return NVPTXISD::Suld1DV4I16Zero; 3197 case Intrinsic::nvvm_suld_1d_v4i32_zero: 3198 return NVPTXISD::Suld1DV4I32Zero; 3199 case Intrinsic::nvvm_suld_1d_array_i8_zero: 3200 return NVPTXISD::Suld1DArrayI8Zero; 3201 case Intrinsic::nvvm_suld_1d_array_i16_zero: 3202 return NVPTXISD::Suld1DArrayI16Zero; 3203 case Intrinsic::nvvm_suld_1d_array_i32_zero: 3204 return NVPTXISD::Suld1DArrayI32Zero; 3205 case Intrinsic::nvvm_suld_1d_array_i64_zero: 3206 return NVPTXISD::Suld1DArrayI64Zero; 3207 case Intrinsic::nvvm_suld_1d_array_v2i8_zero: 3208 return NVPTXISD::Suld1DArrayV2I8Zero; 3209 case Intrinsic::nvvm_suld_1d_array_v2i16_zero: 3210 return NVPTXISD::Suld1DArrayV2I16Zero; 3211 case Intrinsic::nvvm_suld_1d_array_v2i32_zero: 3212 return NVPTXISD::Suld1DArrayV2I32Zero; 3213 case Intrinsic::nvvm_suld_1d_array_v2i64_zero: 3214 return NVPTXISD::Suld1DArrayV2I64Zero; 3215 case Intrinsic::nvvm_suld_1d_array_v4i8_zero: 3216 return NVPTXISD::Suld1DArrayV4I8Zero; 3217 case Intrinsic::nvvm_suld_1d_array_v4i16_zero: 3218 return NVPTXISD::Suld1DArrayV4I16Zero; 3219 case Intrinsic::nvvm_suld_1d_array_v4i32_zero: 3220 return NVPTXISD::Suld1DArrayV4I32Zero; 3221 case Intrinsic::nvvm_suld_2d_i8_zero: 3222 return NVPTXISD::Suld2DI8Zero; 3223 case Intrinsic::nvvm_suld_2d_i16_zero: 3224 return NVPTXISD::Suld2DI16Zero; 3225 case Intrinsic::nvvm_suld_2d_i32_zero: 3226 return NVPTXISD::Suld2DI32Zero; 3227 case Intrinsic::nvvm_suld_2d_i64_zero: 3228 return NVPTXISD::Suld2DI64Zero; 3229 case Intrinsic::nvvm_suld_2d_v2i8_zero: 3230 return NVPTXISD::Suld2DV2I8Zero; 3231 case Intrinsic::nvvm_suld_2d_v2i16_zero: 3232 return NVPTXISD::Suld2DV2I16Zero; 3233 case Intrinsic::nvvm_suld_2d_v2i32_zero: 3234 return NVPTXISD::Suld2DV2I32Zero; 3235 case Intrinsic::nvvm_suld_2d_v2i64_zero: 3236 return NVPTXISD::Suld2DV2I64Zero; 3237 case Intrinsic::nvvm_suld_2d_v4i8_zero: 3238 return NVPTXISD::Suld2DV4I8Zero; 3239 case Intrinsic::nvvm_suld_2d_v4i16_zero: 3240 return NVPTXISD::Suld2DV4I16Zero; 3241 case Intrinsic::nvvm_suld_2d_v4i32_zero: 3242 return NVPTXISD::Suld2DV4I32Zero; 3243 case Intrinsic::nvvm_suld_2d_array_i8_zero: 3244 return NVPTXISD::Suld2DArrayI8Zero; 3245 case Intrinsic::nvvm_suld_2d_array_i16_zero: 3246 return NVPTXISD::Suld2DArrayI16Zero; 3247 case Intrinsic::nvvm_suld_2d_array_i32_zero: 3248 return NVPTXISD::Suld2DArrayI32Zero; 3249 case Intrinsic::nvvm_suld_2d_array_i64_zero: 3250 return NVPTXISD::Suld2DArrayI64Zero; 3251 case Intrinsic::nvvm_suld_2d_array_v2i8_zero: 3252 return NVPTXISD::Suld2DArrayV2I8Zero; 3253 case Intrinsic::nvvm_suld_2d_array_v2i16_zero: 3254 return NVPTXISD::Suld2DArrayV2I16Zero; 3255 case Intrinsic::nvvm_suld_2d_array_v2i32_zero: 3256 return NVPTXISD::Suld2DArrayV2I32Zero; 3257 case Intrinsic::nvvm_suld_2d_array_v2i64_zero: 3258 return NVPTXISD::Suld2DArrayV2I64Zero; 3259 case Intrinsic::nvvm_suld_2d_array_v4i8_zero: 3260 return NVPTXISD::Suld2DArrayV4I8Zero; 3261 case Intrinsic::nvvm_suld_2d_array_v4i16_zero: 3262 return NVPTXISD::Suld2DArrayV4I16Zero; 3263 case Intrinsic::nvvm_suld_2d_array_v4i32_zero: 3264 return NVPTXISD::Suld2DArrayV4I32Zero; 3265 case Intrinsic::nvvm_suld_3d_i8_zero: 3266 return NVPTXISD::Suld3DI8Zero; 3267 case Intrinsic::nvvm_suld_3d_i16_zero: 3268 return NVPTXISD::Suld3DI16Zero; 3269 case Intrinsic::nvvm_suld_3d_i32_zero: 3270 return NVPTXISD::Suld3DI32Zero; 3271 case Intrinsic::nvvm_suld_3d_i64_zero: 3272 return NVPTXISD::Suld3DI64Zero; 3273 case Intrinsic::nvvm_suld_3d_v2i8_zero: 3274 return NVPTXISD::Suld3DV2I8Zero; 3275 case Intrinsic::nvvm_suld_3d_v2i16_zero: 3276 return NVPTXISD::Suld3DV2I16Zero; 3277 case Intrinsic::nvvm_suld_3d_v2i32_zero: 3278 return NVPTXISD::Suld3DV2I32Zero; 3279 case Intrinsic::nvvm_suld_3d_v2i64_zero: 3280 return NVPTXISD::Suld3DV2I64Zero; 3281 case Intrinsic::nvvm_suld_3d_v4i8_zero: 3282 return NVPTXISD::Suld3DV4I8Zero; 3283 case Intrinsic::nvvm_suld_3d_v4i16_zero: 3284 return NVPTXISD::Suld3DV4I16Zero; 3285 case Intrinsic::nvvm_suld_3d_v4i32_zero: 3286 return NVPTXISD::Suld3DV4I32Zero; 3287 } 3288 } 3289 3290 // llvm.ptx.memcpy.const and llvm.ptx.memmove.const need to be modeled as 3291 // TgtMemIntrinsic 3292 // because we need the information that is only available in the "Value" type 3293 // of destination 3294 // pointer. In particular, the address space information. 3295 bool NVPTXTargetLowering::getTgtMemIntrinsic( 3296 IntrinsicInfo &Info, const CallInst &I, 3297 MachineFunction &MF, unsigned Intrinsic) const { 3298 switch (Intrinsic) { 3299 default: 3300 return false; 3301 case Intrinsic::nvvm_match_all_sync_i32p: 3302 case Intrinsic::nvvm_match_all_sync_i64p: 3303 Info.opc = ISD::INTRINSIC_W_CHAIN; 3304 // memVT is bogus. These intrinsics have IntrInaccessibleMemOnly attribute 3305 // in order to model data exchange with other threads, but perform no real 3306 // memory accesses. 3307 Info.memVT = MVT::i1; 3308 3309 // Our result depends on both our and other thread's arguments. 3310 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 3311 return true; 3312 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col: 3313 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row: 3314 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride: 3315 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride: 3316 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col: 3317 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row: 3318 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride: 3319 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride: 3320 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col: 3321 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row: 3322 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride: 3323 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride: 3324 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col: 3325 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row: 3326 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride: 3327 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride: 3328 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col: 3329 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row: 3330 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride: 3331 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride: 3332 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col: 3333 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row: 3334 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride: 3335 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: { 3336 Info.opc = ISD::INTRINSIC_W_CHAIN; 3337 Info.memVT = MVT::v8f16; 3338 Info.ptrVal = I.getArgOperand(0); 3339 Info.offset = 0; 3340 Info.flags = MachineMemOperand::MOLoad; 3341 Info.align = 16; 3342 return true; 3343 } 3344 3345 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col: 3346 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row: 3347 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride: 3348 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride: 3349 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col: 3350 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row: 3351 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride: 3352 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride: 3353 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col: 3354 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row: 3355 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride: 3356 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: { 3357 Info.opc = ISD::INTRINSIC_W_CHAIN; 3358 Info.memVT = MVT::v4f16; 3359 Info.ptrVal = I.getArgOperand(0); 3360 Info.offset = 0; 3361 Info.flags = MachineMemOperand::MOLoad; 3362 Info.align = 16; 3363 return true; 3364 } 3365 3366 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col: 3367 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row: 3368 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride: 3369 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride: 3370 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col: 3371 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row: 3372 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride: 3373 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride: 3374 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col: 3375 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row: 3376 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride: 3377 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride: { 3378 Info.opc = ISD::INTRINSIC_W_CHAIN; 3379 Info.memVT = MVT::v8f32; 3380 Info.ptrVal = I.getArgOperand(0); 3381 Info.offset = 0; 3382 Info.flags = MachineMemOperand::MOLoad; 3383 Info.align = 16; 3384 return true; 3385 } 3386 3387 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col: 3388 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row: 3389 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride: 3390 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride: 3391 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col: 3392 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row: 3393 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride: 3394 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride: 3395 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col: 3396 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row: 3397 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride: 3398 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: { 3399 Info.opc = ISD::INTRINSIC_VOID; 3400 Info.memVT = MVT::v4f16; 3401 Info.ptrVal = I.getArgOperand(0); 3402 Info.offset = 0; 3403 Info.flags = MachineMemOperand::MOStore; 3404 Info.align = 16; 3405 return true; 3406 } 3407 3408 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col: 3409 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row: 3410 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride: 3411 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride: 3412 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col: 3413 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row: 3414 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride: 3415 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride: 3416 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col: 3417 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row: 3418 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride: 3419 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride: { 3420 Info.opc = ISD::INTRINSIC_VOID; 3421 Info.memVT = MVT::v8f32; 3422 Info.ptrVal = I.getArgOperand(0); 3423 Info.offset = 0; 3424 Info.flags = MachineMemOperand::MOStore; 3425 Info.align = 16; 3426 return true; 3427 } 3428 3429 case Intrinsic::nvvm_atomic_load_add_f32: 3430 case Intrinsic::nvvm_atomic_load_add_f64: 3431 case Intrinsic::nvvm_atomic_load_inc_32: 3432 case Intrinsic::nvvm_atomic_load_dec_32: 3433 3434 case Intrinsic::nvvm_atomic_add_gen_f_cta: 3435 case Intrinsic::nvvm_atomic_add_gen_f_sys: 3436 case Intrinsic::nvvm_atomic_add_gen_i_cta: 3437 case Intrinsic::nvvm_atomic_add_gen_i_sys: 3438 case Intrinsic::nvvm_atomic_and_gen_i_cta: 3439 case Intrinsic::nvvm_atomic_and_gen_i_sys: 3440 case Intrinsic::nvvm_atomic_cas_gen_i_cta: 3441 case Intrinsic::nvvm_atomic_cas_gen_i_sys: 3442 case Intrinsic::nvvm_atomic_dec_gen_i_cta: 3443 case Intrinsic::nvvm_atomic_dec_gen_i_sys: 3444 case Intrinsic::nvvm_atomic_inc_gen_i_cta: 3445 case Intrinsic::nvvm_atomic_inc_gen_i_sys: 3446 case Intrinsic::nvvm_atomic_max_gen_i_cta: 3447 case Intrinsic::nvvm_atomic_max_gen_i_sys: 3448 case Intrinsic::nvvm_atomic_min_gen_i_cta: 3449 case Intrinsic::nvvm_atomic_min_gen_i_sys: 3450 case Intrinsic::nvvm_atomic_or_gen_i_cta: 3451 case Intrinsic::nvvm_atomic_or_gen_i_sys: 3452 case Intrinsic::nvvm_atomic_exch_gen_i_cta: 3453 case Intrinsic::nvvm_atomic_exch_gen_i_sys: 3454 case Intrinsic::nvvm_atomic_xor_gen_i_cta: 3455 case Intrinsic::nvvm_atomic_xor_gen_i_sys: { 3456 auto &DL = I.getModule()->getDataLayout(); 3457 Info.opc = ISD::INTRINSIC_W_CHAIN; 3458 Info.memVT = getValueType(DL, I.getType()); 3459 Info.ptrVal = I.getArgOperand(0); 3460 Info.offset = 0; 3461 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; 3462 Info.align = 0; 3463 return true; 3464 } 3465 3466 case Intrinsic::nvvm_ldu_global_i: 3467 case Intrinsic::nvvm_ldu_global_f: 3468 case Intrinsic::nvvm_ldu_global_p: { 3469 auto &DL = I.getModule()->getDataLayout(); 3470 Info.opc = ISD::INTRINSIC_W_CHAIN; 3471 if (Intrinsic == Intrinsic::nvvm_ldu_global_i) 3472 Info.memVT = getValueType(DL, I.getType()); 3473 else if(Intrinsic == Intrinsic::nvvm_ldu_global_p) 3474 Info.memVT = getPointerTy(DL); 3475 else 3476 Info.memVT = getValueType(DL, I.getType()); 3477 Info.ptrVal = I.getArgOperand(0); 3478 Info.offset = 0; 3479 Info.flags = MachineMemOperand::MOLoad; 3480 Info.align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 3481 3482 return true; 3483 } 3484 case Intrinsic::nvvm_ldg_global_i: 3485 case Intrinsic::nvvm_ldg_global_f: 3486 case Intrinsic::nvvm_ldg_global_p: { 3487 auto &DL = I.getModule()->getDataLayout(); 3488 3489 Info.opc = ISD::INTRINSIC_W_CHAIN; 3490 if (Intrinsic == Intrinsic::nvvm_ldg_global_i) 3491 Info.memVT = getValueType(DL, I.getType()); 3492 else if(Intrinsic == Intrinsic::nvvm_ldg_global_p) 3493 Info.memVT = getPointerTy(DL); 3494 else 3495 Info.memVT = getValueType(DL, I.getType()); 3496 Info.ptrVal = I.getArgOperand(0); 3497 Info.offset = 0; 3498 Info.flags = MachineMemOperand::MOLoad; 3499 Info.align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 3500 3501 return true; 3502 } 3503 3504 case Intrinsic::nvvm_tex_1d_v4f32_s32: 3505 case Intrinsic::nvvm_tex_1d_v4f32_f32: 3506 case Intrinsic::nvvm_tex_1d_level_v4f32_f32: 3507 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32: 3508 case Intrinsic::nvvm_tex_1d_array_v4f32_s32: 3509 case Intrinsic::nvvm_tex_1d_array_v4f32_f32: 3510 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32: 3511 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32: 3512 case Intrinsic::nvvm_tex_2d_v4f32_s32: 3513 case Intrinsic::nvvm_tex_2d_v4f32_f32: 3514 case Intrinsic::nvvm_tex_2d_level_v4f32_f32: 3515 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32: 3516 case Intrinsic::nvvm_tex_2d_array_v4f32_s32: 3517 case Intrinsic::nvvm_tex_2d_array_v4f32_f32: 3518 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32: 3519 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32: 3520 case Intrinsic::nvvm_tex_3d_v4f32_s32: 3521 case Intrinsic::nvvm_tex_3d_v4f32_f32: 3522 case Intrinsic::nvvm_tex_3d_level_v4f32_f32: 3523 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32: 3524 case Intrinsic::nvvm_tex_cube_v4f32_f32: 3525 case Intrinsic::nvvm_tex_cube_level_v4f32_f32: 3526 case Intrinsic::nvvm_tex_cube_array_v4f32_f32: 3527 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32: 3528 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32: 3529 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32: 3530 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32: 3531 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32: 3532 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32: 3533 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32: 3534 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32: 3535 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32: 3536 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32: 3537 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32: 3538 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32: 3539 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32: 3540 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32: 3541 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32: 3542 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32: 3543 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32: 3544 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32: 3545 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32: 3546 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32: 3547 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32: 3548 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32: 3549 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32: 3550 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32: 3551 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32: 3552 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32: 3553 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32: 3554 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32: 3555 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32: 3556 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32: 3557 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32: 3558 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32: 3559 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32: 3560 Info.opc = getOpcForTextureInstr(Intrinsic); 3561 Info.memVT = MVT::v4f32; 3562 Info.ptrVal = nullptr; 3563 Info.offset = 0; 3564 Info.flags = MachineMemOperand::MOLoad; 3565 Info.align = 16; 3566 return true; 3567 3568 case Intrinsic::nvvm_tex_1d_v4s32_s32: 3569 case Intrinsic::nvvm_tex_1d_v4s32_f32: 3570 case Intrinsic::nvvm_tex_1d_level_v4s32_f32: 3571 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32: 3572 case Intrinsic::nvvm_tex_1d_array_v4s32_s32: 3573 case Intrinsic::nvvm_tex_1d_array_v4s32_f32: 3574 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32: 3575 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32: 3576 case Intrinsic::nvvm_tex_2d_v4s32_s32: 3577 case Intrinsic::nvvm_tex_2d_v4s32_f32: 3578 case Intrinsic::nvvm_tex_2d_level_v4s32_f32: 3579 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32: 3580 case Intrinsic::nvvm_tex_2d_array_v4s32_s32: 3581 case Intrinsic::nvvm_tex_2d_array_v4s32_f32: 3582 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32: 3583 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32: 3584 case Intrinsic::nvvm_tex_3d_v4s32_s32: 3585 case Intrinsic::nvvm_tex_3d_v4s32_f32: 3586 case Intrinsic::nvvm_tex_3d_level_v4s32_f32: 3587 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32: 3588 case Intrinsic::nvvm_tex_cube_v4s32_f32: 3589 case Intrinsic::nvvm_tex_cube_level_v4s32_f32: 3590 case Intrinsic::nvvm_tex_cube_array_v4s32_f32: 3591 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32: 3592 case Intrinsic::nvvm_tex_cube_v4u32_f32: 3593 case Intrinsic::nvvm_tex_cube_level_v4u32_f32: 3594 case Intrinsic::nvvm_tex_cube_array_v4u32_f32: 3595 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32: 3596 case Intrinsic::nvvm_tex_1d_v4u32_s32: 3597 case Intrinsic::nvvm_tex_1d_v4u32_f32: 3598 case Intrinsic::nvvm_tex_1d_level_v4u32_f32: 3599 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32: 3600 case Intrinsic::nvvm_tex_1d_array_v4u32_s32: 3601 case Intrinsic::nvvm_tex_1d_array_v4u32_f32: 3602 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32: 3603 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32: 3604 case Intrinsic::nvvm_tex_2d_v4u32_s32: 3605 case Intrinsic::nvvm_tex_2d_v4u32_f32: 3606 case Intrinsic::nvvm_tex_2d_level_v4u32_f32: 3607 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32: 3608 case Intrinsic::nvvm_tex_2d_array_v4u32_s32: 3609 case Intrinsic::nvvm_tex_2d_array_v4u32_f32: 3610 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32: 3611 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32: 3612 case Intrinsic::nvvm_tex_3d_v4u32_s32: 3613 case Intrinsic::nvvm_tex_3d_v4u32_f32: 3614 case Intrinsic::nvvm_tex_3d_level_v4u32_f32: 3615 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32: 3616 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32: 3617 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32: 3618 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32: 3619 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32: 3620 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32: 3621 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32: 3622 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32: 3623 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32: 3624 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32: 3625 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32: 3626 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32: 3627 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32: 3628 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32: 3629 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32: 3630 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32: 3631 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32: 3632 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32: 3633 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32: 3634 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32: 3635 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32: 3636 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32: 3637 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32: 3638 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32: 3639 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32: 3640 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32: 3641 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32: 3642 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32: 3643 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32: 3644 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32: 3645 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32: 3646 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32: 3647 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32: 3648 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32: 3649 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32: 3650 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32: 3651 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32: 3652 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32: 3653 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32: 3654 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32: 3655 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32: 3656 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32: 3657 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32: 3658 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32: 3659 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32: 3660 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32: 3661 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32: 3662 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32: 3663 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32: 3664 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32: 3665 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32: 3666 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32: 3667 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32: 3668 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32: 3669 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32: 3670 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32: 3671 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32: 3672 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32: 3673 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32: 3674 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32: 3675 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32: 3676 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32: 3677 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32: 3678 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32: 3679 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32: 3680 Info.opc = getOpcForTextureInstr(Intrinsic); 3681 Info.memVT = MVT::v4i32; 3682 Info.ptrVal = nullptr; 3683 Info.offset = 0; 3684 Info.flags = MachineMemOperand::MOLoad; 3685 Info.align = 16; 3686 return true; 3687 3688 case Intrinsic::nvvm_suld_1d_i8_clamp: 3689 case Intrinsic::nvvm_suld_1d_v2i8_clamp: 3690 case Intrinsic::nvvm_suld_1d_v4i8_clamp: 3691 case Intrinsic::nvvm_suld_1d_array_i8_clamp: 3692 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp: 3693 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp: 3694 case Intrinsic::nvvm_suld_2d_i8_clamp: 3695 case Intrinsic::nvvm_suld_2d_v2i8_clamp: 3696 case Intrinsic::nvvm_suld_2d_v4i8_clamp: 3697 case Intrinsic::nvvm_suld_2d_array_i8_clamp: 3698 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp: 3699 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp: 3700 case Intrinsic::nvvm_suld_3d_i8_clamp: 3701 case Intrinsic::nvvm_suld_3d_v2i8_clamp: 3702 case Intrinsic::nvvm_suld_3d_v4i8_clamp: 3703 case Intrinsic::nvvm_suld_1d_i8_trap: 3704 case Intrinsic::nvvm_suld_1d_v2i8_trap: 3705 case Intrinsic::nvvm_suld_1d_v4i8_trap: 3706 case Intrinsic::nvvm_suld_1d_array_i8_trap: 3707 case Intrinsic::nvvm_suld_1d_array_v2i8_trap: 3708 case Intrinsic::nvvm_suld_1d_array_v4i8_trap: 3709 case Intrinsic::nvvm_suld_2d_i8_trap: 3710 case Intrinsic::nvvm_suld_2d_v2i8_trap: 3711 case Intrinsic::nvvm_suld_2d_v4i8_trap: 3712 case Intrinsic::nvvm_suld_2d_array_i8_trap: 3713 case Intrinsic::nvvm_suld_2d_array_v2i8_trap: 3714 case Intrinsic::nvvm_suld_2d_array_v4i8_trap: 3715 case Intrinsic::nvvm_suld_3d_i8_trap: 3716 case Intrinsic::nvvm_suld_3d_v2i8_trap: 3717 case Intrinsic::nvvm_suld_3d_v4i8_trap: 3718 case Intrinsic::nvvm_suld_1d_i8_zero: 3719 case Intrinsic::nvvm_suld_1d_v2i8_zero: 3720 case Intrinsic::nvvm_suld_1d_v4i8_zero: 3721 case Intrinsic::nvvm_suld_1d_array_i8_zero: 3722 case Intrinsic::nvvm_suld_1d_array_v2i8_zero: 3723 case Intrinsic::nvvm_suld_1d_array_v4i8_zero: 3724 case Intrinsic::nvvm_suld_2d_i8_zero: 3725 case Intrinsic::nvvm_suld_2d_v2i8_zero: 3726 case Intrinsic::nvvm_suld_2d_v4i8_zero: 3727 case Intrinsic::nvvm_suld_2d_array_i8_zero: 3728 case Intrinsic::nvvm_suld_2d_array_v2i8_zero: 3729 case Intrinsic::nvvm_suld_2d_array_v4i8_zero: 3730 case Intrinsic::nvvm_suld_3d_i8_zero: 3731 case Intrinsic::nvvm_suld_3d_v2i8_zero: 3732 case Intrinsic::nvvm_suld_3d_v4i8_zero: 3733 Info.opc = getOpcForSurfaceInstr(Intrinsic); 3734 Info.memVT = MVT::i8; 3735 Info.ptrVal = nullptr; 3736 Info.offset = 0; 3737 Info.flags = MachineMemOperand::MOLoad; 3738 Info.align = 16; 3739 return true; 3740 3741 case Intrinsic::nvvm_suld_1d_i16_clamp: 3742 case Intrinsic::nvvm_suld_1d_v2i16_clamp: 3743 case Intrinsic::nvvm_suld_1d_v4i16_clamp: 3744 case Intrinsic::nvvm_suld_1d_array_i16_clamp: 3745 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp: 3746 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp: 3747 case Intrinsic::nvvm_suld_2d_i16_clamp: 3748 case Intrinsic::nvvm_suld_2d_v2i16_clamp: 3749 case Intrinsic::nvvm_suld_2d_v4i16_clamp: 3750 case Intrinsic::nvvm_suld_2d_array_i16_clamp: 3751 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp: 3752 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp: 3753 case Intrinsic::nvvm_suld_3d_i16_clamp: 3754 case Intrinsic::nvvm_suld_3d_v2i16_clamp: 3755 case Intrinsic::nvvm_suld_3d_v4i16_clamp: 3756 case Intrinsic::nvvm_suld_1d_i16_trap: 3757 case Intrinsic::nvvm_suld_1d_v2i16_trap: 3758 case Intrinsic::nvvm_suld_1d_v4i16_trap: 3759 case Intrinsic::nvvm_suld_1d_array_i16_trap: 3760 case Intrinsic::nvvm_suld_1d_array_v2i16_trap: 3761 case Intrinsic::nvvm_suld_1d_array_v4i16_trap: 3762 case Intrinsic::nvvm_suld_2d_i16_trap: 3763 case Intrinsic::nvvm_suld_2d_v2i16_trap: 3764 case Intrinsic::nvvm_suld_2d_v4i16_trap: 3765 case Intrinsic::nvvm_suld_2d_array_i16_trap: 3766 case Intrinsic::nvvm_suld_2d_array_v2i16_trap: 3767 case Intrinsic::nvvm_suld_2d_array_v4i16_trap: 3768 case Intrinsic::nvvm_suld_3d_i16_trap: 3769 case Intrinsic::nvvm_suld_3d_v2i16_trap: 3770 case Intrinsic::nvvm_suld_3d_v4i16_trap: 3771 case Intrinsic::nvvm_suld_1d_i16_zero: 3772 case Intrinsic::nvvm_suld_1d_v2i16_zero: 3773 case Intrinsic::nvvm_suld_1d_v4i16_zero: 3774 case Intrinsic::nvvm_suld_1d_array_i16_zero: 3775 case Intrinsic::nvvm_suld_1d_array_v2i16_zero: 3776 case Intrinsic::nvvm_suld_1d_array_v4i16_zero: 3777 case Intrinsic::nvvm_suld_2d_i16_zero: 3778 case Intrinsic::nvvm_suld_2d_v2i16_zero: 3779 case Intrinsic::nvvm_suld_2d_v4i16_zero: 3780 case Intrinsic::nvvm_suld_2d_array_i16_zero: 3781 case Intrinsic::nvvm_suld_2d_array_v2i16_zero: 3782 case Intrinsic::nvvm_suld_2d_array_v4i16_zero: 3783 case Intrinsic::nvvm_suld_3d_i16_zero: 3784 case Intrinsic::nvvm_suld_3d_v2i16_zero: 3785 case Intrinsic::nvvm_suld_3d_v4i16_zero: 3786 Info.opc = getOpcForSurfaceInstr(Intrinsic); 3787 Info.memVT = MVT::i16; 3788 Info.ptrVal = nullptr; 3789 Info.offset = 0; 3790 Info.flags = MachineMemOperand::MOLoad; 3791 Info.align = 16; 3792 return true; 3793 3794 case Intrinsic::nvvm_suld_1d_i32_clamp: 3795 case Intrinsic::nvvm_suld_1d_v2i32_clamp: 3796 case Intrinsic::nvvm_suld_1d_v4i32_clamp: 3797 case Intrinsic::nvvm_suld_1d_array_i32_clamp: 3798 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp: 3799 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp: 3800 case Intrinsic::nvvm_suld_2d_i32_clamp: 3801 case Intrinsic::nvvm_suld_2d_v2i32_clamp: 3802 case Intrinsic::nvvm_suld_2d_v4i32_clamp: 3803 case Intrinsic::nvvm_suld_2d_array_i32_clamp: 3804 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp: 3805 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp: 3806 case Intrinsic::nvvm_suld_3d_i32_clamp: 3807 case Intrinsic::nvvm_suld_3d_v2i32_clamp: 3808 case Intrinsic::nvvm_suld_3d_v4i32_clamp: 3809 case Intrinsic::nvvm_suld_1d_i32_trap: 3810 case Intrinsic::nvvm_suld_1d_v2i32_trap: 3811 case Intrinsic::nvvm_suld_1d_v4i32_trap: 3812 case Intrinsic::nvvm_suld_1d_array_i32_trap: 3813 case Intrinsic::nvvm_suld_1d_array_v2i32_trap: 3814 case Intrinsic::nvvm_suld_1d_array_v4i32_trap: 3815 case Intrinsic::nvvm_suld_2d_i32_trap: 3816 case Intrinsic::nvvm_suld_2d_v2i32_trap: 3817 case Intrinsic::nvvm_suld_2d_v4i32_trap: 3818 case Intrinsic::nvvm_suld_2d_array_i32_trap: 3819 case Intrinsic::nvvm_suld_2d_array_v2i32_trap: 3820 case Intrinsic::nvvm_suld_2d_array_v4i32_trap: 3821 case Intrinsic::nvvm_suld_3d_i32_trap: 3822 case Intrinsic::nvvm_suld_3d_v2i32_trap: 3823 case Intrinsic::nvvm_suld_3d_v4i32_trap: 3824 case Intrinsic::nvvm_suld_1d_i32_zero: 3825 case Intrinsic::nvvm_suld_1d_v2i32_zero: 3826 case Intrinsic::nvvm_suld_1d_v4i32_zero: 3827 case Intrinsic::nvvm_suld_1d_array_i32_zero: 3828 case Intrinsic::nvvm_suld_1d_array_v2i32_zero: 3829 case Intrinsic::nvvm_suld_1d_array_v4i32_zero: 3830 case Intrinsic::nvvm_suld_2d_i32_zero: 3831 case Intrinsic::nvvm_suld_2d_v2i32_zero: 3832 case Intrinsic::nvvm_suld_2d_v4i32_zero: 3833 case Intrinsic::nvvm_suld_2d_array_i32_zero: 3834 case Intrinsic::nvvm_suld_2d_array_v2i32_zero: 3835 case Intrinsic::nvvm_suld_2d_array_v4i32_zero: 3836 case Intrinsic::nvvm_suld_3d_i32_zero: 3837 case Intrinsic::nvvm_suld_3d_v2i32_zero: 3838 case Intrinsic::nvvm_suld_3d_v4i32_zero: 3839 Info.opc = getOpcForSurfaceInstr(Intrinsic); 3840 Info.memVT = MVT::i32; 3841 Info.ptrVal = nullptr; 3842 Info.offset = 0; 3843 Info.flags = MachineMemOperand::MOLoad; 3844 Info.align = 16; 3845 return true; 3846 3847 case Intrinsic::nvvm_suld_1d_i64_clamp: 3848 case Intrinsic::nvvm_suld_1d_v2i64_clamp: 3849 case Intrinsic::nvvm_suld_1d_array_i64_clamp: 3850 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp: 3851 case Intrinsic::nvvm_suld_2d_i64_clamp: 3852 case Intrinsic::nvvm_suld_2d_v2i64_clamp: 3853 case Intrinsic::nvvm_suld_2d_array_i64_clamp: 3854 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp: 3855 case Intrinsic::nvvm_suld_3d_i64_clamp: 3856 case Intrinsic::nvvm_suld_3d_v2i64_clamp: 3857 case Intrinsic::nvvm_suld_1d_i64_trap: 3858 case Intrinsic::nvvm_suld_1d_v2i64_trap: 3859 case Intrinsic::nvvm_suld_1d_array_i64_trap: 3860 case Intrinsic::nvvm_suld_1d_array_v2i64_trap: 3861 case Intrinsic::nvvm_suld_2d_i64_trap: 3862 case Intrinsic::nvvm_suld_2d_v2i64_trap: 3863 case Intrinsic::nvvm_suld_2d_array_i64_trap: 3864 case Intrinsic::nvvm_suld_2d_array_v2i64_trap: 3865 case Intrinsic::nvvm_suld_3d_i64_trap: 3866 case Intrinsic::nvvm_suld_3d_v2i64_trap: 3867 case Intrinsic::nvvm_suld_1d_i64_zero: 3868 case Intrinsic::nvvm_suld_1d_v2i64_zero: 3869 case Intrinsic::nvvm_suld_1d_array_i64_zero: 3870 case Intrinsic::nvvm_suld_1d_array_v2i64_zero: 3871 case Intrinsic::nvvm_suld_2d_i64_zero: 3872 case Intrinsic::nvvm_suld_2d_v2i64_zero: 3873 case Intrinsic::nvvm_suld_2d_array_i64_zero: 3874 case Intrinsic::nvvm_suld_2d_array_v2i64_zero: 3875 case Intrinsic::nvvm_suld_3d_i64_zero: 3876 case Intrinsic::nvvm_suld_3d_v2i64_zero: 3877 Info.opc = getOpcForSurfaceInstr(Intrinsic); 3878 Info.memVT = MVT::i64; 3879 Info.ptrVal = nullptr; 3880 Info.offset = 0; 3881 Info.flags = MachineMemOperand::MOLoad; 3882 Info.align = 16; 3883 return true; 3884 } 3885 return false; 3886 } 3887 3888 /// isLegalAddressingMode - Return true if the addressing mode represented 3889 /// by AM is legal for this target, for a load/store of the specified type. 3890 /// Used to guide target specific optimizations, like loop strength reduction 3891 /// (LoopStrengthReduce.cpp) and memory optimization for address mode 3892 /// (CodeGenPrepare.cpp) 3893 bool NVPTXTargetLowering::isLegalAddressingMode(const DataLayout &DL, 3894 const AddrMode &AM, Type *Ty, 3895 unsigned AS, Instruction *I) const { 3896 // AddrMode - This represents an addressing mode of: 3897 // BaseGV + BaseOffs + BaseReg + Scale*ScaleReg 3898 // 3899 // The legal address modes are 3900 // - [avar] 3901 // - [areg] 3902 // - [areg+immoff] 3903 // - [immAddr] 3904 3905 if (AM.BaseGV) { 3906 return !AM.BaseOffs && !AM.HasBaseReg && !AM.Scale; 3907 } 3908 3909 switch (AM.Scale) { 3910 case 0: // "r", "r+i" or "i" is allowed 3911 break; 3912 case 1: 3913 if (AM.HasBaseReg) // "r+r+i" or "r+r" is not allowed. 3914 return false; 3915 // Otherwise we have r+i. 3916 break; 3917 default: 3918 // No scale > 1 is allowed 3919 return false; 3920 } 3921 return true; 3922 } 3923 3924 //===----------------------------------------------------------------------===// 3925 // NVPTX Inline Assembly Support 3926 //===----------------------------------------------------------------------===// 3927 3928 /// getConstraintType - Given a constraint letter, return the type of 3929 /// constraint it is for this target. 3930 NVPTXTargetLowering::ConstraintType 3931 NVPTXTargetLowering::getConstraintType(StringRef Constraint) const { 3932 if (Constraint.size() == 1) { 3933 switch (Constraint[0]) { 3934 default: 3935 break; 3936 case 'b': 3937 case 'r': 3938 case 'h': 3939 case 'c': 3940 case 'l': 3941 case 'f': 3942 case 'd': 3943 case '0': 3944 case 'N': 3945 return C_RegisterClass; 3946 } 3947 } 3948 return TargetLowering::getConstraintType(Constraint); 3949 } 3950 3951 std::pair<unsigned, const TargetRegisterClass *> 3952 NVPTXTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, 3953 StringRef Constraint, 3954 MVT VT) const { 3955 if (Constraint.size() == 1) { 3956 switch (Constraint[0]) { 3957 case 'b': 3958 return std::make_pair(0U, &NVPTX::Int1RegsRegClass); 3959 case 'c': 3960 return std::make_pair(0U, &NVPTX::Int16RegsRegClass); 3961 case 'h': 3962 return std::make_pair(0U, &NVPTX::Int16RegsRegClass); 3963 case 'r': 3964 return std::make_pair(0U, &NVPTX::Int32RegsRegClass); 3965 case 'l': 3966 case 'N': 3967 return std::make_pair(0U, &NVPTX::Int64RegsRegClass); 3968 case 'f': 3969 return std::make_pair(0U, &NVPTX::Float32RegsRegClass); 3970 case 'd': 3971 return std::make_pair(0U, &NVPTX::Float64RegsRegClass); 3972 } 3973 } 3974 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); 3975 } 3976 3977 //===----------------------------------------------------------------------===// 3978 // NVPTX DAG Combining 3979 //===----------------------------------------------------------------------===// 3980 3981 bool NVPTXTargetLowering::allowFMA(MachineFunction &MF, 3982 CodeGenOpt::Level OptLevel) const { 3983 // Always honor command-line argument 3984 if (FMAContractLevelOpt.getNumOccurrences() > 0) 3985 return FMAContractLevelOpt > 0; 3986 3987 // Do not contract if we're not optimizing the code. 3988 if (OptLevel == 0) 3989 return false; 3990 3991 // Honor TargetOptions flags that explicitly say fusion is okay. 3992 if (MF.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast) 3993 return true; 3994 3995 return allowUnsafeFPMath(MF); 3996 } 3997 3998 bool NVPTXTargetLowering::allowUnsafeFPMath(MachineFunction &MF) const { 3999 // Honor TargetOptions flags that explicitly say unsafe math is okay. 4000 if (MF.getTarget().Options.UnsafeFPMath) 4001 return true; 4002 4003 // Allow unsafe math if unsafe-fp-math attribute explicitly says so. 4004 const Function &F = MF.getFunction(); 4005 if (F.hasFnAttribute("unsafe-fp-math")) { 4006 Attribute Attr = F.getFnAttribute("unsafe-fp-math"); 4007 StringRef Val = Attr.getValueAsString(); 4008 if (Val == "true") 4009 return true; 4010 } 4011 4012 return false; 4013 } 4014 4015 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with 4016 /// operands N0 and N1. This is a helper for PerformADDCombine that is 4017 /// called with the default operands, and if that fails, with commuted 4018 /// operands. 4019 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, 4020 TargetLowering::DAGCombinerInfo &DCI, 4021 const NVPTXSubtarget &Subtarget, 4022 CodeGenOpt::Level OptLevel) { 4023 SelectionDAG &DAG = DCI.DAG; 4024 // Skip non-integer, non-scalar case 4025 EVT VT=N0.getValueType(); 4026 if (VT.isVector()) 4027 return SDValue(); 4028 4029 // fold (add (mul a, b), c) -> (mad a, b, c) 4030 // 4031 if (N0.getOpcode() == ISD::MUL) { 4032 assert (VT.isInteger()); 4033 // For integer: 4034 // Since integer multiply-add costs the same as integer multiply 4035 // but is more costly than integer add, do the fusion only when 4036 // the mul is only used in the add. 4037 if (OptLevel==CodeGenOpt::None || VT != MVT::i32 || 4038 !N0.getNode()->hasOneUse()) 4039 return SDValue(); 4040 4041 // Do the folding 4042 return DAG.getNode(NVPTXISD::IMAD, SDLoc(N), VT, 4043 N0.getOperand(0), N0.getOperand(1), N1); 4044 } 4045 else if (N0.getOpcode() == ISD::FMUL) { 4046 if (VT == MVT::f32 || VT == MVT::f64) { 4047 const auto *TLI = static_cast<const NVPTXTargetLowering *>( 4048 &DAG.getTargetLoweringInfo()); 4049 if (!TLI->allowFMA(DAG.getMachineFunction(), OptLevel)) 4050 return SDValue(); 4051 4052 // For floating point: 4053 // Do the fusion only when the mul has less than 5 uses and all 4054 // are add. 4055 // The heuristic is that if a use is not an add, then that use 4056 // cannot be fused into fma, therefore mul is still needed anyway. 4057 // If there are more than 4 uses, even if they are all add, fusing 4058 // them will increase register pressue. 4059 // 4060 int numUses = 0; 4061 int nonAddCount = 0; 4062 for (SDNode::use_iterator UI = N0.getNode()->use_begin(), 4063 UE = N0.getNode()->use_end(); 4064 UI != UE; ++UI) { 4065 numUses++; 4066 SDNode *User = *UI; 4067 if (User->getOpcode() != ISD::FADD) 4068 ++nonAddCount; 4069 } 4070 if (numUses >= 5) 4071 return SDValue(); 4072 if (nonAddCount) { 4073 int orderNo = N->getIROrder(); 4074 int orderNo2 = N0.getNode()->getIROrder(); 4075 // simple heuristics here for considering potential register 4076 // pressure, the logics here is that the differnce are used 4077 // to measure the distance between def and use, the longer distance 4078 // more likely cause register pressure. 4079 if (orderNo - orderNo2 < 500) 4080 return SDValue(); 4081 4082 // Now, check if at least one of the FMUL's operands is live beyond the node N, 4083 // which guarantees that the FMA will not increase register pressure at node N. 4084 bool opIsLive = false; 4085 const SDNode *left = N0.getOperand(0).getNode(); 4086 const SDNode *right = N0.getOperand(1).getNode(); 4087 4088 if (isa<ConstantSDNode>(left) || isa<ConstantSDNode>(right)) 4089 opIsLive = true; 4090 4091 if (!opIsLive) 4092 for (SDNode::use_iterator UI = left->use_begin(), UE = left->use_end(); UI != UE; ++UI) { 4093 SDNode *User = *UI; 4094 int orderNo3 = User->getIROrder(); 4095 if (orderNo3 > orderNo) { 4096 opIsLive = true; 4097 break; 4098 } 4099 } 4100 4101 if (!opIsLive) 4102 for (SDNode::use_iterator UI = right->use_begin(), UE = right->use_end(); UI != UE; ++UI) { 4103 SDNode *User = *UI; 4104 int orderNo3 = User->getIROrder(); 4105 if (orderNo3 > orderNo) { 4106 opIsLive = true; 4107 break; 4108 } 4109 } 4110 4111 if (!opIsLive) 4112 return SDValue(); 4113 } 4114 4115 return DAG.getNode(ISD::FMA, SDLoc(N), VT, 4116 N0.getOperand(0), N0.getOperand(1), N1); 4117 } 4118 } 4119 4120 return SDValue(); 4121 } 4122 4123 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD. 4124 /// 4125 static SDValue PerformADDCombine(SDNode *N, 4126 TargetLowering::DAGCombinerInfo &DCI, 4127 const NVPTXSubtarget &Subtarget, 4128 CodeGenOpt::Level OptLevel) { 4129 SDValue N0 = N->getOperand(0); 4130 SDValue N1 = N->getOperand(1); 4131 4132 // First try with the default operand order. 4133 if (SDValue Result = 4134 PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget, OptLevel)) 4135 return Result; 4136 4137 // If that didn't work, try again with the operands commuted. 4138 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget, OptLevel); 4139 } 4140 4141 static SDValue PerformANDCombine(SDNode *N, 4142 TargetLowering::DAGCombinerInfo &DCI) { 4143 // The type legalizer turns a vector load of i8 values into a zextload to i16 4144 // registers, optionally ANY_EXTENDs it (if target type is integer), 4145 // and ANDs off the high 8 bits. Since we turn this load into a 4146 // target-specific DAG node, the DAG combiner fails to eliminate these AND 4147 // nodes. Do that here. 4148 SDValue Val = N->getOperand(0); 4149 SDValue Mask = N->getOperand(1); 4150 4151 if (isa<ConstantSDNode>(Val)) { 4152 std::swap(Val, Mask); 4153 } 4154 4155 SDValue AExt; 4156 // Generally, we will see zextload -> IMOV16rr -> ANY_EXTEND -> and 4157 if (Val.getOpcode() == ISD::ANY_EXTEND) { 4158 AExt = Val; 4159 Val = Val->getOperand(0); 4160 } 4161 4162 if (Val->isMachineOpcode() && Val->getMachineOpcode() == NVPTX::IMOV16rr) { 4163 Val = Val->getOperand(0); 4164 } 4165 4166 if (Val->getOpcode() == NVPTXISD::LoadV2 || 4167 Val->getOpcode() == NVPTXISD::LoadV4) { 4168 ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Mask); 4169 if (!MaskCnst) { 4170 // Not an AND with a constant 4171 return SDValue(); 4172 } 4173 4174 uint64_t MaskVal = MaskCnst->getZExtValue(); 4175 if (MaskVal != 0xff) { 4176 // Not an AND that chops off top 8 bits 4177 return SDValue(); 4178 } 4179 4180 MemSDNode *Mem = dyn_cast<MemSDNode>(Val); 4181 if (!Mem) { 4182 // Not a MemSDNode?!? 4183 return SDValue(); 4184 } 4185 4186 EVT MemVT = Mem->getMemoryVT(); 4187 if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8) { 4188 // We only handle the i8 case 4189 return SDValue(); 4190 } 4191 4192 unsigned ExtType = 4193 cast<ConstantSDNode>(Val->getOperand(Val->getNumOperands()-1))-> 4194 getZExtValue(); 4195 if (ExtType == ISD::SEXTLOAD) { 4196 // If for some reason the load is a sextload, the and is needed to zero 4197 // out the high 8 bits 4198 return SDValue(); 4199 } 4200 4201 bool AddTo = false; 4202 if (AExt.getNode() != nullptr) { 4203 // Re-insert the ext as a zext. 4204 Val = DCI.DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), 4205 AExt.getValueType(), Val); 4206 AddTo = true; 4207 } 4208 4209 // If we get here, the AND is unnecessary. Just replace it with the load 4210 DCI.CombineTo(N, Val, AddTo); 4211 } 4212 4213 return SDValue(); 4214 } 4215 4216 static SDValue PerformREMCombine(SDNode *N, 4217 TargetLowering::DAGCombinerInfo &DCI, 4218 CodeGenOpt::Level OptLevel) { 4219 assert(N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM); 4220 4221 // Don't do anything at less than -O2. 4222 if (OptLevel < CodeGenOpt::Default) 4223 return SDValue(); 4224 4225 SelectionDAG &DAG = DCI.DAG; 4226 SDLoc DL(N); 4227 EVT VT = N->getValueType(0); 4228 bool IsSigned = N->getOpcode() == ISD::SREM; 4229 unsigned DivOpc = IsSigned ? ISD::SDIV : ISD::UDIV; 4230 4231 const SDValue &Num = N->getOperand(0); 4232 const SDValue &Den = N->getOperand(1); 4233 4234 for (const SDNode *U : Num->uses()) { 4235 if (U->getOpcode() == DivOpc && U->getOperand(0) == Num && 4236 U->getOperand(1) == Den) { 4237 // Num % Den -> Num - (Num / Den) * Den 4238 return DAG.getNode(ISD::SUB, DL, VT, Num, 4239 DAG.getNode(ISD::MUL, DL, VT, 4240 DAG.getNode(DivOpc, DL, VT, Num, Den), 4241 Den)); 4242 } 4243 } 4244 return SDValue(); 4245 } 4246 4247 enum OperandSignedness { 4248 Signed = 0, 4249 Unsigned, 4250 Unknown 4251 }; 4252 4253 /// IsMulWideOperandDemotable - Checks if the provided DAG node is an operand 4254 /// that can be demoted to \p OptSize bits without loss of information. The 4255 /// signedness of the operand, if determinable, is placed in \p S. 4256 static bool IsMulWideOperandDemotable(SDValue Op, 4257 unsigned OptSize, 4258 OperandSignedness &S) { 4259 S = Unknown; 4260 4261 if (Op.getOpcode() == ISD::SIGN_EXTEND || 4262 Op.getOpcode() == ISD::SIGN_EXTEND_INREG) { 4263 EVT OrigVT = Op.getOperand(0).getValueType(); 4264 if (OrigVT.getSizeInBits() <= OptSize) { 4265 S = Signed; 4266 return true; 4267 } 4268 } else if (Op.getOpcode() == ISD::ZERO_EXTEND) { 4269 EVT OrigVT = Op.getOperand(0).getValueType(); 4270 if (OrigVT.getSizeInBits() <= OptSize) { 4271 S = Unsigned; 4272 return true; 4273 } 4274 } 4275 4276 return false; 4277 } 4278 4279 /// AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can 4280 /// be demoted to \p OptSize bits without loss of information. If the operands 4281 /// contain a constant, it should appear as the RHS operand. The signedness of 4282 /// the operands is placed in \p IsSigned. 4283 static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS, 4284 unsigned OptSize, 4285 bool &IsSigned) { 4286 OperandSignedness LHSSign; 4287 4288 // The LHS operand must be a demotable op 4289 if (!IsMulWideOperandDemotable(LHS, OptSize, LHSSign)) 4290 return false; 4291 4292 // We should have been able to determine the signedness from the LHS 4293 if (LHSSign == Unknown) 4294 return false; 4295 4296 IsSigned = (LHSSign == Signed); 4297 4298 // The RHS can be a demotable op or a constant 4299 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(RHS)) { 4300 const APInt &Val = CI->getAPIntValue(); 4301 if (LHSSign == Unsigned) { 4302 return Val.isIntN(OptSize); 4303 } else { 4304 return Val.isSignedIntN(OptSize); 4305 } 4306 } else { 4307 OperandSignedness RHSSign; 4308 if (!IsMulWideOperandDemotable(RHS, OptSize, RHSSign)) 4309 return false; 4310 4311 return LHSSign == RHSSign; 4312 } 4313 } 4314 4315 /// TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply 4316 /// of M/2 bits that produces an M-bit result (i.e. mul.wide). This transform 4317 /// works on both multiply DAG nodes and SHL DAG nodes with a constant shift 4318 /// amount. 4319 static SDValue TryMULWIDECombine(SDNode *N, 4320 TargetLowering::DAGCombinerInfo &DCI) { 4321 EVT MulType = N->getValueType(0); 4322 if (MulType != MVT::i32 && MulType != MVT::i64) { 4323 return SDValue(); 4324 } 4325 4326 SDLoc DL(N); 4327 unsigned OptSize = MulType.getSizeInBits() >> 1; 4328 SDValue LHS = N->getOperand(0); 4329 SDValue RHS = N->getOperand(1); 4330 4331 // Canonicalize the multiply so the constant (if any) is on the right 4332 if (N->getOpcode() == ISD::MUL) { 4333 if (isa<ConstantSDNode>(LHS)) { 4334 std::swap(LHS, RHS); 4335 } 4336 } 4337 4338 // If we have a SHL, determine the actual multiply amount 4339 if (N->getOpcode() == ISD::SHL) { 4340 ConstantSDNode *ShlRHS = dyn_cast<ConstantSDNode>(RHS); 4341 if (!ShlRHS) { 4342 return SDValue(); 4343 } 4344 4345 APInt ShiftAmt = ShlRHS->getAPIntValue(); 4346 unsigned BitWidth = MulType.getSizeInBits(); 4347 if (ShiftAmt.sge(0) && ShiftAmt.slt(BitWidth)) { 4348 APInt MulVal = APInt(BitWidth, 1) << ShiftAmt; 4349 RHS = DCI.DAG.getConstant(MulVal, DL, MulType); 4350 } else { 4351 return SDValue(); 4352 } 4353 } 4354 4355 bool Signed; 4356 // Verify that our operands are demotable 4357 if (!AreMulWideOperandsDemotable(LHS, RHS, OptSize, Signed)) { 4358 return SDValue(); 4359 } 4360 4361 EVT DemotedVT; 4362 if (MulType == MVT::i32) { 4363 DemotedVT = MVT::i16; 4364 } else { 4365 DemotedVT = MVT::i32; 4366 } 4367 4368 // Truncate the operands to the correct size. Note that these are just for 4369 // type consistency and will (likely) be eliminated in later phases. 4370 SDValue TruncLHS = 4371 DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, LHS); 4372 SDValue TruncRHS = 4373 DCI.DAG.getNode(ISD::TRUNCATE, DL, DemotedVT, RHS); 4374 4375 unsigned Opc; 4376 if (Signed) { 4377 Opc = NVPTXISD::MUL_WIDE_SIGNED; 4378 } else { 4379 Opc = NVPTXISD::MUL_WIDE_UNSIGNED; 4380 } 4381 4382 return DCI.DAG.getNode(Opc, DL, MulType, TruncLHS, TruncRHS); 4383 } 4384 4385 /// PerformMULCombine - Runs PTX-specific DAG combine patterns on MUL nodes. 4386 static SDValue PerformMULCombine(SDNode *N, 4387 TargetLowering::DAGCombinerInfo &DCI, 4388 CodeGenOpt::Level OptLevel) { 4389 if (OptLevel > 0) { 4390 // Try mul.wide combining at OptLevel > 0 4391 if (SDValue Ret = TryMULWIDECombine(N, DCI)) 4392 return Ret; 4393 } 4394 4395 return SDValue(); 4396 } 4397 4398 /// PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes. 4399 static SDValue PerformSHLCombine(SDNode *N, 4400 TargetLowering::DAGCombinerInfo &DCI, 4401 CodeGenOpt::Level OptLevel) { 4402 if (OptLevel > 0) { 4403 // Try mul.wide combining at OptLevel > 0 4404 if (SDValue Ret = TryMULWIDECombine(N, DCI)) 4405 return Ret; 4406 } 4407 4408 return SDValue(); 4409 } 4410 4411 static SDValue PerformSETCCCombine(SDNode *N, 4412 TargetLowering::DAGCombinerInfo &DCI) { 4413 EVT CCType = N->getValueType(0); 4414 SDValue A = N->getOperand(0); 4415 SDValue B = N->getOperand(1); 4416 4417 if (CCType != MVT::v2i1 || A.getValueType() != MVT::v2f16) 4418 return SDValue(); 4419 4420 SDLoc DL(N); 4421 // setp.f16x2 returns two scalar predicates, which we need to 4422 // convert back to v2i1. The returned result will be scalarized by 4423 // the legalizer, but the comparison will remain a single vector 4424 // instruction. 4425 SDValue CCNode = DCI.DAG.getNode(NVPTXISD::SETP_F16X2, DL, 4426 DCI.DAG.getVTList(MVT::i1, MVT::i1), 4427 {A, B, N->getOperand(2)}); 4428 return DCI.DAG.getNode(ISD::BUILD_VECTOR, DL, CCType, CCNode.getValue(0), 4429 CCNode.getValue(1)); 4430 } 4431 4432 SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N, 4433 DAGCombinerInfo &DCI) const { 4434 CodeGenOpt::Level OptLevel = getTargetMachine().getOptLevel(); 4435 switch (N->getOpcode()) { 4436 default: break; 4437 case ISD::ADD: 4438 case ISD::FADD: 4439 return PerformADDCombine(N, DCI, STI, OptLevel); 4440 case ISD::MUL: 4441 return PerformMULCombine(N, DCI, OptLevel); 4442 case ISD::SHL: 4443 return PerformSHLCombine(N, DCI, OptLevel); 4444 case ISD::AND: 4445 return PerformANDCombine(N, DCI); 4446 case ISD::UREM: 4447 case ISD::SREM: 4448 return PerformREMCombine(N, DCI, OptLevel); 4449 case ISD::SETCC: 4450 return PerformSETCCCombine(N, DCI); 4451 } 4452 return SDValue(); 4453 } 4454 4455 /// ReplaceVectorLoad - Convert vector loads into multi-output scalar loads. 4456 static void ReplaceLoadVector(SDNode *N, SelectionDAG &DAG, 4457 SmallVectorImpl<SDValue> &Results) { 4458 EVT ResVT = N->getValueType(0); 4459 SDLoc DL(N); 4460 4461 assert(ResVT.isVector() && "Vector load must have vector type"); 4462 4463 // We only handle "native" vector sizes for now, e.g. <4 x double> is not 4464 // legal. We can (and should) split that into 2 loads of <2 x double> here 4465 // but I'm leaving that as a TODO for now. 4466 assert(ResVT.isSimple() && "Can only handle simple types"); 4467 switch (ResVT.getSimpleVT().SimpleTy) { 4468 default: 4469 return; 4470 case MVT::v2i8: 4471 case MVT::v2i16: 4472 case MVT::v2i32: 4473 case MVT::v2i64: 4474 case MVT::v2f16: 4475 case MVT::v2f32: 4476 case MVT::v2f64: 4477 case MVT::v4i8: 4478 case MVT::v4i16: 4479 case MVT::v4i32: 4480 case MVT::v4f16: 4481 case MVT::v4f32: 4482 case MVT::v8f16: // <4 x f16x2> 4483 // This is a "native" vector type 4484 break; 4485 } 4486 4487 LoadSDNode *LD = cast<LoadSDNode>(N); 4488 4489 unsigned Align = LD->getAlignment(); 4490 auto &TD = DAG.getDataLayout(); 4491 unsigned PrefAlign = 4492 TD.getPrefTypeAlignment(ResVT.getTypeForEVT(*DAG.getContext())); 4493 if (Align < PrefAlign) { 4494 // This load is not sufficiently aligned, so bail out and let this vector 4495 // load be scalarized. Note that we may still be able to emit smaller 4496 // vector loads. For example, if we are loading a <4 x float> with an 4497 // alignment of 8, this check will fail but the legalizer will try again 4498 // with 2 x <2 x float>, which will succeed with an alignment of 8. 4499 return; 4500 } 4501 4502 EVT EltVT = ResVT.getVectorElementType(); 4503 unsigned NumElts = ResVT.getVectorNumElements(); 4504 4505 // Since LoadV2 is a target node, we cannot rely on DAG type legalization. 4506 // Therefore, we must ensure the type is legal. For i1 and i8, we set the 4507 // loaded type to i16 and propagate the "real" type as the memory type. 4508 bool NeedTrunc = false; 4509 if (EltVT.getSizeInBits() < 16) { 4510 EltVT = MVT::i16; 4511 NeedTrunc = true; 4512 } 4513 4514 unsigned Opcode = 0; 4515 SDVTList LdResVTs; 4516 bool LoadF16x2 = false; 4517 4518 switch (NumElts) { 4519 default: 4520 return; 4521 case 2: 4522 Opcode = NVPTXISD::LoadV2; 4523 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other); 4524 break; 4525 case 4: { 4526 Opcode = NVPTXISD::LoadV4; 4527 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other }; 4528 LdResVTs = DAG.getVTList(ListVTs); 4529 break; 4530 } 4531 case 8: { 4532 // v8f16 is a special case. PTX doesn't have ld.v8.f16 4533 // instruction. Instead, we split the vector into v2f16 chunks and 4534 // load them with ld.v4.b32. 4535 assert(EltVT == MVT::f16 && "Unsupported v8 vector type."); 4536 LoadF16x2 = true; 4537 Opcode = NVPTXISD::LoadV4; 4538 EVT ListVTs[] = {MVT::v2f16, MVT::v2f16, MVT::v2f16, MVT::v2f16, 4539 MVT::Other}; 4540 LdResVTs = DAG.getVTList(ListVTs); 4541 break; 4542 } 4543 } 4544 4545 // Copy regular operands 4546 SmallVector<SDValue, 8> OtherOps(N->op_begin(), N->op_end()); 4547 4548 // The select routine does not have access to the LoadSDNode instance, so 4549 // pass along the extension information 4550 OtherOps.push_back(DAG.getIntPtrConstant(LD->getExtensionType(), DL)); 4551 4552 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps, 4553 LD->getMemoryVT(), 4554 LD->getMemOperand()); 4555 4556 SmallVector<SDValue, 8> ScalarRes; 4557 if (LoadF16x2) { 4558 // Split v2f16 subvectors back into individual elements. 4559 NumElts /= 2; 4560 for (unsigned i = 0; i < NumElts; ++i) { 4561 SDValue SubVector = NewLD.getValue(i); 4562 SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SubVector, 4563 DAG.getIntPtrConstant(0, DL)); 4564 SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SubVector, 4565 DAG.getIntPtrConstant(1, DL)); 4566 ScalarRes.push_back(E0); 4567 ScalarRes.push_back(E1); 4568 } 4569 } else { 4570 for (unsigned i = 0; i < NumElts; ++i) { 4571 SDValue Res = NewLD.getValue(i); 4572 if (NeedTrunc) 4573 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res); 4574 ScalarRes.push_back(Res); 4575 } 4576 } 4577 4578 SDValue LoadChain = NewLD.getValue(NumElts); 4579 4580 SDValue BuildVec = DAG.getBuildVector(ResVT, DL, ScalarRes); 4581 4582 Results.push_back(BuildVec); 4583 Results.push_back(LoadChain); 4584 } 4585 4586 static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG, 4587 SmallVectorImpl<SDValue> &Results) { 4588 SDValue Chain = N->getOperand(0); 4589 SDValue Intrin = N->getOperand(1); 4590 SDLoc DL(N); 4591 4592 // Get the intrinsic ID 4593 unsigned IntrinNo = cast<ConstantSDNode>(Intrin.getNode())->getZExtValue(); 4594 switch (IntrinNo) { 4595 default: 4596 return; 4597 case Intrinsic::nvvm_ldg_global_i: 4598 case Intrinsic::nvvm_ldg_global_f: 4599 case Intrinsic::nvvm_ldg_global_p: 4600 case Intrinsic::nvvm_ldu_global_i: 4601 case Intrinsic::nvvm_ldu_global_f: 4602 case Intrinsic::nvvm_ldu_global_p: { 4603 EVT ResVT = N->getValueType(0); 4604 4605 if (ResVT.isVector()) { 4606 // Vector LDG/LDU 4607 4608 unsigned NumElts = ResVT.getVectorNumElements(); 4609 EVT EltVT = ResVT.getVectorElementType(); 4610 4611 // Since LDU/LDG are target nodes, we cannot rely on DAG type 4612 // legalization. 4613 // Therefore, we must ensure the type is legal. For i1 and i8, we set the 4614 // loaded type to i16 and propagate the "real" type as the memory type. 4615 bool NeedTrunc = false; 4616 if (EltVT.getSizeInBits() < 16) { 4617 EltVT = MVT::i16; 4618 NeedTrunc = true; 4619 } 4620 4621 unsigned Opcode = 0; 4622 SDVTList LdResVTs; 4623 4624 switch (NumElts) { 4625 default: 4626 return; 4627 case 2: 4628 switch (IntrinNo) { 4629 default: 4630 return; 4631 case Intrinsic::nvvm_ldg_global_i: 4632 case Intrinsic::nvvm_ldg_global_f: 4633 case Intrinsic::nvvm_ldg_global_p: 4634 Opcode = NVPTXISD::LDGV2; 4635 break; 4636 case Intrinsic::nvvm_ldu_global_i: 4637 case Intrinsic::nvvm_ldu_global_f: 4638 case Intrinsic::nvvm_ldu_global_p: 4639 Opcode = NVPTXISD::LDUV2; 4640 break; 4641 } 4642 LdResVTs = DAG.getVTList(EltVT, EltVT, MVT::Other); 4643 break; 4644 case 4: { 4645 switch (IntrinNo) { 4646 default: 4647 return; 4648 case Intrinsic::nvvm_ldg_global_i: 4649 case Intrinsic::nvvm_ldg_global_f: 4650 case Intrinsic::nvvm_ldg_global_p: 4651 Opcode = NVPTXISD::LDGV4; 4652 break; 4653 case Intrinsic::nvvm_ldu_global_i: 4654 case Intrinsic::nvvm_ldu_global_f: 4655 case Intrinsic::nvvm_ldu_global_p: 4656 Opcode = NVPTXISD::LDUV4; 4657 break; 4658 } 4659 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other }; 4660 LdResVTs = DAG.getVTList(ListVTs); 4661 break; 4662 } 4663 } 4664 4665 SmallVector<SDValue, 8> OtherOps; 4666 4667 // Copy regular operands 4668 4669 OtherOps.push_back(Chain); // Chain 4670 // Skip operand 1 (intrinsic ID) 4671 // Others 4672 OtherOps.append(N->op_begin() + 2, N->op_end()); 4673 4674 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N); 4675 4676 SDValue NewLD = DAG.getMemIntrinsicNode(Opcode, DL, LdResVTs, OtherOps, 4677 MemSD->getMemoryVT(), 4678 MemSD->getMemOperand()); 4679 4680 SmallVector<SDValue, 4> ScalarRes; 4681 4682 for (unsigned i = 0; i < NumElts; ++i) { 4683 SDValue Res = NewLD.getValue(i); 4684 if (NeedTrunc) 4685 Res = 4686 DAG.getNode(ISD::TRUNCATE, DL, ResVT.getVectorElementType(), Res); 4687 ScalarRes.push_back(Res); 4688 } 4689 4690 SDValue LoadChain = NewLD.getValue(NumElts); 4691 4692 SDValue BuildVec = 4693 DAG.getBuildVector(ResVT, DL, ScalarRes); 4694 4695 Results.push_back(BuildVec); 4696 Results.push_back(LoadChain); 4697 } else { 4698 // i8 LDG/LDU 4699 assert(ResVT.isSimple() && ResVT.getSimpleVT().SimpleTy == MVT::i8 && 4700 "Custom handling of non-i8 ldu/ldg?"); 4701 4702 // Just copy all operands as-is 4703 SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end()); 4704 4705 // Force output to i16 4706 SDVTList LdResVTs = DAG.getVTList(MVT::i16, MVT::Other); 4707 4708 MemIntrinsicSDNode *MemSD = cast<MemIntrinsicSDNode>(N); 4709 4710 // We make sure the memory type is i8, which will be used during isel 4711 // to select the proper instruction. 4712 SDValue NewLD = 4713 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, LdResVTs, Ops, 4714 MVT::i8, MemSD->getMemOperand()); 4715 4716 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, 4717 NewLD.getValue(0))); 4718 Results.push_back(NewLD.getValue(1)); 4719 } 4720 } 4721 } 4722 } 4723 4724 void NVPTXTargetLowering::ReplaceNodeResults( 4725 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { 4726 switch (N->getOpcode()) { 4727 default: 4728 report_fatal_error("Unhandled custom legalization"); 4729 case ISD::LOAD: 4730 ReplaceLoadVector(N, DAG, Results); 4731 return; 4732 case ISD::INTRINSIC_W_CHAIN: 4733 ReplaceINTRINSIC_W_CHAIN(N, DAG, Results); 4734 return; 4735 } 4736 } 4737 4738 // Pin NVPTXTargetObjectFile's vtables to this file. 4739 NVPTXTargetObjectFile::~NVPTXTargetObjectFile() {} 4740 4741 MCSection *NVPTXTargetObjectFile::SelectSectionForGlobal( 4742 const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const { 4743 return getDataSection(); 4744 } 4745