1 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines an instruction selector for the NVPTX target. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "NVPTXISelDAGToDAG.h" 15 #include "llvm/IR/GlobalValue.h" 16 #include "llvm/IR/Instructions.h" 17 #include "llvm/Support/CommandLine.h" 18 #include "llvm/Support/Debug.h" 19 #include "llvm/Support/ErrorHandling.h" 20 #include "llvm/Support/raw_ostream.h" 21 #include "llvm/Target/TargetIntrinsicInfo.h" 22 23 #undef DEBUG_TYPE 24 #define DEBUG_TYPE "nvptx-isel" 25 26 using namespace llvm; 27 28 static cl::opt<int> 29 FMAContractLevel("nvptx-fma-level", cl::ZeroOrMore, 30 cl::desc("NVPTX Specific: FMA contraction (0: don't do it" 31 " 1: do it 2: do it aggressively"), 32 cl::init(2)); 33 34 static cl::opt<int> UsePrecDivF32( 35 "nvptx-prec-divf32", cl::ZeroOrMore, 36 cl::desc("NVPTX Specifies: 0 use div.approx, 1 use div.full, 2 use" 37 " IEEE Compliant F32 div.rnd if avaiable."), 38 cl::init(2)); 39 40 static cl::opt<bool> 41 UsePrecSqrtF32("nvptx-prec-sqrtf32", 42 cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), 43 cl::init(true)); 44 45 static cl::opt<bool> 46 FtzEnabled("nvptx-f32ftz", cl::ZeroOrMore, 47 cl::desc("NVPTX Specific: Flush f32 subnormals to sign-preserving zero."), 48 cl::init(false)); 49 50 51 /// createNVPTXISelDag - This pass converts a legalized DAG into a 52 /// NVPTX-specific DAG, ready for instruction scheduling. 53 FunctionPass *llvm::createNVPTXISelDag(NVPTXTargetMachine &TM, 54 llvm::CodeGenOpt::Level OptLevel) { 55 return new NVPTXDAGToDAGISel(TM, OptLevel); 56 } 57 58 NVPTXDAGToDAGISel::NVPTXDAGToDAGISel(NVPTXTargetMachine &tm, 59 CodeGenOpt::Level OptLevel) 60 : SelectionDAGISel(tm, OptLevel), 61 Subtarget(tm.getSubtarget<NVPTXSubtarget>()) { 62 63 doFMAF32 = (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel >= 1); 64 doFMAF64 = (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel >= 1); 65 doFMAF32AGG = 66 (OptLevel > 0) && Subtarget.hasFMAF32() && (FMAContractLevel == 2); 67 doFMAF64AGG = 68 (OptLevel > 0) && Subtarget.hasFMAF64() && (FMAContractLevel == 2); 69 70 allowFMA = (FMAContractLevel >= 1); 71 72 doMulWide = (OptLevel > 0); 73 } 74 75 int NVPTXDAGToDAGISel::getDivF32Level() const { 76 if (UsePrecDivF32.getNumOccurrences() > 0) { 77 // If nvptx-prec-div32=N is used on the command-line, always honor it 78 return UsePrecDivF32; 79 } else { 80 // Otherwise, use div.approx if fast math is enabled 81 if (TM.Options.UnsafeFPMath) 82 return 0; 83 else 84 return 2; 85 } 86 } 87 88 bool NVPTXDAGToDAGISel::usePrecSqrtF32() const { 89 if (UsePrecSqrtF32.getNumOccurrences() > 0) { 90 // If nvptx-prec-sqrtf32 is used on the command-line, always honor it 91 return UsePrecSqrtF32; 92 } else { 93 // Otherwise, use sqrt.approx if fast math is enabled 94 if (TM.Options.UnsafeFPMath) 95 return false; 96 else 97 return true; 98 } 99 } 100 101 bool NVPTXDAGToDAGISel::useF32FTZ() const { 102 if (FtzEnabled.getNumOccurrences() > 0) { 103 // If nvptx-f32ftz is used on the command-line, always honor it 104 return FtzEnabled; 105 } else { 106 const Function *F = MF->getFunction(); 107 // Otherwise, check for an nvptx-f32ftz attribute on the function 108 if (F->hasFnAttribute("nvptx-f32ftz")) 109 return (F->getAttributes().getAttribute(AttributeSet::FunctionIndex, 110 "nvptx-f32ftz") 111 .getValueAsString() == "true"); 112 else 113 return false; 114 } 115 } 116 117 /// Select - Select instructions not customized! Used for 118 /// expanded, promoted and normal instructions. 119 SDNode *NVPTXDAGToDAGISel::Select(SDNode *N) { 120 121 if (N->isMachineOpcode()) 122 return NULL; // Already selected. 123 124 SDNode *ResNode = NULL; 125 switch (N->getOpcode()) { 126 case ISD::LOAD: 127 ResNode = SelectLoad(N); 128 break; 129 case ISD::STORE: 130 ResNode = SelectStore(N); 131 break; 132 case NVPTXISD::LoadV2: 133 case NVPTXISD::LoadV4: 134 ResNode = SelectLoadVector(N); 135 break; 136 case NVPTXISD::LDGV2: 137 case NVPTXISD::LDGV4: 138 case NVPTXISD::LDUV2: 139 case NVPTXISD::LDUV4: 140 ResNode = SelectLDGLDUVector(N); 141 break; 142 case NVPTXISD::StoreV2: 143 case NVPTXISD::StoreV4: 144 ResNode = SelectStoreVector(N); 145 break; 146 case NVPTXISD::LoadParam: 147 case NVPTXISD::LoadParamV2: 148 case NVPTXISD::LoadParamV4: 149 ResNode = SelectLoadParam(N); 150 break; 151 case NVPTXISD::StoreRetval: 152 case NVPTXISD::StoreRetvalV2: 153 case NVPTXISD::StoreRetvalV4: 154 ResNode = SelectStoreRetval(N); 155 break; 156 case NVPTXISD::StoreParam: 157 case NVPTXISD::StoreParamV2: 158 case NVPTXISD::StoreParamV4: 159 case NVPTXISD::StoreParamS32: 160 case NVPTXISD::StoreParamU32: 161 ResNode = SelectStoreParam(N); 162 break; 163 default: 164 break; 165 } 166 if (ResNode) 167 return ResNode; 168 return SelectCode(N); 169 } 170 171 static unsigned int getCodeAddrSpace(MemSDNode *N, 172 const NVPTXSubtarget &Subtarget) { 173 const Value *Src = N->getSrcValue(); 174 175 if (!Src) 176 return NVPTX::PTXLdStInstCode::GENERIC; 177 178 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) { 179 switch (PT->getAddressSpace()) { 180 case llvm::ADDRESS_SPACE_LOCAL: return NVPTX::PTXLdStInstCode::LOCAL; 181 case llvm::ADDRESS_SPACE_GLOBAL: return NVPTX::PTXLdStInstCode::GLOBAL; 182 case llvm::ADDRESS_SPACE_SHARED: return NVPTX::PTXLdStInstCode::SHARED; 183 case llvm::ADDRESS_SPACE_GENERIC: return NVPTX::PTXLdStInstCode::GENERIC; 184 case llvm::ADDRESS_SPACE_PARAM: return NVPTX::PTXLdStInstCode::PARAM; 185 case llvm::ADDRESS_SPACE_CONST: return NVPTX::PTXLdStInstCode::CONSTANT; 186 default: break; 187 } 188 } 189 return NVPTX::PTXLdStInstCode::GENERIC; 190 } 191 192 SDNode *NVPTXDAGToDAGISel::SelectLoad(SDNode *N) { 193 SDLoc dl(N); 194 LoadSDNode *LD = cast<LoadSDNode>(N); 195 EVT LoadedVT = LD->getMemoryVT(); 196 SDNode *NVPTXLD = NULL; 197 198 // do not support pre/post inc/dec 199 if (LD->isIndexed()) 200 return NULL; 201 202 if (!LoadedVT.isSimple()) 203 return NULL; 204 205 // Address Space Setting 206 unsigned int codeAddrSpace = getCodeAddrSpace(LD, Subtarget); 207 208 // Volatile Setting 209 // - .volatile is only availalble for .global and .shared 210 bool isVolatile = LD->isVolatile(); 211 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 212 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 213 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 214 isVolatile = false; 215 216 // Vector Setting 217 MVT SimpleVT = LoadedVT.getSimpleVT(); 218 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 219 if (SimpleVT.isVector()) { 220 unsigned num = SimpleVT.getVectorNumElements(); 221 if (num == 2) 222 vecType = NVPTX::PTXLdStInstCode::V2; 223 else if (num == 4) 224 vecType = NVPTX::PTXLdStInstCode::V4; 225 else 226 return NULL; 227 } 228 229 // Type Setting: fromType + fromTypeWidth 230 // 231 // Sign : ISD::SEXTLOAD 232 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 233 // type is integer 234 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 235 MVT ScalarVT = SimpleVT.getScalarType(); 236 // Read at least 8 bits (predicates are stored as 8-bit values) 237 unsigned fromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 238 unsigned int fromType; 239 if ((LD->getExtensionType() == ISD::SEXTLOAD)) 240 fromType = NVPTX::PTXLdStInstCode::Signed; 241 else if (ScalarVT.isFloatingPoint()) 242 fromType = NVPTX::PTXLdStInstCode::Float; 243 else 244 fromType = NVPTX::PTXLdStInstCode::Unsigned; 245 246 // Create the machine instruction DAG 247 SDValue Chain = N->getOperand(0); 248 SDValue N1 = N->getOperand(1); 249 SDValue Addr; 250 SDValue Offset, Base; 251 unsigned Opcode; 252 MVT::SimpleValueType TargetVT = LD->getValueType(0).getSimpleVT().SimpleTy; 253 254 if (SelectDirectAddr(N1, Addr)) { 255 switch (TargetVT) { 256 case MVT::i8: 257 Opcode = NVPTX::LD_i8_avar; 258 break; 259 case MVT::i16: 260 Opcode = NVPTX::LD_i16_avar; 261 break; 262 case MVT::i32: 263 Opcode = NVPTX::LD_i32_avar; 264 break; 265 case MVT::i64: 266 Opcode = NVPTX::LD_i64_avar; 267 break; 268 case MVT::f32: 269 Opcode = NVPTX::LD_f32_avar; 270 break; 271 case MVT::f64: 272 Opcode = NVPTX::LD_f64_avar; 273 break; 274 default: 275 return NULL; 276 } 277 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 278 getI32Imm(vecType), getI32Imm(fromType), 279 getI32Imm(fromTypeWidth), Addr, Chain }; 280 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 281 } else if (Subtarget.is64Bit() 282 ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) 283 : SelectADDRsi(N1.getNode(), N1, Base, Offset)) { 284 switch (TargetVT) { 285 case MVT::i8: 286 Opcode = NVPTX::LD_i8_asi; 287 break; 288 case MVT::i16: 289 Opcode = NVPTX::LD_i16_asi; 290 break; 291 case MVT::i32: 292 Opcode = NVPTX::LD_i32_asi; 293 break; 294 case MVT::i64: 295 Opcode = NVPTX::LD_i64_asi; 296 break; 297 case MVT::f32: 298 Opcode = NVPTX::LD_f32_asi; 299 break; 300 case MVT::f64: 301 Opcode = NVPTX::LD_f64_asi; 302 break; 303 default: 304 return NULL; 305 } 306 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 307 getI32Imm(vecType), getI32Imm(fromType), 308 getI32Imm(fromTypeWidth), Base, Offset, Chain }; 309 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 310 } else if (Subtarget.is64Bit() 311 ? SelectADDRri64(N1.getNode(), N1, Base, Offset) 312 : SelectADDRri(N1.getNode(), N1, Base, Offset)) { 313 if (Subtarget.is64Bit()) { 314 switch (TargetVT) { 315 case MVT::i8: 316 Opcode = NVPTX::LD_i8_ari_64; 317 break; 318 case MVT::i16: 319 Opcode = NVPTX::LD_i16_ari_64; 320 break; 321 case MVT::i32: 322 Opcode = NVPTX::LD_i32_ari_64; 323 break; 324 case MVT::i64: 325 Opcode = NVPTX::LD_i64_ari_64; 326 break; 327 case MVT::f32: 328 Opcode = NVPTX::LD_f32_ari_64; 329 break; 330 case MVT::f64: 331 Opcode = NVPTX::LD_f64_ari_64; 332 break; 333 default: 334 return NULL; 335 } 336 } else { 337 switch (TargetVT) { 338 case MVT::i8: 339 Opcode = NVPTX::LD_i8_ari; 340 break; 341 case MVT::i16: 342 Opcode = NVPTX::LD_i16_ari; 343 break; 344 case MVT::i32: 345 Opcode = NVPTX::LD_i32_ari; 346 break; 347 case MVT::i64: 348 Opcode = NVPTX::LD_i64_ari; 349 break; 350 case MVT::f32: 351 Opcode = NVPTX::LD_f32_ari; 352 break; 353 case MVT::f64: 354 Opcode = NVPTX::LD_f64_ari; 355 break; 356 default: 357 return NULL; 358 } 359 } 360 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 361 getI32Imm(vecType), getI32Imm(fromType), 362 getI32Imm(fromTypeWidth), Base, Offset, Chain }; 363 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 364 } else { 365 if (Subtarget.is64Bit()) { 366 switch (TargetVT) { 367 case MVT::i8: 368 Opcode = NVPTX::LD_i8_areg_64; 369 break; 370 case MVT::i16: 371 Opcode = NVPTX::LD_i16_areg_64; 372 break; 373 case MVT::i32: 374 Opcode = NVPTX::LD_i32_areg_64; 375 break; 376 case MVT::i64: 377 Opcode = NVPTX::LD_i64_areg_64; 378 break; 379 case MVT::f32: 380 Opcode = NVPTX::LD_f32_areg_64; 381 break; 382 case MVT::f64: 383 Opcode = NVPTX::LD_f64_areg_64; 384 break; 385 default: 386 return NULL; 387 } 388 } else { 389 switch (TargetVT) { 390 case MVT::i8: 391 Opcode = NVPTX::LD_i8_areg; 392 break; 393 case MVT::i16: 394 Opcode = NVPTX::LD_i16_areg; 395 break; 396 case MVT::i32: 397 Opcode = NVPTX::LD_i32_areg; 398 break; 399 case MVT::i64: 400 Opcode = NVPTX::LD_i64_areg; 401 break; 402 case MVT::f32: 403 Opcode = NVPTX::LD_f32_areg; 404 break; 405 case MVT::f64: 406 Opcode = NVPTX::LD_f64_areg; 407 break; 408 default: 409 return NULL; 410 } 411 } 412 SDValue Ops[] = { getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 413 getI32Imm(vecType), getI32Imm(fromType), 414 getI32Imm(fromTypeWidth), N1, Chain }; 415 NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); 416 } 417 418 if (NVPTXLD != NULL) { 419 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 420 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 421 cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); 422 } 423 424 return NVPTXLD; 425 } 426 427 SDNode *NVPTXDAGToDAGISel::SelectLoadVector(SDNode *N) { 428 429 SDValue Chain = N->getOperand(0); 430 SDValue Op1 = N->getOperand(1); 431 SDValue Addr, Offset, Base; 432 unsigned Opcode; 433 SDLoc DL(N); 434 SDNode *LD; 435 MemSDNode *MemSD = cast<MemSDNode>(N); 436 EVT LoadedVT = MemSD->getMemoryVT(); 437 438 if (!LoadedVT.isSimple()) 439 return NULL; 440 441 // Address Space Setting 442 unsigned int CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 443 444 // Volatile Setting 445 // - .volatile is only availalble for .global and .shared 446 bool IsVolatile = MemSD->isVolatile(); 447 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 448 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 449 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 450 IsVolatile = false; 451 452 // Vector Setting 453 MVT SimpleVT = LoadedVT.getSimpleVT(); 454 455 // Type Setting: fromType + fromTypeWidth 456 // 457 // Sign : ISD::SEXTLOAD 458 // Unsign : ISD::ZEXTLOAD, ISD::NON_EXTLOAD or ISD::EXTLOAD and the 459 // type is integer 460 // Float : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float 461 MVT ScalarVT = SimpleVT.getScalarType(); 462 // Read at least 8 bits (predicates are stored as 8-bit values) 463 unsigned FromTypeWidth = std::max(8U, ScalarVT.getSizeInBits()); 464 unsigned int FromType; 465 // The last operand holds the original LoadSDNode::getExtensionType() value 466 unsigned ExtensionType = cast<ConstantSDNode>( 467 N->getOperand(N->getNumOperands() - 1))->getZExtValue(); 468 if (ExtensionType == ISD::SEXTLOAD) 469 FromType = NVPTX::PTXLdStInstCode::Signed; 470 else if (ScalarVT.isFloatingPoint()) 471 FromType = NVPTX::PTXLdStInstCode::Float; 472 else 473 FromType = NVPTX::PTXLdStInstCode::Unsigned; 474 475 unsigned VecType; 476 477 switch (N->getOpcode()) { 478 case NVPTXISD::LoadV2: 479 VecType = NVPTX::PTXLdStInstCode::V2; 480 break; 481 case NVPTXISD::LoadV4: 482 VecType = NVPTX::PTXLdStInstCode::V4; 483 break; 484 default: 485 return NULL; 486 } 487 488 EVT EltVT = N->getValueType(0); 489 490 if (SelectDirectAddr(Op1, Addr)) { 491 switch (N->getOpcode()) { 492 default: 493 return NULL; 494 case NVPTXISD::LoadV2: 495 switch (EltVT.getSimpleVT().SimpleTy) { 496 default: 497 return NULL; 498 case MVT::i8: 499 Opcode = NVPTX::LDV_i8_v2_avar; 500 break; 501 case MVT::i16: 502 Opcode = NVPTX::LDV_i16_v2_avar; 503 break; 504 case MVT::i32: 505 Opcode = NVPTX::LDV_i32_v2_avar; 506 break; 507 case MVT::i64: 508 Opcode = NVPTX::LDV_i64_v2_avar; 509 break; 510 case MVT::f32: 511 Opcode = NVPTX::LDV_f32_v2_avar; 512 break; 513 case MVT::f64: 514 Opcode = NVPTX::LDV_f64_v2_avar; 515 break; 516 } 517 break; 518 case NVPTXISD::LoadV4: 519 switch (EltVT.getSimpleVT().SimpleTy) { 520 default: 521 return NULL; 522 case MVT::i8: 523 Opcode = NVPTX::LDV_i8_v4_avar; 524 break; 525 case MVT::i16: 526 Opcode = NVPTX::LDV_i16_v4_avar; 527 break; 528 case MVT::i32: 529 Opcode = NVPTX::LDV_i32_v4_avar; 530 break; 531 case MVT::f32: 532 Opcode = NVPTX::LDV_f32_v4_avar; 533 break; 534 } 535 break; 536 } 537 538 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 539 getI32Imm(VecType), getI32Imm(FromType), 540 getI32Imm(FromTypeWidth), Addr, Chain }; 541 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 542 } else if (Subtarget.is64Bit() 543 ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) 544 : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { 545 switch (N->getOpcode()) { 546 default: 547 return NULL; 548 case NVPTXISD::LoadV2: 549 switch (EltVT.getSimpleVT().SimpleTy) { 550 default: 551 return NULL; 552 case MVT::i8: 553 Opcode = NVPTX::LDV_i8_v2_asi; 554 break; 555 case MVT::i16: 556 Opcode = NVPTX::LDV_i16_v2_asi; 557 break; 558 case MVT::i32: 559 Opcode = NVPTX::LDV_i32_v2_asi; 560 break; 561 case MVT::i64: 562 Opcode = NVPTX::LDV_i64_v2_asi; 563 break; 564 case MVT::f32: 565 Opcode = NVPTX::LDV_f32_v2_asi; 566 break; 567 case MVT::f64: 568 Opcode = NVPTX::LDV_f64_v2_asi; 569 break; 570 } 571 break; 572 case NVPTXISD::LoadV4: 573 switch (EltVT.getSimpleVT().SimpleTy) { 574 default: 575 return NULL; 576 case MVT::i8: 577 Opcode = NVPTX::LDV_i8_v4_asi; 578 break; 579 case MVT::i16: 580 Opcode = NVPTX::LDV_i16_v4_asi; 581 break; 582 case MVT::i32: 583 Opcode = NVPTX::LDV_i32_v4_asi; 584 break; 585 case MVT::f32: 586 Opcode = NVPTX::LDV_f32_v4_asi; 587 break; 588 } 589 break; 590 } 591 592 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 593 getI32Imm(VecType), getI32Imm(FromType), 594 getI32Imm(FromTypeWidth), Base, Offset, Chain }; 595 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 596 } else if (Subtarget.is64Bit() 597 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 598 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 599 if (Subtarget.is64Bit()) { 600 switch (N->getOpcode()) { 601 default: 602 return NULL; 603 case NVPTXISD::LoadV2: 604 switch (EltVT.getSimpleVT().SimpleTy) { 605 default: 606 return NULL; 607 case MVT::i8: 608 Opcode = NVPTX::LDV_i8_v2_ari_64; 609 break; 610 case MVT::i16: 611 Opcode = NVPTX::LDV_i16_v2_ari_64; 612 break; 613 case MVT::i32: 614 Opcode = NVPTX::LDV_i32_v2_ari_64; 615 break; 616 case MVT::i64: 617 Opcode = NVPTX::LDV_i64_v2_ari_64; 618 break; 619 case MVT::f32: 620 Opcode = NVPTX::LDV_f32_v2_ari_64; 621 break; 622 case MVT::f64: 623 Opcode = NVPTX::LDV_f64_v2_ari_64; 624 break; 625 } 626 break; 627 case NVPTXISD::LoadV4: 628 switch (EltVT.getSimpleVT().SimpleTy) { 629 default: 630 return NULL; 631 case MVT::i8: 632 Opcode = NVPTX::LDV_i8_v4_ari_64; 633 break; 634 case MVT::i16: 635 Opcode = NVPTX::LDV_i16_v4_ari_64; 636 break; 637 case MVT::i32: 638 Opcode = NVPTX::LDV_i32_v4_ari_64; 639 break; 640 case MVT::f32: 641 Opcode = NVPTX::LDV_f32_v4_ari_64; 642 break; 643 } 644 break; 645 } 646 } else { 647 switch (N->getOpcode()) { 648 default: 649 return NULL; 650 case NVPTXISD::LoadV2: 651 switch (EltVT.getSimpleVT().SimpleTy) { 652 default: 653 return NULL; 654 case MVT::i8: 655 Opcode = NVPTX::LDV_i8_v2_ari; 656 break; 657 case MVT::i16: 658 Opcode = NVPTX::LDV_i16_v2_ari; 659 break; 660 case MVT::i32: 661 Opcode = NVPTX::LDV_i32_v2_ari; 662 break; 663 case MVT::i64: 664 Opcode = NVPTX::LDV_i64_v2_ari; 665 break; 666 case MVT::f32: 667 Opcode = NVPTX::LDV_f32_v2_ari; 668 break; 669 case MVT::f64: 670 Opcode = NVPTX::LDV_f64_v2_ari; 671 break; 672 } 673 break; 674 case NVPTXISD::LoadV4: 675 switch (EltVT.getSimpleVT().SimpleTy) { 676 default: 677 return NULL; 678 case MVT::i8: 679 Opcode = NVPTX::LDV_i8_v4_ari; 680 break; 681 case MVT::i16: 682 Opcode = NVPTX::LDV_i16_v4_ari; 683 break; 684 case MVT::i32: 685 Opcode = NVPTX::LDV_i32_v4_ari; 686 break; 687 case MVT::f32: 688 Opcode = NVPTX::LDV_f32_v4_ari; 689 break; 690 } 691 break; 692 } 693 } 694 695 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 696 getI32Imm(VecType), getI32Imm(FromType), 697 getI32Imm(FromTypeWidth), Base, Offset, Chain }; 698 699 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 700 } else { 701 if (Subtarget.is64Bit()) { 702 switch (N->getOpcode()) { 703 default: 704 return NULL; 705 case NVPTXISD::LoadV2: 706 switch (EltVT.getSimpleVT().SimpleTy) { 707 default: 708 return NULL; 709 case MVT::i8: 710 Opcode = NVPTX::LDV_i8_v2_areg_64; 711 break; 712 case MVT::i16: 713 Opcode = NVPTX::LDV_i16_v2_areg_64; 714 break; 715 case MVT::i32: 716 Opcode = NVPTX::LDV_i32_v2_areg_64; 717 break; 718 case MVT::i64: 719 Opcode = NVPTX::LDV_i64_v2_areg_64; 720 break; 721 case MVT::f32: 722 Opcode = NVPTX::LDV_f32_v2_areg_64; 723 break; 724 case MVT::f64: 725 Opcode = NVPTX::LDV_f64_v2_areg_64; 726 break; 727 } 728 break; 729 case NVPTXISD::LoadV4: 730 switch (EltVT.getSimpleVT().SimpleTy) { 731 default: 732 return NULL; 733 case MVT::i8: 734 Opcode = NVPTX::LDV_i8_v4_areg_64; 735 break; 736 case MVT::i16: 737 Opcode = NVPTX::LDV_i16_v4_areg_64; 738 break; 739 case MVT::i32: 740 Opcode = NVPTX::LDV_i32_v4_areg_64; 741 break; 742 case MVT::f32: 743 Opcode = NVPTX::LDV_f32_v4_areg_64; 744 break; 745 } 746 break; 747 } 748 } else { 749 switch (N->getOpcode()) { 750 default: 751 return NULL; 752 case NVPTXISD::LoadV2: 753 switch (EltVT.getSimpleVT().SimpleTy) { 754 default: 755 return NULL; 756 case MVT::i8: 757 Opcode = NVPTX::LDV_i8_v2_areg; 758 break; 759 case MVT::i16: 760 Opcode = NVPTX::LDV_i16_v2_areg; 761 break; 762 case MVT::i32: 763 Opcode = NVPTX::LDV_i32_v2_areg; 764 break; 765 case MVT::i64: 766 Opcode = NVPTX::LDV_i64_v2_areg; 767 break; 768 case MVT::f32: 769 Opcode = NVPTX::LDV_f32_v2_areg; 770 break; 771 case MVT::f64: 772 Opcode = NVPTX::LDV_f64_v2_areg; 773 break; 774 } 775 break; 776 case NVPTXISD::LoadV4: 777 switch (EltVT.getSimpleVT().SimpleTy) { 778 default: 779 return NULL; 780 case MVT::i8: 781 Opcode = NVPTX::LDV_i8_v4_areg; 782 break; 783 case MVT::i16: 784 Opcode = NVPTX::LDV_i16_v4_areg; 785 break; 786 case MVT::i32: 787 Opcode = NVPTX::LDV_i32_v4_areg; 788 break; 789 case MVT::f32: 790 Opcode = NVPTX::LDV_f32_v4_areg; 791 break; 792 } 793 break; 794 } 795 } 796 797 SDValue Ops[] = { getI32Imm(IsVolatile), getI32Imm(CodeAddrSpace), 798 getI32Imm(VecType), getI32Imm(FromType), 799 getI32Imm(FromTypeWidth), Op1, Chain }; 800 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); 801 } 802 803 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 804 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 805 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 806 807 return LD; 808 } 809 810 SDNode *NVPTXDAGToDAGISel::SelectLDGLDUVector(SDNode *N) { 811 812 SDValue Chain = N->getOperand(0); 813 SDValue Op1 = N->getOperand(1); 814 unsigned Opcode; 815 SDLoc DL(N); 816 SDNode *LD; 817 MemSDNode *Mem = cast<MemSDNode>(N); 818 SDValue Base, Offset, Addr; 819 820 EVT EltVT = Mem->getMemoryVT().getVectorElementType(); 821 822 if (SelectDirectAddr(Op1, Addr)) { 823 switch (N->getOpcode()) { 824 default: 825 return NULL; 826 case NVPTXISD::LDGV2: 827 switch (EltVT.getSimpleVT().SimpleTy) { 828 default: 829 return NULL; 830 case MVT::i8: 831 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; 832 break; 833 case MVT::i16: 834 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; 835 break; 836 case MVT::i32: 837 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar; 838 break; 839 case MVT::i64: 840 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar; 841 break; 842 case MVT::f32: 843 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar; 844 break; 845 case MVT::f64: 846 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar; 847 break; 848 } 849 break; 850 case NVPTXISD::LDUV2: 851 switch (EltVT.getSimpleVT().SimpleTy) { 852 default: 853 return NULL; 854 case MVT::i8: 855 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; 856 break; 857 case MVT::i16: 858 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; 859 break; 860 case MVT::i32: 861 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar; 862 break; 863 case MVT::i64: 864 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar; 865 break; 866 case MVT::f32: 867 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar; 868 break; 869 case MVT::f64: 870 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar; 871 break; 872 } 873 break; 874 case NVPTXISD::LDGV4: 875 switch (EltVT.getSimpleVT().SimpleTy) { 876 default: 877 return NULL; 878 case MVT::i8: 879 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; 880 break; 881 case MVT::i16: 882 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar; 883 break; 884 case MVT::i32: 885 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar; 886 break; 887 case MVT::f32: 888 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar; 889 break; 890 } 891 break; 892 case NVPTXISD::LDUV4: 893 switch (EltVT.getSimpleVT().SimpleTy) { 894 default: 895 return NULL; 896 case MVT::i8: 897 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; 898 break; 899 case MVT::i16: 900 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar; 901 break; 902 case MVT::i32: 903 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar; 904 break; 905 case MVT::f32: 906 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar; 907 break; 908 } 909 break; 910 } 911 912 SDValue Ops[] = { Addr, Chain }; 913 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 914 ArrayRef<SDValue>(Ops, 2)); 915 } else if (Subtarget.is64Bit() 916 ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) 917 : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { 918 if (Subtarget.is64Bit()) { 919 switch (N->getOpcode()) { 920 default: 921 return NULL; 922 case NVPTXISD::LDGV2: 923 switch (EltVT.getSimpleVT().SimpleTy) { 924 default: 925 return NULL; 926 case MVT::i8: 927 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; 928 break; 929 case MVT::i16: 930 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; 931 break; 932 case MVT::i32: 933 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64; 934 break; 935 case MVT::i64: 936 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64; 937 break; 938 case MVT::f32: 939 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64; 940 break; 941 case MVT::f64: 942 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64; 943 break; 944 } 945 break; 946 case NVPTXISD::LDUV2: 947 switch (EltVT.getSimpleVT().SimpleTy) { 948 default: 949 return NULL; 950 case MVT::i8: 951 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; 952 break; 953 case MVT::i16: 954 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; 955 break; 956 case MVT::i32: 957 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64; 958 break; 959 case MVT::i64: 960 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64; 961 break; 962 case MVT::f32: 963 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64; 964 break; 965 case MVT::f64: 966 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64; 967 break; 968 } 969 break; 970 case NVPTXISD::LDGV4: 971 switch (EltVT.getSimpleVT().SimpleTy) { 972 default: 973 return NULL; 974 case MVT::i8: 975 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; 976 break; 977 case MVT::i16: 978 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64; 979 break; 980 case MVT::i32: 981 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64; 982 break; 983 case MVT::f32: 984 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64; 985 break; 986 } 987 break; 988 case NVPTXISD::LDUV4: 989 switch (EltVT.getSimpleVT().SimpleTy) { 990 default: 991 return NULL; 992 case MVT::i8: 993 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; 994 break; 995 case MVT::i16: 996 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64; 997 break; 998 case MVT::i32: 999 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64; 1000 break; 1001 case MVT::f32: 1002 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64; 1003 break; 1004 } 1005 break; 1006 } 1007 } else { 1008 switch (N->getOpcode()) { 1009 default: 1010 return NULL; 1011 case NVPTXISD::LDGV2: 1012 switch (EltVT.getSimpleVT().SimpleTy) { 1013 default: 1014 return NULL; 1015 case MVT::i8: 1016 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; 1017 break; 1018 case MVT::i16: 1019 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; 1020 break; 1021 case MVT::i32: 1022 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32; 1023 break; 1024 case MVT::i64: 1025 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32; 1026 break; 1027 case MVT::f32: 1028 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32; 1029 break; 1030 case MVT::f64: 1031 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32; 1032 break; 1033 } 1034 break; 1035 case NVPTXISD::LDUV2: 1036 switch (EltVT.getSimpleVT().SimpleTy) { 1037 default: 1038 return NULL; 1039 case MVT::i8: 1040 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; 1041 break; 1042 case MVT::i16: 1043 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; 1044 break; 1045 case MVT::i32: 1046 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32; 1047 break; 1048 case MVT::i64: 1049 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32; 1050 break; 1051 case MVT::f32: 1052 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32; 1053 break; 1054 case MVT::f64: 1055 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32; 1056 break; 1057 } 1058 break; 1059 case NVPTXISD::LDGV4: 1060 switch (EltVT.getSimpleVT().SimpleTy) { 1061 default: 1062 return NULL; 1063 case MVT::i8: 1064 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; 1065 break; 1066 case MVT::i16: 1067 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32; 1068 break; 1069 case MVT::i32: 1070 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32; 1071 break; 1072 case MVT::f32: 1073 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32; 1074 break; 1075 } 1076 break; 1077 case NVPTXISD::LDUV4: 1078 switch (EltVT.getSimpleVT().SimpleTy) { 1079 default: 1080 return NULL; 1081 case MVT::i8: 1082 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; 1083 break; 1084 case MVT::i16: 1085 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32; 1086 break; 1087 case MVT::i32: 1088 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32; 1089 break; 1090 case MVT::f32: 1091 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32; 1092 break; 1093 } 1094 break; 1095 } 1096 } 1097 1098 SDValue Ops[] = { Base, Offset, Chain }; 1099 1100 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1101 ArrayRef<SDValue>(Ops, 3)); 1102 } else { 1103 if (Subtarget.is64Bit()) { 1104 switch (N->getOpcode()) { 1105 default: 1106 return NULL; 1107 case NVPTXISD::LDGV2: 1108 switch (EltVT.getSimpleVT().SimpleTy) { 1109 default: 1110 return NULL; 1111 case MVT::i8: 1112 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; 1113 break; 1114 case MVT::i16: 1115 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; 1116 break; 1117 case MVT::i32: 1118 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64; 1119 break; 1120 case MVT::i64: 1121 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64; 1122 break; 1123 case MVT::f32: 1124 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64; 1125 break; 1126 case MVT::f64: 1127 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64; 1128 break; 1129 } 1130 break; 1131 case NVPTXISD::LDUV2: 1132 switch (EltVT.getSimpleVT().SimpleTy) { 1133 default: 1134 return NULL; 1135 case MVT::i8: 1136 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; 1137 break; 1138 case MVT::i16: 1139 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; 1140 break; 1141 case MVT::i32: 1142 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64; 1143 break; 1144 case MVT::i64: 1145 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64; 1146 break; 1147 case MVT::f32: 1148 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64; 1149 break; 1150 case MVT::f64: 1151 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64; 1152 break; 1153 } 1154 break; 1155 case NVPTXISD::LDGV4: 1156 switch (EltVT.getSimpleVT().SimpleTy) { 1157 default: 1158 return NULL; 1159 case MVT::i8: 1160 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; 1161 break; 1162 case MVT::i16: 1163 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64; 1164 break; 1165 case MVT::i32: 1166 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64; 1167 break; 1168 case MVT::f32: 1169 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64; 1170 break; 1171 } 1172 break; 1173 case NVPTXISD::LDUV4: 1174 switch (EltVT.getSimpleVT().SimpleTy) { 1175 default: 1176 return NULL; 1177 case MVT::i8: 1178 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; 1179 break; 1180 case MVT::i16: 1181 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64; 1182 break; 1183 case MVT::i32: 1184 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64; 1185 break; 1186 case MVT::f32: 1187 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64; 1188 break; 1189 } 1190 break; 1191 } 1192 } else { 1193 switch (N->getOpcode()) { 1194 default: 1195 return NULL; 1196 case NVPTXISD::LDGV2: 1197 switch (EltVT.getSimpleVT().SimpleTy) { 1198 default: 1199 return NULL; 1200 case MVT::i8: 1201 Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; 1202 break; 1203 case MVT::i16: 1204 Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; 1205 break; 1206 case MVT::i32: 1207 Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32; 1208 break; 1209 case MVT::i64: 1210 Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32; 1211 break; 1212 case MVT::f32: 1213 Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32; 1214 break; 1215 case MVT::f64: 1216 Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32; 1217 break; 1218 } 1219 break; 1220 case NVPTXISD::LDUV2: 1221 switch (EltVT.getSimpleVT().SimpleTy) { 1222 default: 1223 return NULL; 1224 case MVT::i8: 1225 Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; 1226 break; 1227 case MVT::i16: 1228 Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; 1229 break; 1230 case MVT::i32: 1231 Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32; 1232 break; 1233 case MVT::i64: 1234 Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32; 1235 break; 1236 case MVT::f32: 1237 Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32; 1238 break; 1239 case MVT::f64: 1240 Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32; 1241 break; 1242 } 1243 break; 1244 case NVPTXISD::LDGV4: 1245 switch (EltVT.getSimpleVT().SimpleTy) { 1246 default: 1247 return NULL; 1248 case MVT::i8: 1249 Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; 1250 break; 1251 case MVT::i16: 1252 Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32; 1253 break; 1254 case MVT::i32: 1255 Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32; 1256 break; 1257 case MVT::f32: 1258 Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32; 1259 break; 1260 } 1261 break; 1262 case NVPTXISD::LDUV4: 1263 switch (EltVT.getSimpleVT().SimpleTy) { 1264 default: 1265 return NULL; 1266 case MVT::i8: 1267 Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; 1268 break; 1269 case MVT::i16: 1270 Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32; 1271 break; 1272 case MVT::i32: 1273 Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32; 1274 break; 1275 case MVT::f32: 1276 Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32; 1277 break; 1278 } 1279 break; 1280 } 1281 } 1282 1283 SDValue Ops[] = { Op1, Chain }; 1284 LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), 1285 ArrayRef<SDValue>(Ops, 2)); 1286 } 1287 1288 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1289 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1290 cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); 1291 1292 return LD; 1293 } 1294 1295 SDNode *NVPTXDAGToDAGISel::SelectStore(SDNode *N) { 1296 SDLoc dl(N); 1297 StoreSDNode *ST = cast<StoreSDNode>(N); 1298 EVT StoreVT = ST->getMemoryVT(); 1299 SDNode *NVPTXST = NULL; 1300 1301 // do not support pre/post inc/dec 1302 if (ST->isIndexed()) 1303 return NULL; 1304 1305 if (!StoreVT.isSimple()) 1306 return NULL; 1307 1308 // Address Space Setting 1309 unsigned int codeAddrSpace = getCodeAddrSpace(ST, Subtarget); 1310 1311 // Volatile Setting 1312 // - .volatile is only availalble for .global and .shared 1313 bool isVolatile = ST->isVolatile(); 1314 if (codeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1315 codeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1316 codeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1317 isVolatile = false; 1318 1319 // Vector Setting 1320 MVT SimpleVT = StoreVT.getSimpleVT(); 1321 unsigned vecType = NVPTX::PTXLdStInstCode::Scalar; 1322 if (SimpleVT.isVector()) { 1323 unsigned num = SimpleVT.getVectorNumElements(); 1324 if (num == 2) 1325 vecType = NVPTX::PTXLdStInstCode::V2; 1326 else if (num == 4) 1327 vecType = NVPTX::PTXLdStInstCode::V4; 1328 else 1329 return NULL; 1330 } 1331 1332 // Type Setting: toType + toTypeWidth 1333 // - for integer type, always use 'u' 1334 // 1335 MVT ScalarVT = SimpleVT.getScalarType(); 1336 unsigned toTypeWidth = ScalarVT.getSizeInBits(); 1337 unsigned int toType; 1338 if (ScalarVT.isFloatingPoint()) 1339 toType = NVPTX::PTXLdStInstCode::Float; 1340 else 1341 toType = NVPTX::PTXLdStInstCode::Unsigned; 1342 1343 // Create the machine instruction DAG 1344 SDValue Chain = N->getOperand(0); 1345 SDValue N1 = N->getOperand(1); 1346 SDValue N2 = N->getOperand(2); 1347 SDValue Addr; 1348 SDValue Offset, Base; 1349 unsigned Opcode; 1350 MVT::SimpleValueType SourceVT = 1351 N1.getNode()->getValueType(0).getSimpleVT().SimpleTy; 1352 1353 if (SelectDirectAddr(N2, Addr)) { 1354 switch (SourceVT) { 1355 case MVT::i8: 1356 Opcode = NVPTX::ST_i8_avar; 1357 break; 1358 case MVT::i16: 1359 Opcode = NVPTX::ST_i16_avar; 1360 break; 1361 case MVT::i32: 1362 Opcode = NVPTX::ST_i32_avar; 1363 break; 1364 case MVT::i64: 1365 Opcode = NVPTX::ST_i64_avar; 1366 break; 1367 case MVT::f32: 1368 Opcode = NVPTX::ST_f32_avar; 1369 break; 1370 case MVT::f64: 1371 Opcode = NVPTX::ST_f64_avar; 1372 break; 1373 default: 1374 return NULL; 1375 } 1376 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1377 getI32Imm(vecType), getI32Imm(toType), 1378 getI32Imm(toTypeWidth), Addr, Chain }; 1379 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1380 } else if (Subtarget.is64Bit() 1381 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1382 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1383 switch (SourceVT) { 1384 case MVT::i8: 1385 Opcode = NVPTX::ST_i8_asi; 1386 break; 1387 case MVT::i16: 1388 Opcode = NVPTX::ST_i16_asi; 1389 break; 1390 case MVT::i32: 1391 Opcode = NVPTX::ST_i32_asi; 1392 break; 1393 case MVT::i64: 1394 Opcode = NVPTX::ST_i64_asi; 1395 break; 1396 case MVT::f32: 1397 Opcode = NVPTX::ST_f32_asi; 1398 break; 1399 case MVT::f64: 1400 Opcode = NVPTX::ST_f64_asi; 1401 break; 1402 default: 1403 return NULL; 1404 } 1405 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1406 getI32Imm(vecType), getI32Imm(toType), 1407 getI32Imm(toTypeWidth), Base, Offset, Chain }; 1408 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1409 } else if (Subtarget.is64Bit() 1410 ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1411 : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1412 if (Subtarget.is64Bit()) { 1413 switch (SourceVT) { 1414 case MVT::i8: 1415 Opcode = NVPTX::ST_i8_ari_64; 1416 break; 1417 case MVT::i16: 1418 Opcode = NVPTX::ST_i16_ari_64; 1419 break; 1420 case MVT::i32: 1421 Opcode = NVPTX::ST_i32_ari_64; 1422 break; 1423 case MVT::i64: 1424 Opcode = NVPTX::ST_i64_ari_64; 1425 break; 1426 case MVT::f32: 1427 Opcode = NVPTX::ST_f32_ari_64; 1428 break; 1429 case MVT::f64: 1430 Opcode = NVPTX::ST_f64_ari_64; 1431 break; 1432 default: 1433 return NULL; 1434 } 1435 } else { 1436 switch (SourceVT) { 1437 case MVT::i8: 1438 Opcode = NVPTX::ST_i8_ari; 1439 break; 1440 case MVT::i16: 1441 Opcode = NVPTX::ST_i16_ari; 1442 break; 1443 case MVT::i32: 1444 Opcode = NVPTX::ST_i32_ari; 1445 break; 1446 case MVT::i64: 1447 Opcode = NVPTX::ST_i64_ari; 1448 break; 1449 case MVT::f32: 1450 Opcode = NVPTX::ST_f32_ari; 1451 break; 1452 case MVT::f64: 1453 Opcode = NVPTX::ST_f64_ari; 1454 break; 1455 default: 1456 return NULL; 1457 } 1458 } 1459 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1460 getI32Imm(vecType), getI32Imm(toType), 1461 getI32Imm(toTypeWidth), Base, Offset, Chain }; 1462 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1463 } else { 1464 if (Subtarget.is64Bit()) { 1465 switch (SourceVT) { 1466 case MVT::i8: 1467 Opcode = NVPTX::ST_i8_areg_64; 1468 break; 1469 case MVT::i16: 1470 Opcode = NVPTX::ST_i16_areg_64; 1471 break; 1472 case MVT::i32: 1473 Opcode = NVPTX::ST_i32_areg_64; 1474 break; 1475 case MVT::i64: 1476 Opcode = NVPTX::ST_i64_areg_64; 1477 break; 1478 case MVT::f32: 1479 Opcode = NVPTX::ST_f32_areg_64; 1480 break; 1481 case MVT::f64: 1482 Opcode = NVPTX::ST_f64_areg_64; 1483 break; 1484 default: 1485 return NULL; 1486 } 1487 } else { 1488 switch (SourceVT) { 1489 case MVT::i8: 1490 Opcode = NVPTX::ST_i8_areg; 1491 break; 1492 case MVT::i16: 1493 Opcode = NVPTX::ST_i16_areg; 1494 break; 1495 case MVT::i32: 1496 Opcode = NVPTX::ST_i32_areg; 1497 break; 1498 case MVT::i64: 1499 Opcode = NVPTX::ST_i64_areg; 1500 break; 1501 case MVT::f32: 1502 Opcode = NVPTX::ST_f32_areg; 1503 break; 1504 case MVT::f64: 1505 Opcode = NVPTX::ST_f64_areg; 1506 break; 1507 default: 1508 return NULL; 1509 } 1510 } 1511 SDValue Ops[] = { N1, getI32Imm(isVolatile), getI32Imm(codeAddrSpace), 1512 getI32Imm(vecType), getI32Imm(toType), 1513 getI32Imm(toTypeWidth), N2, Chain }; 1514 NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); 1515 } 1516 1517 if (NVPTXST != NULL) { 1518 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1519 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1520 cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1521 } 1522 1523 return NVPTXST; 1524 } 1525 1526 SDNode *NVPTXDAGToDAGISel::SelectStoreVector(SDNode *N) { 1527 SDValue Chain = N->getOperand(0); 1528 SDValue Op1 = N->getOperand(1); 1529 SDValue Addr, Offset, Base; 1530 unsigned Opcode; 1531 SDLoc DL(N); 1532 SDNode *ST; 1533 EVT EltVT = Op1.getValueType(); 1534 MemSDNode *MemSD = cast<MemSDNode>(N); 1535 EVT StoreVT = MemSD->getMemoryVT(); 1536 1537 // Address Space Setting 1538 unsigned CodeAddrSpace = getCodeAddrSpace(MemSD, Subtarget); 1539 1540 if (CodeAddrSpace == NVPTX::PTXLdStInstCode::CONSTANT) { 1541 report_fatal_error("Cannot store to pointer that points to constant " 1542 "memory space"); 1543 } 1544 1545 // Volatile Setting 1546 // - .volatile is only availalble for .global and .shared 1547 bool IsVolatile = MemSD->isVolatile(); 1548 if (CodeAddrSpace != NVPTX::PTXLdStInstCode::GLOBAL && 1549 CodeAddrSpace != NVPTX::PTXLdStInstCode::SHARED && 1550 CodeAddrSpace != NVPTX::PTXLdStInstCode::GENERIC) 1551 IsVolatile = false; 1552 1553 // Type Setting: toType + toTypeWidth 1554 // - for integer type, always use 'u' 1555 assert(StoreVT.isSimple() && "Store value is not simple"); 1556 MVT ScalarVT = StoreVT.getSimpleVT().getScalarType(); 1557 unsigned ToTypeWidth = ScalarVT.getSizeInBits(); 1558 unsigned ToType; 1559 if (ScalarVT.isFloatingPoint()) 1560 ToType = NVPTX::PTXLdStInstCode::Float; 1561 else 1562 ToType = NVPTX::PTXLdStInstCode::Unsigned; 1563 1564 SmallVector<SDValue, 12> StOps; 1565 SDValue N2; 1566 unsigned VecType; 1567 1568 switch (N->getOpcode()) { 1569 case NVPTXISD::StoreV2: 1570 VecType = NVPTX::PTXLdStInstCode::V2; 1571 StOps.push_back(N->getOperand(1)); 1572 StOps.push_back(N->getOperand(2)); 1573 N2 = N->getOperand(3); 1574 break; 1575 case NVPTXISD::StoreV4: 1576 VecType = NVPTX::PTXLdStInstCode::V4; 1577 StOps.push_back(N->getOperand(1)); 1578 StOps.push_back(N->getOperand(2)); 1579 StOps.push_back(N->getOperand(3)); 1580 StOps.push_back(N->getOperand(4)); 1581 N2 = N->getOperand(5); 1582 break; 1583 default: 1584 return NULL; 1585 } 1586 1587 StOps.push_back(getI32Imm(IsVolatile)); 1588 StOps.push_back(getI32Imm(CodeAddrSpace)); 1589 StOps.push_back(getI32Imm(VecType)); 1590 StOps.push_back(getI32Imm(ToType)); 1591 StOps.push_back(getI32Imm(ToTypeWidth)); 1592 1593 if (SelectDirectAddr(N2, Addr)) { 1594 switch (N->getOpcode()) { 1595 default: 1596 return NULL; 1597 case NVPTXISD::StoreV2: 1598 switch (EltVT.getSimpleVT().SimpleTy) { 1599 default: 1600 return NULL; 1601 case MVT::i8: 1602 Opcode = NVPTX::STV_i8_v2_avar; 1603 break; 1604 case MVT::i16: 1605 Opcode = NVPTX::STV_i16_v2_avar; 1606 break; 1607 case MVT::i32: 1608 Opcode = NVPTX::STV_i32_v2_avar; 1609 break; 1610 case MVT::i64: 1611 Opcode = NVPTX::STV_i64_v2_avar; 1612 break; 1613 case MVT::f32: 1614 Opcode = NVPTX::STV_f32_v2_avar; 1615 break; 1616 case MVT::f64: 1617 Opcode = NVPTX::STV_f64_v2_avar; 1618 break; 1619 } 1620 break; 1621 case NVPTXISD::StoreV4: 1622 switch (EltVT.getSimpleVT().SimpleTy) { 1623 default: 1624 return NULL; 1625 case MVT::i8: 1626 Opcode = NVPTX::STV_i8_v4_avar; 1627 break; 1628 case MVT::i16: 1629 Opcode = NVPTX::STV_i16_v4_avar; 1630 break; 1631 case MVT::i32: 1632 Opcode = NVPTX::STV_i32_v4_avar; 1633 break; 1634 case MVT::f32: 1635 Opcode = NVPTX::STV_f32_v4_avar; 1636 break; 1637 } 1638 break; 1639 } 1640 StOps.push_back(Addr); 1641 } else if (Subtarget.is64Bit() 1642 ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) 1643 : SelectADDRsi(N2.getNode(), N2, Base, Offset)) { 1644 switch (N->getOpcode()) { 1645 default: 1646 return NULL; 1647 case NVPTXISD::StoreV2: 1648 switch (EltVT.getSimpleVT().SimpleTy) { 1649 default: 1650 return NULL; 1651 case MVT::i8: 1652 Opcode = NVPTX::STV_i8_v2_asi; 1653 break; 1654 case MVT::i16: 1655 Opcode = NVPTX::STV_i16_v2_asi; 1656 break; 1657 case MVT::i32: 1658 Opcode = NVPTX::STV_i32_v2_asi; 1659 break; 1660 case MVT::i64: 1661 Opcode = NVPTX::STV_i64_v2_asi; 1662 break; 1663 case MVT::f32: 1664 Opcode = NVPTX::STV_f32_v2_asi; 1665 break; 1666 case MVT::f64: 1667 Opcode = NVPTX::STV_f64_v2_asi; 1668 break; 1669 } 1670 break; 1671 case NVPTXISD::StoreV4: 1672 switch (EltVT.getSimpleVT().SimpleTy) { 1673 default: 1674 return NULL; 1675 case MVT::i8: 1676 Opcode = NVPTX::STV_i8_v4_asi; 1677 break; 1678 case MVT::i16: 1679 Opcode = NVPTX::STV_i16_v4_asi; 1680 break; 1681 case MVT::i32: 1682 Opcode = NVPTX::STV_i32_v4_asi; 1683 break; 1684 case MVT::f32: 1685 Opcode = NVPTX::STV_f32_v4_asi; 1686 break; 1687 } 1688 break; 1689 } 1690 StOps.push_back(Base); 1691 StOps.push_back(Offset); 1692 } else if (Subtarget.is64Bit() 1693 ? SelectADDRri64(N2.getNode(), N2, Base, Offset) 1694 : SelectADDRri(N2.getNode(), N2, Base, Offset)) { 1695 if (Subtarget.is64Bit()) { 1696 switch (N->getOpcode()) { 1697 default: 1698 return NULL; 1699 case NVPTXISD::StoreV2: 1700 switch (EltVT.getSimpleVT().SimpleTy) { 1701 default: 1702 return NULL; 1703 case MVT::i8: 1704 Opcode = NVPTX::STV_i8_v2_ari_64; 1705 break; 1706 case MVT::i16: 1707 Opcode = NVPTX::STV_i16_v2_ari_64; 1708 break; 1709 case MVT::i32: 1710 Opcode = NVPTX::STV_i32_v2_ari_64; 1711 break; 1712 case MVT::i64: 1713 Opcode = NVPTX::STV_i64_v2_ari_64; 1714 break; 1715 case MVT::f32: 1716 Opcode = NVPTX::STV_f32_v2_ari_64; 1717 break; 1718 case MVT::f64: 1719 Opcode = NVPTX::STV_f64_v2_ari_64; 1720 break; 1721 } 1722 break; 1723 case NVPTXISD::StoreV4: 1724 switch (EltVT.getSimpleVT().SimpleTy) { 1725 default: 1726 return NULL; 1727 case MVT::i8: 1728 Opcode = NVPTX::STV_i8_v4_ari_64; 1729 break; 1730 case MVT::i16: 1731 Opcode = NVPTX::STV_i16_v4_ari_64; 1732 break; 1733 case MVT::i32: 1734 Opcode = NVPTX::STV_i32_v4_ari_64; 1735 break; 1736 case MVT::f32: 1737 Opcode = NVPTX::STV_f32_v4_ari_64; 1738 break; 1739 } 1740 break; 1741 } 1742 } else { 1743 switch (N->getOpcode()) { 1744 default: 1745 return NULL; 1746 case NVPTXISD::StoreV2: 1747 switch (EltVT.getSimpleVT().SimpleTy) { 1748 default: 1749 return NULL; 1750 case MVT::i8: 1751 Opcode = NVPTX::STV_i8_v2_ari; 1752 break; 1753 case MVT::i16: 1754 Opcode = NVPTX::STV_i16_v2_ari; 1755 break; 1756 case MVT::i32: 1757 Opcode = NVPTX::STV_i32_v2_ari; 1758 break; 1759 case MVT::i64: 1760 Opcode = NVPTX::STV_i64_v2_ari; 1761 break; 1762 case MVT::f32: 1763 Opcode = NVPTX::STV_f32_v2_ari; 1764 break; 1765 case MVT::f64: 1766 Opcode = NVPTX::STV_f64_v2_ari; 1767 break; 1768 } 1769 break; 1770 case NVPTXISD::StoreV4: 1771 switch (EltVT.getSimpleVT().SimpleTy) { 1772 default: 1773 return NULL; 1774 case MVT::i8: 1775 Opcode = NVPTX::STV_i8_v4_ari; 1776 break; 1777 case MVT::i16: 1778 Opcode = NVPTX::STV_i16_v4_ari; 1779 break; 1780 case MVT::i32: 1781 Opcode = NVPTX::STV_i32_v4_ari; 1782 break; 1783 case MVT::f32: 1784 Opcode = NVPTX::STV_f32_v4_ari; 1785 break; 1786 } 1787 break; 1788 } 1789 } 1790 StOps.push_back(Base); 1791 StOps.push_back(Offset); 1792 } else { 1793 if (Subtarget.is64Bit()) { 1794 switch (N->getOpcode()) { 1795 default: 1796 return NULL; 1797 case NVPTXISD::StoreV2: 1798 switch (EltVT.getSimpleVT().SimpleTy) { 1799 default: 1800 return NULL; 1801 case MVT::i8: 1802 Opcode = NVPTX::STV_i8_v2_areg_64; 1803 break; 1804 case MVT::i16: 1805 Opcode = NVPTX::STV_i16_v2_areg_64; 1806 break; 1807 case MVT::i32: 1808 Opcode = NVPTX::STV_i32_v2_areg_64; 1809 break; 1810 case MVT::i64: 1811 Opcode = NVPTX::STV_i64_v2_areg_64; 1812 break; 1813 case MVT::f32: 1814 Opcode = NVPTX::STV_f32_v2_areg_64; 1815 break; 1816 case MVT::f64: 1817 Opcode = NVPTX::STV_f64_v2_areg_64; 1818 break; 1819 } 1820 break; 1821 case NVPTXISD::StoreV4: 1822 switch (EltVT.getSimpleVT().SimpleTy) { 1823 default: 1824 return NULL; 1825 case MVT::i8: 1826 Opcode = NVPTX::STV_i8_v4_areg_64; 1827 break; 1828 case MVT::i16: 1829 Opcode = NVPTX::STV_i16_v4_areg_64; 1830 break; 1831 case MVT::i32: 1832 Opcode = NVPTX::STV_i32_v4_areg_64; 1833 break; 1834 case MVT::f32: 1835 Opcode = NVPTX::STV_f32_v4_areg_64; 1836 break; 1837 } 1838 break; 1839 } 1840 } else { 1841 switch (N->getOpcode()) { 1842 default: 1843 return NULL; 1844 case NVPTXISD::StoreV2: 1845 switch (EltVT.getSimpleVT().SimpleTy) { 1846 default: 1847 return NULL; 1848 case MVT::i8: 1849 Opcode = NVPTX::STV_i8_v2_areg; 1850 break; 1851 case MVT::i16: 1852 Opcode = NVPTX::STV_i16_v2_areg; 1853 break; 1854 case MVT::i32: 1855 Opcode = NVPTX::STV_i32_v2_areg; 1856 break; 1857 case MVT::i64: 1858 Opcode = NVPTX::STV_i64_v2_areg; 1859 break; 1860 case MVT::f32: 1861 Opcode = NVPTX::STV_f32_v2_areg; 1862 break; 1863 case MVT::f64: 1864 Opcode = NVPTX::STV_f64_v2_areg; 1865 break; 1866 } 1867 break; 1868 case NVPTXISD::StoreV4: 1869 switch (EltVT.getSimpleVT().SimpleTy) { 1870 default: 1871 return NULL; 1872 case MVT::i8: 1873 Opcode = NVPTX::STV_i8_v4_areg; 1874 break; 1875 case MVT::i16: 1876 Opcode = NVPTX::STV_i16_v4_areg; 1877 break; 1878 case MVT::i32: 1879 Opcode = NVPTX::STV_i32_v4_areg; 1880 break; 1881 case MVT::f32: 1882 Opcode = NVPTX::STV_f32_v4_areg; 1883 break; 1884 } 1885 break; 1886 } 1887 } 1888 StOps.push_back(N2); 1889 } 1890 1891 StOps.push_back(Chain); 1892 1893 ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); 1894 1895 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 1896 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 1897 cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); 1898 1899 return ST; 1900 } 1901 1902 SDNode *NVPTXDAGToDAGISel::SelectLoadParam(SDNode *Node) { 1903 SDValue Chain = Node->getOperand(0); 1904 SDValue Offset = Node->getOperand(2); 1905 SDValue Flag = Node->getOperand(3); 1906 SDLoc DL(Node); 1907 MemSDNode *Mem = cast<MemSDNode>(Node); 1908 1909 unsigned VecSize; 1910 switch (Node->getOpcode()) { 1911 default: 1912 return NULL; 1913 case NVPTXISD::LoadParam: 1914 VecSize = 1; 1915 break; 1916 case NVPTXISD::LoadParamV2: 1917 VecSize = 2; 1918 break; 1919 case NVPTXISD::LoadParamV4: 1920 VecSize = 4; 1921 break; 1922 } 1923 1924 EVT EltVT = Node->getValueType(0); 1925 EVT MemVT = Mem->getMemoryVT(); 1926 1927 unsigned Opc = 0; 1928 1929 switch (VecSize) { 1930 default: 1931 return NULL; 1932 case 1: 1933 switch (MemVT.getSimpleVT().SimpleTy) { 1934 default: 1935 return NULL; 1936 case MVT::i1: 1937 Opc = NVPTX::LoadParamMemI8; 1938 break; 1939 case MVT::i8: 1940 Opc = NVPTX::LoadParamMemI8; 1941 break; 1942 case MVT::i16: 1943 Opc = NVPTX::LoadParamMemI16; 1944 break; 1945 case MVT::i32: 1946 Opc = NVPTX::LoadParamMemI32; 1947 break; 1948 case MVT::i64: 1949 Opc = NVPTX::LoadParamMemI64; 1950 break; 1951 case MVT::f32: 1952 Opc = NVPTX::LoadParamMemF32; 1953 break; 1954 case MVT::f64: 1955 Opc = NVPTX::LoadParamMemF64; 1956 break; 1957 } 1958 break; 1959 case 2: 1960 switch (MemVT.getSimpleVT().SimpleTy) { 1961 default: 1962 return NULL; 1963 case MVT::i1: 1964 Opc = NVPTX::LoadParamMemV2I8; 1965 break; 1966 case MVT::i8: 1967 Opc = NVPTX::LoadParamMemV2I8; 1968 break; 1969 case MVT::i16: 1970 Opc = NVPTX::LoadParamMemV2I16; 1971 break; 1972 case MVT::i32: 1973 Opc = NVPTX::LoadParamMemV2I32; 1974 break; 1975 case MVT::i64: 1976 Opc = NVPTX::LoadParamMemV2I64; 1977 break; 1978 case MVT::f32: 1979 Opc = NVPTX::LoadParamMemV2F32; 1980 break; 1981 case MVT::f64: 1982 Opc = NVPTX::LoadParamMemV2F64; 1983 break; 1984 } 1985 break; 1986 case 4: 1987 switch (MemVT.getSimpleVT().SimpleTy) { 1988 default: 1989 return NULL; 1990 case MVT::i1: 1991 Opc = NVPTX::LoadParamMemV4I8; 1992 break; 1993 case MVT::i8: 1994 Opc = NVPTX::LoadParamMemV4I8; 1995 break; 1996 case MVT::i16: 1997 Opc = NVPTX::LoadParamMemV4I16; 1998 break; 1999 case MVT::i32: 2000 Opc = NVPTX::LoadParamMemV4I32; 2001 break; 2002 case MVT::f32: 2003 Opc = NVPTX::LoadParamMemV4F32; 2004 break; 2005 } 2006 break; 2007 } 2008 2009 SDVTList VTs; 2010 if (VecSize == 1) { 2011 VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); 2012 } else if (VecSize == 2) { 2013 VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); 2014 } else { 2015 EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; 2016 VTs = CurDAG->getVTList(&EVTs[0], 5); 2017 } 2018 2019 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2020 2021 SmallVector<SDValue, 2> Ops; 2022 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2023 Ops.push_back(Chain); 2024 Ops.push_back(Flag); 2025 2026 SDNode *Ret = 2027 CurDAG->getMachineNode(Opc, DL, VTs, Ops); 2028 return Ret; 2029 } 2030 2031 SDNode *NVPTXDAGToDAGISel::SelectStoreRetval(SDNode *N) { 2032 SDLoc DL(N); 2033 SDValue Chain = N->getOperand(0); 2034 SDValue Offset = N->getOperand(1); 2035 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2036 MemSDNode *Mem = cast<MemSDNode>(N); 2037 2038 // How many elements do we have? 2039 unsigned NumElts = 1; 2040 switch (N->getOpcode()) { 2041 default: 2042 return NULL; 2043 case NVPTXISD::StoreRetval: 2044 NumElts = 1; 2045 break; 2046 case NVPTXISD::StoreRetvalV2: 2047 NumElts = 2; 2048 break; 2049 case NVPTXISD::StoreRetvalV4: 2050 NumElts = 4; 2051 break; 2052 } 2053 2054 // Build vector of operands 2055 SmallVector<SDValue, 6> Ops; 2056 for (unsigned i = 0; i < NumElts; ++i) 2057 Ops.push_back(N->getOperand(i + 2)); 2058 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2059 Ops.push_back(Chain); 2060 2061 // Determine target opcode 2062 // If we have an i1, use an 8-bit store. The lowering code in 2063 // NVPTXISelLowering will have already emitted an upcast. 2064 unsigned Opcode = 0; 2065 switch (NumElts) { 2066 default: 2067 return NULL; 2068 case 1: 2069 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2070 default: 2071 return NULL; 2072 case MVT::i1: 2073 Opcode = NVPTX::StoreRetvalI8; 2074 break; 2075 case MVT::i8: 2076 Opcode = NVPTX::StoreRetvalI8; 2077 break; 2078 case MVT::i16: 2079 Opcode = NVPTX::StoreRetvalI16; 2080 break; 2081 case MVT::i32: 2082 Opcode = NVPTX::StoreRetvalI32; 2083 break; 2084 case MVT::i64: 2085 Opcode = NVPTX::StoreRetvalI64; 2086 break; 2087 case MVT::f32: 2088 Opcode = NVPTX::StoreRetvalF32; 2089 break; 2090 case MVT::f64: 2091 Opcode = NVPTX::StoreRetvalF64; 2092 break; 2093 } 2094 break; 2095 case 2: 2096 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2097 default: 2098 return NULL; 2099 case MVT::i1: 2100 Opcode = NVPTX::StoreRetvalV2I8; 2101 break; 2102 case MVT::i8: 2103 Opcode = NVPTX::StoreRetvalV2I8; 2104 break; 2105 case MVT::i16: 2106 Opcode = NVPTX::StoreRetvalV2I16; 2107 break; 2108 case MVT::i32: 2109 Opcode = NVPTX::StoreRetvalV2I32; 2110 break; 2111 case MVT::i64: 2112 Opcode = NVPTX::StoreRetvalV2I64; 2113 break; 2114 case MVT::f32: 2115 Opcode = NVPTX::StoreRetvalV2F32; 2116 break; 2117 case MVT::f64: 2118 Opcode = NVPTX::StoreRetvalV2F64; 2119 break; 2120 } 2121 break; 2122 case 4: 2123 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2124 default: 2125 return NULL; 2126 case MVT::i1: 2127 Opcode = NVPTX::StoreRetvalV4I8; 2128 break; 2129 case MVT::i8: 2130 Opcode = NVPTX::StoreRetvalV4I8; 2131 break; 2132 case MVT::i16: 2133 Opcode = NVPTX::StoreRetvalV4I16; 2134 break; 2135 case MVT::i32: 2136 Opcode = NVPTX::StoreRetvalV4I32; 2137 break; 2138 case MVT::f32: 2139 Opcode = NVPTX::StoreRetvalV4F32; 2140 break; 2141 } 2142 break; 2143 } 2144 2145 SDNode *Ret = 2146 CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); 2147 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2148 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2149 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2150 2151 return Ret; 2152 } 2153 2154 SDNode *NVPTXDAGToDAGISel::SelectStoreParam(SDNode *N) { 2155 SDLoc DL(N); 2156 SDValue Chain = N->getOperand(0); 2157 SDValue Param = N->getOperand(1); 2158 unsigned ParamVal = cast<ConstantSDNode>(Param)->getZExtValue(); 2159 SDValue Offset = N->getOperand(2); 2160 unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); 2161 MemSDNode *Mem = cast<MemSDNode>(N); 2162 SDValue Flag = N->getOperand(N->getNumOperands() - 1); 2163 2164 // How many elements do we have? 2165 unsigned NumElts = 1; 2166 switch (N->getOpcode()) { 2167 default: 2168 return NULL; 2169 case NVPTXISD::StoreParamU32: 2170 case NVPTXISD::StoreParamS32: 2171 case NVPTXISD::StoreParam: 2172 NumElts = 1; 2173 break; 2174 case NVPTXISD::StoreParamV2: 2175 NumElts = 2; 2176 break; 2177 case NVPTXISD::StoreParamV4: 2178 NumElts = 4; 2179 break; 2180 } 2181 2182 // Build vector of operands 2183 SmallVector<SDValue, 8> Ops; 2184 for (unsigned i = 0; i < NumElts; ++i) 2185 Ops.push_back(N->getOperand(i + 3)); 2186 Ops.push_back(CurDAG->getTargetConstant(ParamVal, MVT::i32)); 2187 Ops.push_back(CurDAG->getTargetConstant(OffsetVal, MVT::i32)); 2188 Ops.push_back(Chain); 2189 Ops.push_back(Flag); 2190 2191 // Determine target opcode 2192 // If we have an i1, use an 8-bit store. The lowering code in 2193 // NVPTXISelLowering will have already emitted an upcast. 2194 unsigned Opcode = 0; 2195 switch (N->getOpcode()) { 2196 default: 2197 switch (NumElts) { 2198 default: 2199 return NULL; 2200 case 1: 2201 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2202 default: 2203 return NULL; 2204 case MVT::i1: 2205 Opcode = NVPTX::StoreParamI8; 2206 break; 2207 case MVT::i8: 2208 Opcode = NVPTX::StoreParamI8; 2209 break; 2210 case MVT::i16: 2211 Opcode = NVPTX::StoreParamI16; 2212 break; 2213 case MVT::i32: 2214 Opcode = NVPTX::StoreParamI32; 2215 break; 2216 case MVT::i64: 2217 Opcode = NVPTX::StoreParamI64; 2218 break; 2219 case MVT::f32: 2220 Opcode = NVPTX::StoreParamF32; 2221 break; 2222 case MVT::f64: 2223 Opcode = NVPTX::StoreParamF64; 2224 break; 2225 } 2226 break; 2227 case 2: 2228 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2229 default: 2230 return NULL; 2231 case MVT::i1: 2232 Opcode = NVPTX::StoreParamV2I8; 2233 break; 2234 case MVT::i8: 2235 Opcode = NVPTX::StoreParamV2I8; 2236 break; 2237 case MVT::i16: 2238 Opcode = NVPTX::StoreParamV2I16; 2239 break; 2240 case MVT::i32: 2241 Opcode = NVPTX::StoreParamV2I32; 2242 break; 2243 case MVT::i64: 2244 Opcode = NVPTX::StoreParamV2I64; 2245 break; 2246 case MVT::f32: 2247 Opcode = NVPTX::StoreParamV2F32; 2248 break; 2249 case MVT::f64: 2250 Opcode = NVPTX::StoreParamV2F64; 2251 break; 2252 } 2253 break; 2254 case 4: 2255 switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { 2256 default: 2257 return NULL; 2258 case MVT::i1: 2259 Opcode = NVPTX::StoreParamV4I8; 2260 break; 2261 case MVT::i8: 2262 Opcode = NVPTX::StoreParamV4I8; 2263 break; 2264 case MVT::i16: 2265 Opcode = NVPTX::StoreParamV4I16; 2266 break; 2267 case MVT::i32: 2268 Opcode = NVPTX::StoreParamV4I32; 2269 break; 2270 case MVT::f32: 2271 Opcode = NVPTX::StoreParamV4F32; 2272 break; 2273 } 2274 break; 2275 } 2276 break; 2277 // Special case: if we have a sign-extend/zero-extend node, insert the 2278 // conversion instruction first, and use that as the value operand to 2279 // the selected StoreParam node. 2280 case NVPTXISD::StoreParamU32: { 2281 Opcode = NVPTX::StoreParamI32; 2282 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2283 MVT::i32); 2284 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_u32_u16, DL, 2285 MVT::i32, Ops[0], CvtNone); 2286 Ops[0] = SDValue(Cvt, 0); 2287 break; 2288 } 2289 case NVPTXISD::StoreParamS32: { 2290 Opcode = NVPTX::StoreParamI32; 2291 SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, 2292 MVT::i32); 2293 SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL, 2294 MVT::i32, Ops[0], CvtNone); 2295 Ops[0] = SDValue(Cvt, 0); 2296 break; 2297 } 2298 } 2299 2300 SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); 2301 SDNode *Ret = 2302 CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); 2303 MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); 2304 MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); 2305 cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); 2306 2307 return Ret; 2308 } 2309 2310 // SelectDirectAddr - Match a direct address for DAG. 2311 // A direct address could be a globaladdress or externalsymbol. 2312 bool NVPTXDAGToDAGISel::SelectDirectAddr(SDValue N, SDValue &Address) { 2313 // Return true if TGA or ES. 2314 if (N.getOpcode() == ISD::TargetGlobalAddress || 2315 N.getOpcode() == ISD::TargetExternalSymbol) { 2316 Address = N; 2317 return true; 2318 } 2319 if (N.getOpcode() == NVPTXISD::Wrapper) { 2320 Address = N.getOperand(0); 2321 return true; 2322 } 2323 if (N.getOpcode() == ISD::INTRINSIC_WO_CHAIN) { 2324 unsigned IID = cast<ConstantSDNode>(N.getOperand(0))->getZExtValue(); 2325 if (IID == Intrinsic::nvvm_ptr_gen_to_param) 2326 if (N.getOperand(1).getOpcode() == NVPTXISD::MoveParam) 2327 return (SelectDirectAddr(N.getOperand(1).getOperand(0), Address)); 2328 } 2329 return false; 2330 } 2331 2332 // symbol+offset 2333 bool NVPTXDAGToDAGISel::SelectADDRsi_imp( 2334 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 2335 if (Addr.getOpcode() == ISD::ADD) { 2336 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 2337 SDValue base = Addr.getOperand(0); 2338 if (SelectDirectAddr(base, Base)) { 2339 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 2340 return true; 2341 } 2342 } 2343 } 2344 return false; 2345 } 2346 2347 // symbol+offset 2348 bool NVPTXDAGToDAGISel::SelectADDRsi(SDNode *OpNode, SDValue Addr, 2349 SDValue &Base, SDValue &Offset) { 2350 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i32); 2351 } 2352 2353 // symbol+offset 2354 bool NVPTXDAGToDAGISel::SelectADDRsi64(SDNode *OpNode, SDValue Addr, 2355 SDValue &Base, SDValue &Offset) { 2356 return SelectADDRsi_imp(OpNode, Addr, Base, Offset, MVT::i64); 2357 } 2358 2359 // register+offset 2360 bool NVPTXDAGToDAGISel::SelectADDRri_imp( 2361 SDNode *OpNode, SDValue Addr, SDValue &Base, SDValue &Offset, MVT mvt) { 2362 if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 2363 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 2364 Offset = CurDAG->getTargetConstant(0, mvt); 2365 return true; 2366 } 2367 if (Addr.getOpcode() == ISD::TargetExternalSymbol || 2368 Addr.getOpcode() == ISD::TargetGlobalAddress) 2369 return false; // direct calls. 2370 2371 if (Addr.getOpcode() == ISD::ADD) { 2372 if (SelectDirectAddr(Addr.getOperand(0), Addr)) { 2373 return false; 2374 } 2375 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) { 2376 if (FrameIndexSDNode *FIN = 2377 dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) 2378 // Constant offset from frame ref. 2379 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), mvt); 2380 else 2381 Base = Addr.getOperand(0); 2382 Offset = CurDAG->getTargetConstant(CN->getZExtValue(), mvt); 2383 return true; 2384 } 2385 } 2386 return false; 2387 } 2388 2389 // register+offset 2390 bool NVPTXDAGToDAGISel::SelectADDRri(SDNode *OpNode, SDValue Addr, 2391 SDValue &Base, SDValue &Offset) { 2392 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i32); 2393 } 2394 2395 // register+offset 2396 bool NVPTXDAGToDAGISel::SelectADDRri64(SDNode *OpNode, SDValue Addr, 2397 SDValue &Base, SDValue &Offset) { 2398 return SelectADDRri_imp(OpNode, Addr, Base, Offset, MVT::i64); 2399 } 2400 2401 bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N, 2402 unsigned int spN) const { 2403 const Value *Src = NULL; 2404 // Even though MemIntrinsicSDNode is a subclas of MemSDNode, 2405 // the classof() for MemSDNode does not include MemIntrinsicSDNode 2406 // (See SelectionDAGNodes.h). So we need to check for both. 2407 if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) { 2408 Src = mN->getSrcValue(); 2409 } else if (MemSDNode *mN = dyn_cast<MemIntrinsicSDNode>(N)) { 2410 Src = mN->getSrcValue(); 2411 } 2412 if (!Src) 2413 return false; 2414 if (const PointerType *PT = dyn_cast<PointerType>(Src->getType())) 2415 return (PT->getAddressSpace() == spN); 2416 return false; 2417 } 2418 2419 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 2420 /// inline asm expressions. 2421 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand( 2422 const SDValue &Op, char ConstraintCode, std::vector<SDValue> &OutOps) { 2423 SDValue Op0, Op1; 2424 switch (ConstraintCode) { 2425 default: 2426 return true; 2427 case 'm': // memory 2428 if (SelectDirectAddr(Op, Op0)) { 2429 OutOps.push_back(Op0); 2430 OutOps.push_back(CurDAG->getTargetConstant(0, MVT::i32)); 2431 return false; 2432 } 2433 if (SelectADDRri(Op.getNode(), Op, Op0, Op1)) { 2434 OutOps.push_back(Op0); 2435 OutOps.push_back(Op1); 2436 return false; 2437 } 2438 break; 2439 } 2440 return true; 2441 } 2442 2443 // Return true if N is a undef or a constant. 2444 // If N was undef, return a (i8imm 0) in Retval 2445 // If N was imm, convert it to i8imm and return in Retval 2446 // Note: The convert to i8imm is required, otherwise the 2447 // pattern matcher inserts a bunch of IMOVi8rr to convert 2448 // the imm to i8imm, and this causes instruction selection 2449 // to fail. 2450 bool NVPTXDAGToDAGISel::UndefOrImm(SDValue Op, SDValue N, SDValue &Retval) { 2451 if (!(N.getOpcode() == ISD::UNDEF) && !(N.getOpcode() == ISD::Constant)) 2452 return false; 2453 2454 if (N.getOpcode() == ISD::UNDEF) 2455 Retval = CurDAG->getTargetConstant(0, MVT::i8); 2456 else { 2457 ConstantSDNode *cn = cast<ConstantSDNode>(N.getNode()); 2458 unsigned retval = cn->getZExtValue(); 2459 Retval = CurDAG->getTargetConstant(retval, MVT::i8); 2460 } 2461 return true; 2462 } 2463
