1 /* 2 * Copyright 2011 Christoph Bumiller 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include "codegen/nv50_ir.h" 24 #include "codegen/nv50_ir_build_util.h" 25 26 namespace nv50_ir { 27 28 BuildUtil::BuildUtil() 29 { 30 init(NULL); 31 } 32 33 BuildUtil::BuildUtil(Program *prog) 34 { 35 init(prog); 36 } 37 38 void 39 BuildUtil::init(Program *prog) 40 { 41 this->prog = prog; 42 43 func = NULL; 44 bb = NULL; 45 pos = NULL; 46 47 memset(imms, 0, sizeof(imms)); 48 immCount = 0; 49 } 50 51 void 52 BuildUtil::addImmediate(ImmediateValue *imm) 53 { 54 if (immCount > (NV50_IR_BUILD_IMM_HT_SIZE * 3) / 4) 55 return; 56 57 unsigned int pos = u32Hash(imm->reg.data.u32); 58 59 while (imms[pos]) 60 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE; 61 imms[pos] = imm; 62 immCount++; 63 } 64 65 Instruction * 66 BuildUtil::mkOp1(operation op, DataType ty, Value *dst, Value *src) 67 { 68 Instruction *insn = new_Instruction(func, op, ty); 69 70 insn->setDef(0, dst); 71 insn->setSrc(0, src); 72 73 insert(insn); 74 return insn; 75 } 76 77 Instruction * 78 BuildUtil::mkOp2(operation op, DataType ty, Value *dst, 79 Value *src0, Value *src1) 80 { 81 Instruction *insn = new_Instruction(func, op, ty); 82 83 insn->setDef(0, dst); 84 insn->setSrc(0, src0); 85 insn->setSrc(1, src1); 86 87 insert(insn); 88 return insn; 89 } 90 91 Instruction * 92 BuildUtil::mkOp3(operation op, DataType ty, Value *dst, 93 Value *src0, Value *src1, Value *src2) 94 { 95 Instruction *insn = new_Instruction(func, op, ty); 96 97 insn->setDef(0, dst); 98 insn->setSrc(0, src0); 99 insn->setSrc(1, src1); 100 insn->setSrc(2, src2); 101 102 insert(insn); 103 return insn; 104 } 105 106 Instruction * 107 BuildUtil::mkLoad(DataType ty, Value *dst, Symbol *mem, Value *ptr) 108 { 109 Instruction *insn = new_Instruction(func, OP_LOAD, ty); 110 111 insn->setDef(0, dst); 112 insn->setSrc(0, mem); 113 if (ptr) 114 insn->setIndirect(0, 0, ptr); 115 116 insert(insn); 117 return insn; 118 } 119 120 Instruction * 121 BuildUtil::mkStore(operation op, DataType ty, Symbol *mem, Value *ptr, 122 Value *stVal) 123 { 124 Instruction *insn = new_Instruction(func, op, ty); 125 126 insn->setSrc(0, mem); 127 insn->setSrc(1, stVal); 128 if (ptr) 129 insn->setIndirect(0, 0, ptr); 130 131 insert(insn); 132 return insn; 133 } 134 135 Instruction * 136 BuildUtil::mkFetch(Value *dst, DataType ty, DataFile file, int32_t offset, 137 Value *attrRel, Value *primRel) 138 { 139 Symbol *sym = mkSymbol(file, 0, ty, offset); 140 141 Instruction *insn = mkOp1(OP_VFETCH, ty, dst, sym); 142 143 insn->setIndirect(0, 0, attrRel); 144 insn->setIndirect(0, 1, primRel); 145 146 // already inserted 147 return insn; 148 } 149 150 Instruction * 151 BuildUtil::mkInterp(unsigned mode, Value *dst, int32_t offset, Value *rel) 152 { 153 operation op = OP_LINTERP; 154 DataType ty = TYPE_F32; 155 156 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_FLAT) 157 ty = TYPE_U32; 158 else 159 if ((mode & NV50_IR_INTERP_MODE_MASK) == NV50_IR_INTERP_PERSPECTIVE) 160 op = OP_PINTERP; 161 162 Symbol *sym = mkSymbol(FILE_SHADER_INPUT, 0, ty, offset); 163 164 Instruction *insn = mkOp1(op, ty, dst, sym); 165 insn->setIndirect(0, 0, rel); 166 insn->setInterpolate(mode); 167 return insn; 168 } 169 170 Instruction * 171 BuildUtil::mkMov(Value *dst, Value *src, DataType ty) 172 { 173 Instruction *insn = new_Instruction(func, OP_MOV, ty); 174 175 insn->setDef(0, dst); 176 insn->setSrc(0, src); 177 178 insert(insn); 179 return insn; 180 } 181 182 Instruction * 183 BuildUtil::mkMovToReg(int id, Value *src) 184 { 185 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(src->reg.size)); 186 187 insn->setDef(0, new_LValue(func, FILE_GPR)); 188 insn->getDef(0)->reg.data.id = id; 189 insn->setSrc(0, src); 190 191 insert(insn); 192 return insn; 193 } 194 195 Instruction * 196 BuildUtil::mkMovFromReg(Value *dst, int id) 197 { 198 Instruction *insn = new_Instruction(func, OP_MOV, typeOfSize(dst->reg.size)); 199 200 insn->setDef(0, dst); 201 insn->setSrc(0, new_LValue(func, FILE_GPR)); 202 insn->getSrc(0)->reg.data.id = id; 203 204 insert(insn); 205 return insn; 206 } 207 208 Instruction * 209 BuildUtil::mkCvt(operation op, 210 DataType dstTy, Value *dst, DataType srcTy, Value *src) 211 { 212 Instruction *insn = new_Instruction(func, op, dstTy); 213 214 insn->setType(dstTy, srcTy); 215 insn->setDef(0, dst); 216 insn->setSrc(0, src); 217 218 insert(insn); 219 return insn; 220 } 221 222 CmpInstruction * 223 BuildUtil::mkCmp(operation op, CondCode cc, DataType dstTy, Value *dst, 224 DataType srcTy, Value *src0, Value *src1, Value *src2) 225 { 226 CmpInstruction *insn = new_CmpInstruction(func, op); 227 228 insn->setType((dst->reg.file == FILE_PREDICATE || 229 dst->reg.file == FILE_FLAGS) ? TYPE_U8 : dstTy, srcTy); 230 insn->setCondition(cc); 231 insn->setDef(0, dst); 232 insn->setSrc(0, src0); 233 insn->setSrc(1, src1); 234 if (src2) 235 insn->setSrc(2, src2); 236 237 if (dst->reg.file == FILE_FLAGS) 238 insn->flagsDef = 0; 239 240 insert(insn); 241 return insn; 242 } 243 244 TexInstruction * 245 BuildUtil::mkTex(operation op, TexTarget targ, 246 uint16_t tic, uint16_t tsc, 247 const std::vector<Value *> &def, 248 const std::vector<Value *> &src) 249 { 250 TexInstruction *tex = new_TexInstruction(func, op); 251 252 for (size_t d = 0; d < def.size() && def[d]; ++d) 253 tex->setDef(d, def[d]); 254 for (size_t s = 0; s < src.size() && src[s]; ++s) 255 tex->setSrc(s, src[s]); 256 257 tex->setTexture(targ, tic, tsc); 258 259 insert(tex); 260 return tex; 261 } 262 263 Instruction * 264 BuildUtil::mkQuadop(uint8_t q, Value *def, uint8_t l, Value *src0, Value *src1) 265 { 266 Instruction *quadop = mkOp2(OP_QUADOP, TYPE_F32, def, src0, src1); 267 quadop->subOp = q; 268 quadop->lanes = l; 269 return quadop; 270 } 271 272 Instruction * 273 BuildUtil::mkSelect(Value *pred, Value *dst, Value *trSrc, Value *flSrc) 274 { 275 LValue *def0 = getSSA(); 276 LValue *def1 = getSSA(); 277 278 mkMov(def0, trSrc)->setPredicate(CC_P, pred); 279 mkMov(def1, flSrc)->setPredicate(CC_NOT_P, pred); 280 281 return mkOp2(OP_UNION, typeOfSize(dst->reg.size), dst, def0, def1); 282 } 283 284 Instruction * 285 BuildUtil::mkSplit(Value *h[2], uint8_t halfSize, Value *val) 286 { 287 Instruction *insn = NULL; 288 289 const DataType fTy = typeOfSize(halfSize * 2); 290 291 if (val->reg.file == FILE_IMMEDIATE) 292 val = mkMov(getSSA(halfSize * 2), val, fTy)->getDef(0); 293 294 if (isMemoryFile(val->reg.file)) { 295 h[0] = cloneShallow(getFunction(), val); 296 h[1] = cloneShallow(getFunction(), val); 297 h[0]->reg.size = halfSize; 298 h[1]->reg.size = halfSize; 299 h[1]->reg.data.offset += halfSize; 300 } else { 301 h[0] = getSSA(halfSize, val->reg.file); 302 h[1] = getSSA(halfSize, val->reg.file); 303 insn = mkOp1(OP_SPLIT, fTy, h[0], val); 304 insn->setDef(1, h[1]); 305 } 306 return insn; 307 } 308 309 FlowInstruction * 310 BuildUtil::mkFlow(operation op, void *targ, CondCode cc, Value *pred) 311 { 312 FlowInstruction *insn = new_FlowInstruction(func, op, targ); 313 314 if (pred) 315 insn->setPredicate(cc, pred); 316 317 insert(insn); 318 return insn; 319 } 320 321 void 322 BuildUtil::mkClobber(DataFile f, uint32_t rMask, int unit) 323 { 324 static const uint16_t baseSize2[16] = 325 { 326 0x0000, 0x0010, 0x0011, 0x0020, 0x0012, 0x1210, 0x1211, 0x1220, 327 0x0013, 0x1310, 0x1311, 0x1320, 0x0022, 0x2210, 0x2211, 0x0040, 328 }; 329 330 int base = 0; 331 332 for (; rMask; rMask >>= 4, base += 4) { 333 const uint32_t mask = rMask & 0xf; 334 if (!mask) 335 continue; 336 int base1 = (baseSize2[mask] >> 0) & 0xf; 337 int size1 = (baseSize2[mask] >> 4) & 0xf; 338 int base2 = (baseSize2[mask] >> 8) & 0xf; 339 int size2 = (baseSize2[mask] >> 12) & 0xf; 340 Instruction *insn = mkOp(OP_NOP, TYPE_NONE, NULL); 341 if (1) { // size1 can't be 0 342 LValue *reg = new_LValue(func, f); 343 reg->reg.size = size1 << unit; 344 reg->reg.data.id = base + base1; 345 insn->setDef(0, reg); 346 } 347 if (size2) { 348 LValue *reg = new_LValue(func, f); 349 reg->reg.size = size2 << unit; 350 reg->reg.data.id = base + base2; 351 insn->setDef(1, reg); 352 } 353 } 354 } 355 356 ImmediateValue * 357 BuildUtil::mkImm(uint32_t u) 358 { 359 unsigned int pos = u32Hash(u); 360 361 while (imms[pos] && imms[pos]->reg.data.u32 != u) 362 pos = (pos + 1) % NV50_IR_BUILD_IMM_HT_SIZE; 363 364 ImmediateValue *imm = imms[pos]; 365 if (!imm) { 366 imm = new_ImmediateValue(prog, u); 367 addImmediate(imm); 368 } 369 return imm; 370 } 371 372 ImmediateValue * 373 BuildUtil::mkImm(uint64_t u) 374 { 375 ImmediateValue *imm = new_ImmediateValue(prog, (uint32_t)0); 376 377 imm->reg.size = 8; 378 imm->reg.type = TYPE_U64; 379 imm->reg.data.u64 = u; 380 381 return imm; 382 } 383 384 ImmediateValue * 385 BuildUtil::mkImm(float f) 386 { 387 union { 388 float f32; 389 uint32_t u32; 390 } u; 391 u.f32 = f; 392 return mkImm(u.u32); 393 } 394 395 ImmediateValue * 396 BuildUtil::mkImm(double d) 397 { 398 return new_ImmediateValue(prog, d); 399 } 400 401 Value * 402 BuildUtil::loadImm(Value *dst, float f) 403 { 404 return mkOp1v(OP_MOV, TYPE_F32, dst ? dst : getScratch(), mkImm(f)); 405 } 406 407 Value * 408 BuildUtil::loadImm(Value *dst, double d) 409 { 410 return mkOp1v(OP_MOV, TYPE_F64, dst ? dst : getScratch(8), mkImm(d)); 411 } 412 413 Value * 414 BuildUtil::loadImm(Value *dst, uint32_t u) 415 { 416 return mkOp1v(OP_MOV, TYPE_U32, dst ? dst : getScratch(), mkImm(u)); 417 } 418 419 Value * 420 BuildUtil::loadImm(Value *dst, uint64_t u) 421 { 422 return mkOp1v(OP_MOV, TYPE_U64, dst ? dst : getScratch(8), mkImm(u)); 423 } 424 425 Symbol * 426 BuildUtil::mkSymbol(DataFile file, int8_t fileIndex, DataType ty, 427 uint32_t baseAddr) 428 { 429 Symbol *sym = new_Symbol(prog, file, fileIndex); 430 431 sym->setOffset(baseAddr); 432 sym->reg.type = ty; 433 sym->reg.size = typeSizeof(ty); 434 435 return sym; 436 } 437 438 Symbol * 439 BuildUtil::mkSysVal(SVSemantic svName, uint32_t svIndex) 440 { 441 Symbol *sym = new_Symbol(prog, FILE_SYSTEM_VALUE, 0); 442 443 assert(svIndex < 4 || svName == SV_CLIP_DISTANCE); 444 445 switch (svName) { 446 case SV_POSITION: 447 case SV_FACE: 448 case SV_YDIR: 449 case SV_POINT_SIZE: 450 case SV_POINT_COORD: 451 case SV_CLIP_DISTANCE: 452 case SV_TESS_OUTER: 453 case SV_TESS_INNER: 454 case SV_TESS_COORD: 455 sym->reg.type = TYPE_F32; 456 break; 457 default: 458 sym->reg.type = TYPE_U32; 459 break; 460 } 461 sym->reg.size = typeSizeof(sym->reg.type); 462 463 sym->reg.data.sv.sv = svName; 464 sym->reg.data.sv.index = svIndex; 465 466 return sym; 467 } 468 469 void 470 BuildUtil::DataArray::setup(unsigned array, unsigned arrayIdx, 471 uint32_t base, int len, int vecDim, int eltSize, 472 DataFile file, int8_t fileIdx) 473 { 474 this->array = array; 475 this->arrayIdx = arrayIdx; 476 this->baseAddr = base; 477 this->arrayLen = len; 478 this->vecDim = vecDim; 479 this->eltSize = eltSize; 480 this->file = file; 481 this->regOnly = !isMemoryFile(file); 482 483 if (!regOnly) { 484 baseSym = new_Symbol(up->getProgram(), file, fileIdx); 485 baseSym->setOffset(baseAddr); 486 baseSym->reg.size = eltSize; 487 } else { 488 baseSym = NULL; 489 } 490 } 491 492 Value * 493 BuildUtil::DataArray::acquire(ValueMap &m, int i, int c) 494 { 495 if (regOnly) { 496 Value *v = lookup(m, i, c); 497 if (!v) 498 v = insert(m, i, c, new_LValue(up->getFunction(), file)); 499 500 return v; 501 } else { 502 return up->getScratch(eltSize); 503 } 504 } 505 506 Value * 507 BuildUtil::DataArray::load(ValueMap &m, int i, int c, Value *ptr) 508 { 509 if (regOnly) { 510 Value *v = lookup(m, i, c); 511 if (!v) 512 v = insert(m, i, c, new_LValue(up->getFunction(), file)); 513 514 return v; 515 } else { 516 Value *sym = lookup(m, i, c); 517 if (!sym) 518 sym = insert(m, i, c, mkSymbol(i, c)); 519 520 return up->mkLoadv(typeOfSize(eltSize), static_cast<Symbol *>(sym), ptr); 521 } 522 } 523 524 void 525 BuildUtil::DataArray::store(ValueMap &m, int i, int c, Value *ptr, Value *value) 526 { 527 if (regOnly) { 528 assert(!ptr); 529 if (!lookup(m, i, c)) 530 insert(m, i, c, value); 531 532 assert(lookup(m, i, c) == value); 533 } else { 534 Value *sym = lookup(m, i, c); 535 if (!sym) 536 sym = insert(m, i, c, mkSymbol(i, c)); 537 538 const DataType stTy = typeOfSize(value->reg.size); 539 540 up->mkStore(OP_STORE, stTy, static_cast<Symbol *>(sym), ptr, value); 541 } 542 } 543 544 Symbol * 545 BuildUtil::DataArray::mkSymbol(int i, int c) 546 { 547 const unsigned int idx = i * vecDim + c; 548 Symbol *sym = new_Symbol(up->getProgram(), file, 0); 549 550 assert(baseSym || (idx < arrayLen && c < vecDim)); 551 552 sym->reg.size = eltSize; 553 sym->reg.type = typeOfSize(eltSize); 554 sym->setAddress(baseSym, baseAddr + idx * eltSize); 555 return sym; 556 } 557 558 559 Instruction * 560 BuildUtil::split64BitOpPostRA(Function *fn, Instruction *i, 561 Value *zero, 562 Value *carry) 563 { 564 DataType hTy; 565 int srcNr; 566 567 switch (i->dType) { 568 case TYPE_U64: hTy = TYPE_U32; break; 569 case TYPE_S64: hTy = TYPE_S32; break; 570 case TYPE_F64: 571 if (i->op == OP_MOV) { 572 hTy = TYPE_U32; 573 break; 574 } 575 /* fallthrough */ 576 default: 577 return NULL; 578 } 579 580 switch (i->op) { 581 case OP_MOV: srcNr = 1; break; 582 case OP_ADD: 583 case OP_SUB: 584 if (!carry) 585 return NULL; 586 srcNr = 2; 587 break; 588 case OP_SELP: srcNr = 3; break; 589 default: 590 // TODO when needed 591 return NULL; 592 } 593 594 i->setType(hTy); 595 i->setDef(0, cloneShallow(fn, i->getDef(0))); 596 i->getDef(0)->reg.size = 4; 597 Instruction *lo = i; 598 Instruction *hi = cloneForward(fn, i); 599 lo->bb->insertAfter(lo, hi); 600 601 hi->getDef(0)->reg.data.id++; 602 603 for (int s = 0; s < srcNr; ++s) { 604 if (lo->getSrc(s)->reg.size < 8) { 605 if (s == 2) 606 hi->setSrc(s, lo->getSrc(s)); 607 else 608 hi->setSrc(s, zero); 609 } else { 610 if (lo->getSrc(s)->refCount() > 1) 611 lo->setSrc(s, cloneShallow(fn, lo->getSrc(s))); 612 lo->getSrc(s)->reg.size /= 2; 613 hi->setSrc(s, cloneShallow(fn, lo->getSrc(s))); 614 615 switch (hi->src(s).getFile()) { 616 case FILE_IMMEDIATE: 617 hi->getSrc(s)->reg.data.u64 >>= 32; 618 break; 619 case FILE_MEMORY_CONST: 620 case FILE_MEMORY_SHARED: 621 case FILE_SHADER_INPUT: 622 case FILE_SHADER_OUTPUT: 623 hi->getSrc(s)->reg.data.offset += 4; 624 break; 625 default: 626 assert(hi->src(s).getFile() == FILE_GPR); 627 hi->getSrc(s)->reg.data.id++; 628 break; 629 } 630 } 631 } 632 if (srcNr == 2) { 633 lo->setFlagsDef(1, carry); 634 hi->setFlagsSrc(hi->srcCount(), carry); 635 } 636 return hi; 637 } 638 639 } // namespace nv50_ir 640