1 /* 2 * Copyright 2011 Christoph Bumiller 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #ifndef __NV50_IR_H__ 24 #define __NV50_IR_H__ 25 26 #include <stdio.h> 27 #include <stdlib.h> 28 #include <stdint.h> 29 #include <deque> 30 #include <list> 31 #include <vector> 32 33 #include "codegen/unordered_set.h" 34 #include "codegen/nv50_ir_util.h" 35 #include "codegen/nv50_ir_graph.h" 36 37 #include "codegen/nv50_ir_driver.h" 38 39 namespace nv50_ir { 40 41 enum operation 42 { 43 OP_NOP = 0, 44 OP_PHI, 45 OP_UNION, // unify a new definition and several source values 46 OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced) 47 OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value 48 OP_CONSTRAINT, // copy values into consecutive registers 49 OP_MOV, // simple copy, no modifiers allowed 50 OP_LOAD, 51 OP_STORE, 52 OP_ADD, // NOTE: add u64 + u32 is legal for targets w/o 64-bit integer adds 53 OP_SUB, 54 OP_MUL, 55 OP_DIV, 56 OP_MOD, 57 OP_MAD, 58 OP_FMA, 59 OP_SAD, // abs(src0 - src1) + src2 60 OP_SHLADD, 61 OP_ABS, 62 OP_NEG, 63 OP_NOT, 64 OP_AND, 65 OP_OR, 66 OP_XOR, 67 OP_SHL, 68 OP_SHR, 69 OP_MAX, 70 OP_MIN, 71 OP_SAT, // CLAMP(f32, 0.0, 1.0) 72 OP_CEIL, 73 OP_FLOOR, 74 OP_TRUNC, 75 OP_CVT, 76 OP_SET_AND, // dst = (src0 CMP src1) & src2 77 OP_SET_OR, 78 OP_SET_XOR, 79 OP_SET, 80 OP_SELP, // dst = src2 ? src0 : src1 81 OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1 82 OP_RCP, 83 OP_RSQ, 84 OP_LG2, 85 OP_SIN, 86 OP_COS, 87 OP_EX2, 88 OP_EXP, // exponential (base M_E) 89 OP_LOG, // natural logarithm 90 OP_PRESIN, 91 OP_PREEX2, 92 OP_SQRT, 93 OP_POW, 94 OP_BRA, 95 OP_CALL, 96 OP_RET, 97 OP_CONT, 98 OP_BREAK, 99 OP_PRERET, 100 OP_PRECONT, 101 OP_PREBREAK, 102 OP_BRKPT, // breakpoint (not related to loops) 103 OP_JOINAT, // push control flow convergence point 104 OP_JOIN, // converge 105 OP_DISCARD, 106 OP_EXIT, 107 OP_MEMBAR, // memory barrier (mfence, lfence, sfence) 108 OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base 109 OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1] 110 OP_AFETCH, // fetch base address of shader input (a[%r1+0x10]) 111 OP_EXPORT, 112 OP_LINTERP, 113 OP_PINTERP, 114 OP_EMIT, // emit vertex 115 OP_RESTART, // restart primitive 116 OP_TEX, 117 OP_TXB, // texture bias 118 OP_TXL, // texure lod 119 OP_TXF, // texel fetch 120 OP_TXQ, // texture size query 121 OP_TXD, // texture derivatives 122 OP_TXG, // texture gather 123 OP_TXLQ, // texture query lod 124 OP_TEXCSAA, // texture op for coverage sampling 125 OP_TEXPREP, // turn cube map array into 2d array coordinates 126 OP_SULDB, // surface load (raw) 127 OP_SULDP, // surface load (formatted) 128 OP_SUSTB, // surface store (raw) 129 OP_SUSTP, // surface store (formatted) 130 OP_SUREDB, 131 OP_SUREDP, // surface reduction (atomic op) 132 OP_SULEA, // surface load effective address 133 OP_SUBFM, // surface bitfield manipulation 134 OP_SUCLAMP, // clamp surface coordinates 135 OP_SUEAU, // surface effective address 136 OP_SUQ, // surface query 137 OP_MADSP, // special integer multiply-add 138 OP_TEXBAR, // texture dependency barrier 139 OP_DFDX, 140 OP_DFDY, 141 OP_RDSV, // read system value 142 OP_WRSV, // write system value 143 OP_PIXLD, // get info about raster object or surfaces 144 OP_QUADOP, 145 OP_QUADON, 146 OP_QUADPOP, 147 OP_POPCNT, // bitcount(src0 & src1) 148 OP_INSBF, // insert first src1[8:15] bits of src0 into src2 at src1[0:7] 149 OP_EXTBF, // place bits [K,K+N) of src0 into dst, src1 = 0xNNKK 150 OP_BFIND, // find highest/lowest set bit 151 OP_PERMT, // dst = bytes from src2,src0 selected by src1 (nvc0's src order) 152 OP_ATOM, 153 OP_BAR, // execution barrier, sources = { id, thread count, predicate } 154 OP_VADD, // byte/word vector operations 155 OP_VAVG, 156 OP_VMIN, 157 OP_VMAX, 158 OP_VSAD, 159 OP_VSET, 160 OP_VSHR, 161 OP_VSHL, 162 OP_VSEL, 163 OP_CCTL, // cache control 164 OP_SHFL, // warp shuffle 165 OP_VOTE, 166 OP_BUFQ, // buffer query 167 OP_LAST 168 }; 169 170 // various instruction-specific modifier definitions Instruction::subOp 171 // MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs) 172 #define NV50_IR_SUBOP_MUL_HIGH 1 173 #define NV50_IR_SUBOP_EMIT_RESTART 1 174 #define NV50_IR_SUBOP_LDC_IL 1 175 #define NV50_IR_SUBOP_LDC_IS 2 176 #define NV50_IR_SUBOP_LDC_ISL 3 177 #define NV50_IR_SUBOP_SHIFT_WRAP 1 178 #define NV50_IR_SUBOP_SHIFT_HIGH 2 179 #define NV50_IR_SUBOP_EMU_PRERET 1 180 #define NV50_IR_SUBOP_TEXBAR(n) n 181 #define NV50_IR_SUBOP_MOV_FINAL 1 182 #define NV50_IR_SUBOP_EXTBF_REV 1 183 #define NV50_IR_SUBOP_BFIND_SAMT 1 184 #define NV50_IR_SUBOP_RCPRSQ_64H 1 185 #define NV50_IR_SUBOP_PERMT_F4E 1 186 #define NV50_IR_SUBOP_PERMT_B4E 2 187 #define NV50_IR_SUBOP_PERMT_RC8 3 188 #define NV50_IR_SUBOP_PERMT_ECL 4 189 #define NV50_IR_SUBOP_PERMT_ECR 5 190 #define NV50_IR_SUBOP_PERMT_RC16 6 191 #define NV50_IR_SUBOP_BAR_SYNC 0 192 #define NV50_IR_SUBOP_BAR_ARRIVE 1 193 #define NV50_IR_SUBOP_BAR_RED_AND 2 194 #define NV50_IR_SUBOP_BAR_RED_OR 3 195 #define NV50_IR_SUBOP_BAR_RED_POPC 4 196 #define NV50_IR_SUBOP_MEMBAR_L 1 197 #define NV50_IR_SUBOP_MEMBAR_S 2 198 #define NV50_IR_SUBOP_MEMBAR_M 3 199 #define NV50_IR_SUBOP_MEMBAR_CTA (0 << 2) 200 #define NV50_IR_SUBOP_MEMBAR_GL (1 << 2) 201 #define NV50_IR_SUBOP_MEMBAR_SYS (2 << 2) 202 #define NV50_IR_SUBOP_MEMBAR_DIR(m) ((m) & 0x3) 203 #define NV50_IR_SUBOP_MEMBAR_SCOPE(m) ((m) & ~0x3) 204 #define NV50_IR_SUBOP_MEMBAR(d,s) \ 205 (NV50_IR_SUBOP_MEMBAR_##d | NV50_IR_SUBOP_MEMBAR_##s) 206 #define NV50_IR_SUBOP_ATOM_ADD 0 207 #define NV50_IR_SUBOP_ATOM_MIN 1 208 #define NV50_IR_SUBOP_ATOM_MAX 2 209 #define NV50_IR_SUBOP_ATOM_INC 3 210 #define NV50_IR_SUBOP_ATOM_DEC 4 211 #define NV50_IR_SUBOP_ATOM_AND 5 212 #define NV50_IR_SUBOP_ATOM_OR 6 213 #define NV50_IR_SUBOP_ATOM_XOR 7 214 #define NV50_IR_SUBOP_ATOM_CAS 8 215 #define NV50_IR_SUBOP_ATOM_EXCH 9 216 #define NV50_IR_SUBOP_CCTL_IV 5 217 #define NV50_IR_SUBOP_CCTL_IVALL 6 218 #define NV50_IR_SUBOP_SUST_IGN 0 219 #define NV50_IR_SUBOP_SUST_TRAP 1 220 #define NV50_IR_SUBOP_SUST_SDCL 3 221 #define NV50_IR_SUBOP_SULD_ZERO 0 222 #define NV50_IR_SUBOP_SULD_TRAP 1 223 #define NV50_IR_SUBOP_SULD_SDCL 3 224 #define NV50_IR_SUBOP_SUBFM_3D 1 225 #define NV50_IR_SUBOP_SUCLAMP_2D 0x10 226 #define NV50_IR_SUBOP_SUCLAMP_SD(r, d) (( 0 + (r)) | ((d == 2) ? 0x10 : 0)) 227 #define NV50_IR_SUBOP_SUCLAMP_PL(r, d) (( 5 + (r)) | ((d == 2) ? 0x10 : 0)) 228 #define NV50_IR_SUBOP_SUCLAMP_BL(r, d) ((10 + (r)) | ((d == 2) ? 0x10 : 0)) 229 #define NV50_IR_SUBOP_PIXLD_COUNT 0 230 #define NV50_IR_SUBOP_PIXLD_COVMASK 1 231 #define NV50_IR_SUBOP_PIXLD_COVERED 2 232 #define NV50_IR_SUBOP_PIXLD_OFFSET 3 233 #define NV50_IR_SUBOP_PIXLD_CENT_OFFSET 4 234 #define NV50_IR_SUBOP_PIXLD_SAMPLEID 5 235 #define NV50_IR_SUBOP_SHFL_IDX 0 236 #define NV50_IR_SUBOP_SHFL_UP 1 237 #define NV50_IR_SUBOP_SHFL_DOWN 2 238 #define NV50_IR_SUBOP_SHFL_BFLY 3 239 #define NV50_IR_SUBOP_LOAD_LOCKED 1 240 #define NV50_IR_SUBOP_STORE_UNLOCKED 2 241 #define NV50_IR_SUBOP_MADSP_SD 0xffff 242 // Yes, we could represent those with DataType. 243 // Or put the type into operation and have a couple 1000 values in that enum. 244 // This will have to do for now. 245 // The bitfields are supposed to correspond to nve4 ISA. 246 #define NV50_IR_SUBOP_MADSP(a,b,c) (((c) << 8) | ((b) << 4) | (a)) 247 #define NV50_IR_SUBOP_V1(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x0000) 248 #define NV50_IR_SUBOP_V2(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x4000) 249 #define NV50_IR_SUBOP_V4(d,a,b) (((d) << 10) | ((b) << 5) | (a) | 0x8000) 250 #define NV50_IR_SUBOP_Vn(n) ((n) >> 14) 251 #define NV50_IR_SUBOP_VOTE_ALL 0 252 #define NV50_IR_SUBOP_VOTE_ANY 1 253 #define NV50_IR_SUBOP_VOTE_UNI 2 254 255 #define NV50_IR_SUBOP_MINMAX_LOW 1 256 #define NV50_IR_SUBOP_MINMAX_MED 2 257 #define NV50_IR_SUBOP_MINMAX_HIGH 3 258 259 enum DataType 260 { 261 TYPE_NONE, 262 TYPE_U8, 263 TYPE_S8, 264 TYPE_U16, 265 TYPE_S16, 266 TYPE_U32, 267 TYPE_S32, 268 TYPE_U64, // 64 bit operations are only lowered after register allocation 269 TYPE_S64, 270 TYPE_F16, 271 TYPE_F32, 272 TYPE_F64, 273 TYPE_B96, 274 TYPE_B128 275 }; 276 277 enum CondCode 278 { 279 CC_FL = 0, 280 CC_NEVER = CC_FL, // when used with FILE_FLAGS 281 CC_LT = 1, 282 CC_EQ = 2, 283 CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE 284 CC_LE = 3, 285 CC_GT = 4, 286 CC_NE = 5, 287 CC_P = CC_NE, 288 CC_GE = 6, 289 CC_TR = 7, 290 CC_ALWAYS = CC_TR, 291 CC_U = 8, 292 CC_LTU = 9, 293 CC_EQU = 10, 294 CC_LEU = 11, 295 CC_GTU = 12, 296 CC_NEU = 13, 297 CC_GEU = 14, 298 CC_NO = 0x10, 299 CC_NC = 0x11, 300 CC_NS = 0x12, 301 CC_NA = 0x13, 302 CC_A = 0x14, 303 CC_S = 0x15, 304 CC_C = 0x16, 305 CC_O = 0x17 306 }; 307 308 enum RoundMode 309 { 310 ROUND_N, // nearest 311 ROUND_M, // towards -inf 312 ROUND_Z, // towards 0 313 ROUND_P, // towards +inf 314 ROUND_NI, // nearest integer 315 ROUND_MI, // to integer towards -inf 316 ROUND_ZI, // to integer towards 0 317 ROUND_PI, // to integer towards +inf 318 }; 319 320 enum CacheMode 321 { 322 CACHE_CA, // cache at all levels 323 CACHE_WB = CACHE_CA, // cache write back 324 CACHE_CG, // cache at global level 325 CACHE_CS, // cache streaming 326 CACHE_CV, // cache as volatile 327 CACHE_WT = CACHE_CV // cache write-through 328 }; 329 330 enum DataFile 331 { 332 FILE_NULL = 0, 333 FILE_GPR, 334 FILE_PREDICATE, // boolean predicate 335 FILE_FLAGS, // zero/sign/carry/overflow bits 336 FILE_ADDRESS, 337 LAST_REGISTER_FILE = FILE_ADDRESS, 338 FILE_IMMEDIATE, 339 FILE_MEMORY_CONST, 340 FILE_SHADER_INPUT, 341 FILE_SHADER_OUTPUT, 342 FILE_MEMORY_BUFFER, 343 FILE_MEMORY_GLOBAL, 344 FILE_MEMORY_SHARED, 345 FILE_MEMORY_LOCAL, 346 FILE_SYSTEM_VALUE, 347 DATA_FILE_COUNT 348 }; 349 350 enum TexTarget 351 { 352 TEX_TARGET_1D, 353 TEX_TARGET_2D, 354 TEX_TARGET_2D_MS, 355 TEX_TARGET_3D, 356 TEX_TARGET_CUBE, 357 TEX_TARGET_1D_SHADOW, 358 TEX_TARGET_2D_SHADOW, 359 TEX_TARGET_CUBE_SHADOW, 360 TEX_TARGET_1D_ARRAY, 361 TEX_TARGET_2D_ARRAY, 362 TEX_TARGET_2D_MS_ARRAY, 363 TEX_TARGET_CUBE_ARRAY, 364 TEX_TARGET_1D_ARRAY_SHADOW, 365 TEX_TARGET_2D_ARRAY_SHADOW, 366 TEX_TARGET_RECT, 367 TEX_TARGET_RECT_SHADOW, 368 TEX_TARGET_CUBE_ARRAY_SHADOW, 369 TEX_TARGET_BUFFER, 370 TEX_TARGET_COUNT 371 }; 372 373 enum ImgFormat 374 { 375 FMT_NONE, 376 377 FMT_RGBA32F, 378 FMT_RGBA16F, 379 FMT_RG32F, 380 FMT_RG16F, 381 FMT_R11G11B10F, 382 FMT_R32F, 383 FMT_R16F, 384 385 FMT_RGBA32UI, 386 FMT_RGBA16UI, 387 FMT_RGB10A2UI, 388 FMT_RGBA8UI, 389 FMT_RG32UI, 390 FMT_RG16UI, 391 FMT_RG8UI, 392 FMT_R32UI, 393 FMT_R16UI, 394 FMT_R8UI, 395 396 FMT_RGBA32I, 397 FMT_RGBA16I, 398 FMT_RGBA8I, 399 FMT_RG32I, 400 FMT_RG16I, 401 FMT_RG8I, 402 FMT_R32I, 403 FMT_R16I, 404 FMT_R8I, 405 406 FMT_RGBA16, 407 FMT_RGB10A2, 408 FMT_RGBA8, 409 FMT_RG16, 410 FMT_RG8, 411 FMT_R16, 412 FMT_R8, 413 414 FMT_RGBA16_SNORM, 415 FMT_RGBA8_SNORM, 416 FMT_RG16_SNORM, 417 FMT_RG8_SNORM, 418 FMT_R16_SNORM, 419 FMT_R8_SNORM, 420 421 FMT_BGRA8, 422 423 IMG_FORMAT_COUNT, 424 }; 425 426 enum ImgType { 427 UINT, 428 SINT, 429 UNORM, 430 SNORM, 431 FLOAT, 432 }; 433 434 enum SVSemantic 435 { 436 SV_POSITION, // WPOS 437 SV_VERTEX_ID, 438 SV_INSTANCE_ID, 439 SV_INVOCATION_ID, 440 SV_PRIMITIVE_ID, 441 SV_VERTEX_COUNT, // gl_PatchVerticesIn 442 SV_LAYER, 443 SV_VIEWPORT_INDEX, 444 SV_YDIR, 445 SV_FACE, 446 SV_POINT_SIZE, 447 SV_POINT_COORD, 448 SV_CLIP_DISTANCE, 449 SV_SAMPLE_INDEX, 450 SV_SAMPLE_POS, 451 SV_SAMPLE_MASK, 452 SV_TESS_OUTER, 453 SV_TESS_INNER, 454 SV_TESS_COORD, 455 SV_TID, 456 SV_CTAID, 457 SV_NTID, 458 SV_GRIDID, 459 SV_NCTAID, 460 SV_LANEID, 461 SV_PHYSID, 462 SV_NPHYSID, 463 SV_CLOCK, 464 SV_LBASE, 465 SV_SBASE, 466 SV_VERTEX_STRIDE, 467 SV_INVOCATION_INFO, 468 SV_THREAD_KILL, 469 SV_BASEVERTEX, 470 SV_BASEINSTANCE, 471 SV_DRAWID, 472 SV_WORK_DIM, 473 SV_LANEMASK_EQ, 474 SV_LANEMASK_LT, 475 SV_LANEMASK_LE, 476 SV_LANEMASK_GT, 477 SV_LANEMASK_GE, 478 SV_UNDEFINED, 479 SV_LAST 480 }; 481 482 class Program; 483 class Function; 484 class BasicBlock; 485 486 class Target; 487 488 class Instruction; 489 class CmpInstruction; 490 class TexInstruction; 491 class FlowInstruction; 492 493 class Value; 494 class LValue; 495 class Symbol; 496 class ImmediateValue; 497 498 struct Storage 499 { 500 DataFile file; 501 int8_t fileIndex; // signed, may be indirect for CONST[] 502 uint8_t size; // this should match the Instruction type's size 503 DataType type; // mainly for pretty printing 504 union { 505 uint64_t u64; // immediate values 506 uint32_t u32; 507 uint16_t u16; 508 uint8_t u8; 509 int64_t s64; 510 int32_t s32; 511 int16_t s16; 512 int8_t s8; 513 float f32; 514 double f64; 515 int32_t offset; // offset from 0 (base of address space) 516 int32_t id; // register id (< 0 if virtual/unassigned, in units <= 4) 517 struct { 518 SVSemantic sv; 519 int index; 520 } sv; 521 } data; 522 }; 523 524 // precedence: NOT after SAT after NEG after ABS 525 #define NV50_IR_MOD_ABS (1 << 0) 526 #define NV50_IR_MOD_NEG (1 << 1) 527 #define NV50_IR_MOD_SAT (1 << 2) 528 #define NV50_IR_MOD_NOT (1 << 3) 529 #define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS) 530 531 #define NV50_IR_INTERP_MODE_MASK 0x3 532 #define NV50_IR_INTERP_LINEAR (0 << 0) 533 #define NV50_IR_INTERP_PERSPECTIVE (1 << 0) 534 #define NV50_IR_INTERP_FLAT (2 << 0) 535 #define NV50_IR_INTERP_SC (3 << 0) // what exactly is that ? 536 #define NV50_IR_INTERP_SAMPLE_MASK 0xc 537 #define NV50_IR_INTERP_DEFAULT (0 << 2) 538 #define NV50_IR_INTERP_CENTROID (1 << 2) 539 #define NV50_IR_INTERP_OFFSET (2 << 2) 540 #define NV50_IR_INTERP_SAMPLEID (3 << 2) 541 542 // do we really want this to be a class ? 543 class Modifier 544 { 545 public: 546 Modifier() : bits(0) { } 547 Modifier(unsigned int m) : bits(m) { } 548 Modifier(operation op); 549 550 // @return new Modifier applying a after b (asserts if unrepresentable) 551 Modifier operator*(const Modifier) const; 552 Modifier operator*=(const Modifier m) { *this = *this * m; return *this; } 553 Modifier operator==(const Modifier m) const { return m.bits == bits; } 554 Modifier operator!=(const Modifier m) const { return m.bits != bits; } 555 556 inline Modifier operator&(const Modifier m) const { return bits & m.bits; } 557 inline Modifier operator|(const Modifier m) const { return bits | m.bits; } 558 inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; } 559 560 operation getOp() const; 561 562 inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; } 563 inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; } 564 565 inline operator bool() const { return bits ? true : false; } 566 567 void applyTo(ImmediateValue &imm) const; 568 569 int print(char *buf, size_t size) const; 570 571 private: 572 uint8_t bits; 573 }; 574 575 class ValueRef 576 { 577 public: 578 ValueRef(Value * = NULL); 579 ValueRef(const ValueRef&); 580 ~ValueRef(); 581 582 inline bool exists() const { return value != NULL; } 583 584 void set(Value *); 585 void set(const ValueRef&); 586 inline Value *get() const { return value; } 587 inline Value *rep() const; 588 589 inline Instruction *getInsn() const { return insn; } 590 inline void setInsn(Instruction *inst) { insn = inst; } 591 592 inline bool isIndirect(int dim) const { return indirect[dim] >= 0; } 593 inline const ValueRef *getIndirect(int dim) const; 594 595 inline DataFile getFile() const; 596 inline unsigned getSize() const; 597 598 // SSA: return eventual (traverse MOVs) literal value, if it exists 599 bool getImmediate(ImmediateValue&) const; 600 601 public: 602 Modifier mod; 603 int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i]) 604 605 bool usedAsPtr; // for printing 606 607 private: 608 Value *value; 609 Instruction *insn; 610 }; 611 612 class ValueDef 613 { 614 public: 615 ValueDef(Value * = NULL); 616 ValueDef(const ValueDef&); 617 ~ValueDef(); 618 619 inline bool exists() const { return value != NULL; } 620 621 inline Value *get() const { return value; } 622 inline Value *rep() const; 623 void set(Value *); 624 bool mayReplace(const ValueRef &); 625 void replace(const ValueRef &, bool doSet); // replace all uses of the old value 626 627 inline Instruction *getInsn() const { return insn; } 628 inline void setInsn(Instruction *inst) { insn = inst; } 629 630 inline DataFile getFile() const; 631 inline unsigned getSize() const; 632 633 inline void setSSA(LValue *); 634 inline const LValue *preSSA() const; 635 636 private: 637 Value *value; // should make this LValue * ... 638 LValue *origin; // pre SSA value 639 Instruction *insn; 640 }; 641 642 class Value 643 { 644 public: 645 Value(); 646 virtual ~Value() { } 647 648 virtual Value *clone(ClonePolicy<Function>&) const = 0; 649 650 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0; 651 652 virtual bool equals(const Value *, bool strict = false) const; 653 virtual bool interfers(const Value *) const; 654 virtual bool isUniform() const { return true; } 655 656 inline Value *rep() const { return join; } 657 658 inline Instruction *getUniqueInsn() const; 659 inline Instruction *getInsn() const; // use when uniqueness is certain 660 661 inline int refCount() { return uses.size(); } 662 663 inline LValue *asLValue(); 664 inline Symbol *asSym(); 665 inline ImmediateValue *asImm(); 666 inline const Symbol *asSym() const; 667 inline const ImmediateValue *asImm() const; 668 669 inline bool inFile(DataFile f) const { return reg.file == f; } 670 671 static inline Value *get(Iterator&); 672 673 unordered_set<ValueRef *> uses; 674 std::list<ValueDef *> defs; 675 typedef unordered_set<ValueRef *>::iterator UseIterator; 676 typedef unordered_set<ValueRef *>::const_iterator UseCIterator; 677 typedef std::list<ValueDef *>::iterator DefIterator; 678 typedef std::list<ValueDef *>::const_iterator DefCIterator; 679 680 int id; 681 Storage reg; 682 683 // TODO: these should be in LValue: 684 Interval livei; 685 Value *join; 686 }; 687 688 class LValue : public Value 689 { 690 public: 691 LValue(Function *, DataFile file); 692 LValue(Function *, LValue *); 693 ~LValue() { } 694 695 virtual bool isUniform() const; 696 697 virtual LValue *clone(ClonePolicy<Function>&) const; 698 699 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const; 700 701 public: 702 unsigned compMask : 8; // compound/component mask 703 unsigned compound : 1; // used by RA, value involved in split/merge 704 unsigned ssa : 1; 705 unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0) 706 unsigned noSpill : 1; // do not spill (e.g. if spill temporary already) 707 }; 708 709 class Symbol : public Value 710 { 711 public: 712 Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0); 713 ~Symbol() { } 714 715 virtual Symbol *clone(ClonePolicy<Function>&) const; 716 717 virtual bool equals(const Value *that, bool strict) const; 718 719 virtual bool isUniform() const; 720 721 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const; 722 723 // print with indirect values 724 int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const; 725 726 inline void setFile(DataFile file, ubyte fileIndex = 0) 727 { 728 reg.file = file; 729 reg.fileIndex = fileIndex; 730 } 731 732 inline void setOffset(int32_t offset); 733 inline void setAddress(Symbol *base, int32_t offset); 734 inline void setSV(SVSemantic sv, uint32_t idx = 0); 735 736 inline const Symbol *getBase() const { return baseSym; } 737 738 private: 739 Symbol *baseSym; // array base for Symbols representing array elements 740 }; 741 742 class ImmediateValue : public Value 743 { 744 public: 745 ImmediateValue() { } 746 ImmediateValue(Program *, uint32_t); 747 ImmediateValue(Program *, float); 748 ImmediateValue(Program *, double); 749 // NOTE: not added to program with 750 ImmediateValue(const ImmediateValue *, DataType ty); 751 ~ImmediateValue() { }; 752 753 virtual ImmediateValue *clone(ClonePolicy<Function>&) const; 754 755 virtual bool equals(const Value *that, bool strict) const; 756 757 // these only work if 'type' is valid (we mostly use untyped literals): 758 bool isInteger(const int ival) const; // ival is cast to this' type 759 bool isNegative() const; 760 bool isPow2() const; 761 762 void applyLog2(); 763 764 // for constant folding: 765 ImmediateValue operator+(const ImmediateValue&) const; 766 ImmediateValue operator-(const ImmediateValue&) const; 767 ImmediateValue operator*(const ImmediateValue&) const; 768 ImmediateValue operator/(const ImmediateValue&) const; 769 770 ImmediateValue& operator=(const ImmediateValue&); // only sets value ! 771 772 bool compare(CondCode cc, float fval) const; 773 774 virtual int print(char *, size_t, DataType ty = TYPE_NONE) const; 775 }; 776 777 class Instruction 778 { 779 public: 780 Instruction(); 781 Instruction(Function *, operation, DataType); 782 virtual ~Instruction(); 783 784 virtual Instruction *clone(ClonePolicy<Function>&, 785 Instruction * = NULL) const; 786 787 void setDef(int i, Value *); 788 void setSrc(int s, Value *); 789 void setSrc(int s, const ValueRef&); 790 void swapSources(int a, int b); 791 void moveSources(int s, int delta); 792 bool setIndirect(int s, int dim, Value *); 793 794 inline ValueRef& src(int s) { return srcs[s]; } 795 inline ValueDef& def(int s) { return defs[s]; } 796 inline const ValueRef& src(int s) const { return srcs[s]; } 797 inline const ValueDef& def(int s) const { return defs[s]; } 798 799 inline Value *getDef(int d) const { return defs[d].get(); } 800 inline Value *getSrc(int s) const { return srcs[s].get(); } 801 inline Value *getIndirect(int s, int dim) const; 802 803 inline bool defExists(unsigned d) const 804 { 805 return d < defs.size() && defs[d].exists(); 806 } 807 inline bool srcExists(unsigned s) const 808 { 809 return s < srcs.size() && srcs[s].exists(); 810 } 811 812 inline bool constrainedDefs() const; 813 814 bool setPredicate(CondCode ccode, Value *); 815 inline Value *getPredicate() const; 816 bool writesPredicate() const; 817 inline bool isPredicated() const { return predSrc >= 0; } 818 819 inline void setFlagsSrc(int s, Value *); 820 inline void setFlagsDef(int d, Value *); 821 inline bool usesFlags() const { return flagsSrc >= 0; } 822 823 unsigned int defCount() const { return defs.size(); }; 824 unsigned int defCount(unsigned int mask, bool singleFile = false) const; 825 unsigned int srcCount() const { return srcs.size(); }; 826 unsigned int srcCount(unsigned int mask, bool singleFile = false) const; 827 828 // save & remove / set indirect[0,1] and predicate source 829 void takeExtraSources(int s, Value *[3]); 830 void putExtraSources(int s, Value *[3]); 831 832 inline void setType(DataType type) { dType = sType = type; } 833 834 inline void setType(DataType dtype, DataType stype) 835 { 836 dType = dtype; 837 sType = stype; 838 } 839 840 inline bool isPseudo() const { return op < OP_MOV; } 841 bool isDead() const; 842 bool isNop() const; 843 bool isCommutationLegal(const Instruction *) const; // must be adjacent ! 844 bool isActionEqual(const Instruction *) const; 845 bool isResultEqual(const Instruction *) const; 846 847 // check whether the defs interfere with srcs and defs of another instruction 848 bool canCommuteDefDef(const Instruction *) const; 849 bool canCommuteDefSrc(const Instruction *) const; 850 851 void print() const; 852 853 inline CmpInstruction *asCmp(); 854 inline TexInstruction *asTex(); 855 inline FlowInstruction *asFlow(); 856 inline const TexInstruction *asTex() const; 857 inline const CmpInstruction *asCmp() const; 858 inline const FlowInstruction *asFlow() const; 859 860 public: 861 Instruction *next; 862 Instruction *prev; 863 int id; 864 int serial; // CFG order 865 866 operation op; 867 DataType dType; // destination or defining type 868 DataType sType; // source or secondary type 869 CondCode cc; 870 RoundMode rnd; 871 CacheMode cache; 872 873 uint16_t subOp; // quadop, 1 for mul-high, etc. 874 875 unsigned encSize : 4; // encoding size in bytes 876 unsigned saturate : 1; // to [0.0f, 1.0f] 877 unsigned join : 1; // converge control flow (use OP_JOIN until end) 878 unsigned fixed : 1; // prevent dead code elimination 879 unsigned terminator : 1; // end of basic block 880 unsigned ftz : 1; // flush denormal to zero 881 unsigned dnz : 1; // denormals, NaN are zero 882 unsigned ipa : 4; // interpolation mode 883 unsigned lanes : 4; 884 unsigned perPatch : 1; 885 unsigned exit : 1; // terminate program after insn 886 unsigned mask : 4; // for vector ops 887 // prevent algebraic optimisations that aren't bit-for-bit identical 888 unsigned precise : 1; 889 890 int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor 891 892 int8_t predSrc; 893 int8_t flagsDef; 894 int8_t flagsSrc; 895 896 uint32_t sched; // scheduling data (NOTE: maybe move to separate storage) 897 898 BasicBlock *bb; 899 900 protected: 901 std::deque<ValueDef> defs; // no gaps ! 902 std::deque<ValueRef> srcs; // no gaps ! 903 904 // instruction specific methods: 905 // (don't want to subclass, would need more constructors and memory pools) 906 public: 907 inline void setInterpolate(unsigned int mode) { ipa = mode; } 908 909 unsigned int getInterpMode() const { return ipa & 0x3; } 910 unsigned int getSampleMode() const { return ipa & 0xc; } 911 912 private: 913 void init(); 914 }; 915 916 enum TexQuery 917 { 918 TXQ_DIMS, /* x, y, z, levels */ 919 TXQ_TYPE, /* ?, ?, samples, ? */ 920 TXQ_SAMPLE_POSITION, 921 TXQ_FILTER, 922 TXQ_LOD, 923 TXQ_WRAP, 924 TXQ_BORDER_COLOUR 925 }; 926 927 class TexInstruction : public Instruction 928 { 929 public: 930 class Target 931 { 932 public: 933 Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { } 934 935 const char *getName() const { return descTable[target].name; } 936 unsigned int getArgCount() const { return descTable[target].argc; } 937 unsigned int getDim() const { return descTable[target].dim; } 938 int isArray() const { return descTable[target].array ? 1 : 0; } 939 int isCube() const { return descTable[target].cube ? 1 : 0; } 940 int isShadow() const { return descTable[target].shadow ? 1 : 0; } 941 int isMS() const { 942 return target == TEX_TARGET_2D_MS || target == TEX_TARGET_2D_MS_ARRAY; } 943 void clearMS() { 944 if (isMS()) { 945 if (isArray()) 946 target = TEX_TARGET_2D_ARRAY; 947 else 948 target = TEX_TARGET_2D; 949 } 950 } 951 952 Target& operator=(TexTarget targ) 953 { 954 assert(targ < TEX_TARGET_COUNT); 955 target = targ; 956 return *this; 957 } 958 959 inline bool operator==(TexTarget targ) const { return target == targ; } 960 inline bool operator!=(TexTarget targ) const { return target != targ; } 961 962 enum TexTarget getEnum() const { return target; } 963 964 private: 965 struct Desc 966 { 967 char name[19]; 968 uint8_t dim; 969 uint8_t argc; 970 bool array; 971 bool cube; 972 bool shadow; 973 }; 974 975 static const struct Desc descTable[TEX_TARGET_COUNT]; 976 977 private: 978 enum TexTarget target; 979 }; 980 981 public: 982 struct ImgFormatDesc 983 { 984 char name[19]; 985 uint8_t components; 986 uint8_t bits[4]; 987 ImgType type; 988 bool bgra; 989 }; 990 991 static const struct ImgFormatDesc formatTable[IMG_FORMAT_COUNT]; 992 993 public: 994 TexInstruction(Function *, operation); 995 virtual ~TexInstruction(); 996 997 virtual TexInstruction *clone(ClonePolicy<Function>&, 998 Instruction * = NULL) const; 999 1000 inline void setTexture(Target targ, uint8_t r, uint8_t s) 1001 { 1002 tex.r = r; 1003 tex.s = s; 1004 tex.target = targ; 1005 } 1006 1007 void setIndirectR(Value *); 1008 void setIndirectS(Value *); 1009 inline Value *getIndirectR() const; 1010 inline Value *getIndirectS() const; 1011 1012 public: 1013 struct { 1014 Target target; 1015 1016 uint16_t r; 1017 uint16_t s; 1018 int8_t rIndirectSrc; 1019 int8_t sIndirectSrc; 1020 1021 uint8_t mask; 1022 uint8_t gatherComp; 1023 1024 bool liveOnly; // only execute on live pixels of a quad (optimization) 1025 bool levelZero; 1026 bool derivAll; 1027 bool bindless; 1028 1029 int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets 1030 int8_t offset[3]; // only used on nv50 1031 1032 enum TexQuery query; 1033 const struct ImgFormatDesc *format; 1034 } tex; 1035 1036 ValueRef dPdx[3]; 1037 ValueRef dPdy[3]; 1038 ValueRef offset[4][3]; 1039 }; 1040 1041 class CmpInstruction : public Instruction 1042 { 1043 public: 1044 CmpInstruction(Function *, operation); 1045 1046 virtual CmpInstruction *clone(ClonePolicy<Function>&, 1047 Instruction * = NULL) const; 1048 1049 void setCondition(CondCode cond) { setCond = cond; } 1050 CondCode getCondition() const { return setCond; } 1051 1052 public: 1053 CondCode setCond; 1054 }; 1055 1056 class FlowInstruction : public Instruction 1057 { 1058 public: 1059 FlowInstruction(Function *, operation, void *target); 1060 1061 virtual FlowInstruction *clone(ClonePolicy<Function>&, 1062 Instruction * = NULL) const; 1063 1064 public: 1065 unsigned allWarp : 1; 1066 unsigned absolute : 1; 1067 unsigned limit : 1; 1068 unsigned builtin : 1; // true for calls to emulation code 1069 unsigned indirect : 1; // target in src(0) 1070 1071 union { 1072 BasicBlock *bb; 1073 int builtin; 1074 Function *fn; 1075 } target; 1076 }; 1077 1078 class BasicBlock 1079 { 1080 public: 1081 BasicBlock(Function *); 1082 ~BasicBlock(); 1083 1084 BasicBlock *clone(ClonePolicy<Function>&) const; 1085 1086 inline int getId() const { return id; } 1087 inline unsigned int getInsnCount() const { return numInsns; } 1088 inline bool isTerminated() const { return exit && exit->terminator; } 1089 1090 bool dominatedBy(BasicBlock *bb); 1091 inline bool reachableBy(const BasicBlock *by, const BasicBlock *term); 1092 1093 // returns mask of conditional out blocks 1094 // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF 1095 unsigned int initiatesSimpleConditional() const; 1096 1097 public: 1098 Function *getFunction() const { return func; } 1099 Program *getProgram() const { return program; } 1100 1101 Instruction *getEntry() const { return entry; } // first non-phi instruction 1102 Instruction *getPhi() const { return phi; } 1103 Instruction *getFirst() const { return phi ? phi : entry; } 1104 Instruction *getExit() const { return exit; } 1105 1106 void insertHead(Instruction *); 1107 void insertTail(Instruction *); 1108 void insertBefore(Instruction *, Instruction *); 1109 void insertAfter(Instruction *, Instruction *); 1110 void remove(Instruction *); 1111 void permuteAdjacent(Instruction *, Instruction *); 1112 1113 BasicBlock *idom() const; 1114 1115 // NOTE: currently does not rebuild the dominator tree 1116 BasicBlock *splitBefore(Instruction *, bool attach = true); 1117 BasicBlock *splitAfter(Instruction *, bool attach = true); 1118 1119 DLList& getDF() { return df; } 1120 DLList::Iterator iterDF() { return df.iterator(); } 1121 1122 static inline BasicBlock *get(Iterator&); 1123 static inline BasicBlock *get(Graph::Node *); 1124 1125 public: 1126 Graph::Node cfg; // first edge is branch *taken* (the ELSE branch) 1127 Graph::Node dom; 1128 1129 BitSet liveSet; 1130 BitSet defSet; 1131 1132 uint32_t binPos; 1133 uint32_t binSize; 1134 1135 Instruction *joinAt; // for quick reference 1136 1137 bool explicitCont; // loop headers: true if loop contains continue stmts 1138 1139 private: 1140 int id; 1141 DLList df; 1142 1143 Instruction *phi; 1144 Instruction *entry; 1145 Instruction *exit; 1146 1147 unsigned int numInsns; 1148 1149 private: 1150 Function *func; 1151 Program *program; 1152 1153 void splitCommon(Instruction *, BasicBlock *, bool attach); 1154 }; 1155 1156 class Function 1157 { 1158 public: 1159 Function(Program *, const char *name, uint32_t label); 1160 ~Function(); 1161 1162 static inline Function *get(Graph::Node *node); 1163 1164 inline Program *getProgram() const { return prog; } 1165 inline const char *getName() const { return name; } 1166 inline int getId() const { return id; } 1167 inline uint32_t getLabel() const { return label; } 1168 1169 void print(); 1170 void printLiveIntervals() const; 1171 void printCFGraph(const char *filePath); 1172 1173 bool setEntry(BasicBlock *); 1174 bool setExit(BasicBlock *); 1175 1176 unsigned int orderInstructions(ArrayList&); 1177 1178 inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); } 1179 inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); } 1180 inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); } 1181 1182 inline LValue *getLValue(int id); 1183 1184 void buildLiveSets(); 1185 void buildDefSets(); 1186 bool convertToSSA(); 1187 1188 public: 1189 std::deque<ValueDef> ins; 1190 std::deque<ValueRef> outs; 1191 std::deque<Value *> clobbers; 1192 1193 Graph cfg; 1194 Graph::Node *cfgExit; 1195 Graph *domTree; 1196 Graph::Node call; // node in the call graph 1197 1198 BasicBlock **bbArray; // BBs in emission order 1199 int bbCount; 1200 1201 unsigned int loopNestingBound; 1202 int regClobberMax; 1203 1204 uint32_t binPos; 1205 uint32_t binSize; 1206 1207 Value *stackPtr; 1208 1209 uint32_t tlsBase; // base address for l[] space (if no stack pointer is used) 1210 uint32_t tlsSize; 1211 1212 ArrayList allBBlocks; 1213 ArrayList allInsns; 1214 ArrayList allLValues; 1215 1216 private: 1217 void buildLiveSetsPreSSA(BasicBlock *, const int sequence); 1218 void buildDefSetsPreSSA(BasicBlock *bb, const int seq); 1219 1220 private: 1221 uint32_t label; 1222 int id; 1223 const char *const name; 1224 Program *prog; 1225 }; 1226 1227 enum CGStage 1228 { 1229 CG_STAGE_PRE_SSA, 1230 CG_STAGE_SSA, // expected directly before register allocation 1231 CG_STAGE_POST_RA 1232 }; 1233 1234 class Program 1235 { 1236 public: 1237 enum Type 1238 { 1239 TYPE_VERTEX, 1240 TYPE_TESSELLATION_CONTROL, 1241 TYPE_TESSELLATION_EVAL, 1242 TYPE_GEOMETRY, 1243 TYPE_FRAGMENT, 1244 TYPE_COMPUTE 1245 }; 1246 1247 Program(Type type, Target *targ); 1248 ~Program(); 1249 1250 void print(); 1251 1252 Type getType() const { return progType; } 1253 1254 inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); } 1255 inline void del(Function *fn, int& id) { allFuncs.remove(id); } 1256 inline void add(Value *rval, int& id) { allRValues.insert(rval, id); } 1257 1258 bool makeFromTGSI(struct nv50_ir_prog_info *); 1259 bool convertToSSA(); 1260 bool optimizeSSA(int level); 1261 bool optimizePostRA(int level); 1262 bool registerAllocation(); 1263 bool emitBinary(struct nv50_ir_prog_info *); 1264 1265 const Target *getTarget() const { return target; } 1266 1267 private: 1268 void emitSymbolTable(struct nv50_ir_prog_info *); 1269 1270 Type progType; 1271 Target *target; 1272 1273 public: 1274 Function *main; 1275 Graph calls; 1276 1277 ArrayList allFuncs; 1278 ArrayList allRValues; 1279 1280 uint32_t *code; 1281 uint32_t binSize; 1282 uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL 1283 1284 int maxGPR; 1285 1286 MemoryPool mem_Instruction; 1287 MemoryPool mem_CmpInstruction; 1288 MemoryPool mem_TexInstruction; 1289 MemoryPool mem_FlowInstruction; 1290 MemoryPool mem_LValue; 1291 MemoryPool mem_Symbol; 1292 MemoryPool mem_ImmediateValue; 1293 1294 uint32_t dbgFlags; 1295 uint8_t optLevel; 1296 1297 void *targetPriv; // e.g. to carry information between passes 1298 1299 const struct nv50_ir_prog_info *driver; // for driver configuration 1300 1301 void releaseInstruction(Instruction *); 1302 void releaseValue(Value *); 1303 }; 1304 1305 // TODO: add const version 1306 class Pass 1307 { 1308 public: 1309 bool run(Program *, bool ordered = false, bool skipPhi = false); 1310 bool run(Function *, bool ordered = false, bool skipPhi = false); 1311 1312 private: 1313 // return false to continue with next entity on next higher level 1314 virtual bool visit(Function *) { return true; } 1315 virtual bool visit(BasicBlock *) { return true; } 1316 virtual bool visit(Instruction *) { return false; } 1317 1318 bool doRun(Program *, bool ordered, bool skipPhi); 1319 bool doRun(Function *, bool ordered, bool skipPhi); 1320 1321 protected: 1322 bool err; 1323 Function *func; 1324 Program *prog; 1325 }; 1326 1327 // ============================================================================= 1328 1329 #include "codegen/nv50_ir_inlines.h" 1330 1331 } // namespace nv50_ir 1332 1333 #endif // __NV50_IR_H__ 1334
