1 /* 2 * Copyright 2014 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Connor Abbott (cwabbott0 (at) gmail.com) 25 * 26 */ 27 28 #include "nir.h" 29 #include "nir_control_flow_private.h" 30 #include <assert.h> 31 32 nir_shader * 33 nir_shader_create(void *mem_ctx, 34 gl_shader_stage stage, 35 const nir_shader_compiler_options *options, 36 shader_info *si) 37 { 38 nir_shader *shader = rzalloc(mem_ctx, nir_shader); 39 40 exec_list_make_empty(&shader->uniforms); 41 exec_list_make_empty(&shader->inputs); 42 exec_list_make_empty(&shader->outputs); 43 exec_list_make_empty(&shader->shared); 44 45 shader->options = options; 46 47 shader->info = si ? si : rzalloc(shader, shader_info); 48 49 exec_list_make_empty(&shader->functions); 50 exec_list_make_empty(&shader->registers); 51 exec_list_make_empty(&shader->globals); 52 exec_list_make_empty(&shader->system_values); 53 shader->reg_alloc = 0; 54 55 shader->num_inputs = 0; 56 shader->num_outputs = 0; 57 shader->num_uniforms = 0; 58 shader->num_shared = 0; 59 60 shader->stage = stage; 61 62 return shader; 63 } 64 65 static nir_register * 66 reg_create(void *mem_ctx, struct exec_list *list) 67 { 68 nir_register *reg = ralloc(mem_ctx, nir_register); 69 70 list_inithead(®->uses); 71 list_inithead(®->defs); 72 list_inithead(®->if_uses); 73 74 reg->num_components = 0; 75 reg->bit_size = 32; 76 reg->num_array_elems = 0; 77 reg->is_packed = false; 78 reg->name = NULL; 79 80 exec_list_push_tail(list, ®->node); 81 82 return reg; 83 } 84 85 nir_register * 86 nir_global_reg_create(nir_shader *shader) 87 { 88 nir_register *reg = reg_create(shader, &shader->registers); 89 reg->index = shader->reg_alloc++; 90 reg->is_global = true; 91 92 return reg; 93 } 94 95 nir_register * 96 nir_local_reg_create(nir_function_impl *impl) 97 { 98 nir_register *reg = reg_create(ralloc_parent(impl), &impl->registers); 99 reg->index = impl->reg_alloc++; 100 reg->is_global = false; 101 102 return reg; 103 } 104 105 void 106 nir_reg_remove(nir_register *reg) 107 { 108 exec_node_remove(®->node); 109 } 110 111 void 112 nir_shader_add_variable(nir_shader *shader, nir_variable *var) 113 { 114 switch (var->data.mode) { 115 case nir_var_all: 116 assert(!"invalid mode"); 117 break; 118 119 case nir_var_local: 120 assert(!"nir_shader_add_variable cannot be used for local variables"); 121 break; 122 123 case nir_var_param: 124 assert(!"nir_shader_add_variable cannot be used for function parameters"); 125 break; 126 127 case nir_var_global: 128 exec_list_push_tail(&shader->globals, &var->node); 129 break; 130 131 case nir_var_shader_in: 132 exec_list_push_tail(&shader->inputs, &var->node); 133 break; 134 135 case nir_var_shader_out: 136 exec_list_push_tail(&shader->outputs, &var->node); 137 break; 138 139 case nir_var_uniform: 140 case nir_var_shader_storage: 141 exec_list_push_tail(&shader->uniforms, &var->node); 142 break; 143 144 case nir_var_shared: 145 assert(shader->stage == MESA_SHADER_COMPUTE); 146 exec_list_push_tail(&shader->shared, &var->node); 147 break; 148 149 case nir_var_system_value: 150 exec_list_push_tail(&shader->system_values, &var->node); 151 break; 152 } 153 } 154 155 nir_variable * 156 nir_variable_create(nir_shader *shader, nir_variable_mode mode, 157 const struct glsl_type *type, const char *name) 158 { 159 nir_variable *var = rzalloc(shader, nir_variable); 160 var->name = ralloc_strdup(var, name); 161 var->type = type; 162 var->data.mode = mode; 163 164 if ((mode == nir_var_shader_in && shader->stage != MESA_SHADER_VERTEX) || 165 (mode == nir_var_shader_out && shader->stage != MESA_SHADER_FRAGMENT)) 166 var->data.interpolation = INTERP_MODE_SMOOTH; 167 168 if (mode == nir_var_shader_in || mode == nir_var_uniform) 169 var->data.read_only = true; 170 171 nir_shader_add_variable(shader, var); 172 173 return var; 174 } 175 176 nir_variable * 177 nir_local_variable_create(nir_function_impl *impl, 178 const struct glsl_type *type, const char *name) 179 { 180 nir_variable *var = rzalloc(impl->function->shader, nir_variable); 181 var->name = ralloc_strdup(var, name); 182 var->type = type; 183 var->data.mode = nir_var_local; 184 185 nir_function_impl_add_variable(impl, var); 186 187 return var; 188 } 189 190 nir_function * 191 nir_function_create(nir_shader *shader, const char *name) 192 { 193 nir_function *func = ralloc(shader, nir_function); 194 195 exec_list_push_tail(&shader->functions, &func->node); 196 197 func->name = ralloc_strdup(func, name); 198 func->shader = shader; 199 func->num_params = 0; 200 func->params = NULL; 201 func->return_type = glsl_void_type(); 202 func->impl = NULL; 203 204 return func; 205 } 206 207 void nir_src_copy(nir_src *dest, const nir_src *src, void *mem_ctx) 208 { 209 dest->is_ssa = src->is_ssa; 210 if (src->is_ssa) { 211 dest->ssa = src->ssa; 212 } else { 213 dest->reg.base_offset = src->reg.base_offset; 214 dest->reg.reg = src->reg.reg; 215 if (src->reg.indirect) { 216 dest->reg.indirect = ralloc(mem_ctx, nir_src); 217 nir_src_copy(dest->reg.indirect, src->reg.indirect, mem_ctx); 218 } else { 219 dest->reg.indirect = NULL; 220 } 221 } 222 } 223 224 void nir_dest_copy(nir_dest *dest, const nir_dest *src, nir_instr *instr) 225 { 226 /* Copying an SSA definition makes no sense whatsoever. */ 227 assert(!src->is_ssa); 228 229 dest->is_ssa = false; 230 231 dest->reg.base_offset = src->reg.base_offset; 232 dest->reg.reg = src->reg.reg; 233 if (src->reg.indirect) { 234 dest->reg.indirect = ralloc(instr, nir_src); 235 nir_src_copy(dest->reg.indirect, src->reg.indirect, instr); 236 } else { 237 dest->reg.indirect = NULL; 238 } 239 } 240 241 void 242 nir_alu_src_copy(nir_alu_src *dest, const nir_alu_src *src, 243 nir_alu_instr *instr) 244 { 245 nir_src_copy(&dest->src, &src->src, &instr->instr); 246 dest->abs = src->abs; 247 dest->negate = src->negate; 248 for (unsigned i = 0; i < 4; i++) 249 dest->swizzle[i] = src->swizzle[i]; 250 } 251 252 void 253 nir_alu_dest_copy(nir_alu_dest *dest, const nir_alu_dest *src, 254 nir_alu_instr *instr) 255 { 256 nir_dest_copy(&dest->dest, &src->dest, &instr->instr); 257 dest->write_mask = src->write_mask; 258 dest->saturate = src->saturate; 259 } 260 261 262 static void 263 cf_init(nir_cf_node *node, nir_cf_node_type type) 264 { 265 exec_node_init(&node->node); 266 node->parent = NULL; 267 node->type = type; 268 } 269 270 nir_function_impl * 271 nir_function_impl_create_bare(nir_shader *shader) 272 { 273 nir_function_impl *impl = ralloc(shader, nir_function_impl); 274 275 impl->function = NULL; 276 277 cf_init(&impl->cf_node, nir_cf_node_function); 278 279 exec_list_make_empty(&impl->body); 280 exec_list_make_empty(&impl->registers); 281 exec_list_make_empty(&impl->locals); 282 impl->num_params = 0; 283 impl->params = NULL; 284 impl->return_var = NULL; 285 impl->reg_alloc = 0; 286 impl->ssa_alloc = 0; 287 impl->valid_metadata = nir_metadata_none; 288 289 /* create start & end blocks */ 290 nir_block *start_block = nir_block_create(shader); 291 nir_block *end_block = nir_block_create(shader); 292 start_block->cf_node.parent = &impl->cf_node; 293 end_block->cf_node.parent = &impl->cf_node; 294 impl->end_block = end_block; 295 296 exec_list_push_tail(&impl->body, &start_block->cf_node.node); 297 298 start_block->successors[0] = end_block; 299 _mesa_set_add(end_block->predecessors, start_block); 300 return impl; 301 } 302 303 nir_function_impl * 304 nir_function_impl_create(nir_function *function) 305 { 306 assert(function->impl == NULL); 307 308 nir_function_impl *impl = nir_function_impl_create_bare(function->shader); 309 310 function->impl = impl; 311 impl->function = function; 312 313 impl->num_params = function->num_params; 314 impl->params = ralloc_array(function->shader, 315 nir_variable *, impl->num_params); 316 317 for (unsigned i = 0; i < impl->num_params; i++) { 318 impl->params[i] = rzalloc(function->shader, nir_variable); 319 impl->params[i]->type = function->params[i].type; 320 impl->params[i]->data.mode = nir_var_param; 321 impl->params[i]->data.location = i; 322 } 323 324 if (!glsl_type_is_void(function->return_type)) { 325 impl->return_var = rzalloc(function->shader, nir_variable); 326 impl->return_var->type = function->return_type; 327 impl->return_var->data.mode = nir_var_param; 328 impl->return_var->data.location = -1; 329 } else { 330 impl->return_var = NULL; 331 } 332 333 return impl; 334 } 335 336 nir_block * 337 nir_block_create(nir_shader *shader) 338 { 339 nir_block *block = rzalloc(shader, nir_block); 340 341 cf_init(&block->cf_node, nir_cf_node_block); 342 343 block->successors[0] = block->successors[1] = NULL; 344 block->predecessors = _mesa_set_create(block, _mesa_hash_pointer, 345 _mesa_key_pointer_equal); 346 block->imm_dom = NULL; 347 /* XXX maybe it would be worth it to defer allocation? This 348 * way it doesn't get allocated for shader ref's that never run 349 * nir_calc_dominance? For example, state-tracker creates an 350 * initial IR, clones that, runs appropriate lowering pass, passes 351 * to driver which does common lowering/opt, and then stores ref 352 * which is later used to do state specific lowering and futher 353 * opt. Do any of the references not need dominance metadata? 354 */ 355 block->dom_frontier = _mesa_set_create(block, _mesa_hash_pointer, 356 _mesa_key_pointer_equal); 357 358 exec_list_make_empty(&block->instr_list); 359 360 return block; 361 } 362 363 static inline void 364 src_init(nir_src *src) 365 { 366 src->is_ssa = false; 367 src->reg.reg = NULL; 368 src->reg.indirect = NULL; 369 src->reg.base_offset = 0; 370 } 371 372 nir_if * 373 nir_if_create(nir_shader *shader) 374 { 375 nir_if *if_stmt = ralloc(shader, nir_if); 376 377 cf_init(&if_stmt->cf_node, nir_cf_node_if); 378 src_init(&if_stmt->condition); 379 380 nir_block *then = nir_block_create(shader); 381 exec_list_make_empty(&if_stmt->then_list); 382 exec_list_push_tail(&if_stmt->then_list, &then->cf_node.node); 383 then->cf_node.parent = &if_stmt->cf_node; 384 385 nir_block *else_stmt = nir_block_create(shader); 386 exec_list_make_empty(&if_stmt->else_list); 387 exec_list_push_tail(&if_stmt->else_list, &else_stmt->cf_node.node); 388 else_stmt->cf_node.parent = &if_stmt->cf_node; 389 390 return if_stmt; 391 } 392 393 nir_loop * 394 nir_loop_create(nir_shader *shader) 395 { 396 nir_loop *loop = rzalloc(shader, nir_loop); 397 398 cf_init(&loop->cf_node, nir_cf_node_loop); 399 400 nir_block *body = nir_block_create(shader); 401 exec_list_make_empty(&loop->body); 402 exec_list_push_tail(&loop->body, &body->cf_node.node); 403 body->cf_node.parent = &loop->cf_node; 404 405 body->successors[0] = body; 406 _mesa_set_add(body->predecessors, body); 407 408 return loop; 409 } 410 411 static void 412 instr_init(nir_instr *instr, nir_instr_type type) 413 { 414 instr->type = type; 415 instr->block = NULL; 416 exec_node_init(&instr->node); 417 } 418 419 static void 420 dest_init(nir_dest *dest) 421 { 422 dest->is_ssa = false; 423 dest->reg.reg = NULL; 424 dest->reg.indirect = NULL; 425 dest->reg.base_offset = 0; 426 } 427 428 static void 429 alu_dest_init(nir_alu_dest *dest) 430 { 431 dest_init(&dest->dest); 432 dest->saturate = false; 433 dest->write_mask = 0xf; 434 } 435 436 static void 437 alu_src_init(nir_alu_src *src) 438 { 439 src_init(&src->src); 440 src->abs = src->negate = false; 441 src->swizzle[0] = 0; 442 src->swizzle[1] = 1; 443 src->swizzle[2] = 2; 444 src->swizzle[3] = 3; 445 } 446 447 nir_alu_instr * 448 nir_alu_instr_create(nir_shader *shader, nir_op op) 449 { 450 unsigned num_srcs = nir_op_infos[op].num_inputs; 451 /* TODO: don't use rzalloc */ 452 nir_alu_instr *instr = 453 rzalloc_size(shader, 454 sizeof(nir_alu_instr) + num_srcs * sizeof(nir_alu_src)); 455 456 instr_init(&instr->instr, nir_instr_type_alu); 457 instr->op = op; 458 alu_dest_init(&instr->dest); 459 for (unsigned i = 0; i < num_srcs; i++) 460 alu_src_init(&instr->src[i]); 461 462 return instr; 463 } 464 465 nir_jump_instr * 466 nir_jump_instr_create(nir_shader *shader, nir_jump_type type) 467 { 468 nir_jump_instr *instr = ralloc(shader, nir_jump_instr); 469 instr_init(&instr->instr, nir_instr_type_jump); 470 instr->type = type; 471 return instr; 472 } 473 474 nir_load_const_instr * 475 nir_load_const_instr_create(nir_shader *shader, unsigned num_components, 476 unsigned bit_size) 477 { 478 nir_load_const_instr *instr = ralloc(shader, nir_load_const_instr); 479 instr_init(&instr->instr, nir_instr_type_load_const); 480 481 nir_ssa_def_init(&instr->instr, &instr->def, num_components, bit_size, NULL); 482 483 return instr; 484 } 485 486 nir_intrinsic_instr * 487 nir_intrinsic_instr_create(nir_shader *shader, nir_intrinsic_op op) 488 { 489 unsigned num_srcs = nir_intrinsic_infos[op].num_srcs; 490 /* TODO: don't use rzalloc */ 491 nir_intrinsic_instr *instr = 492 rzalloc_size(shader, 493 sizeof(nir_intrinsic_instr) + num_srcs * sizeof(nir_src)); 494 495 instr_init(&instr->instr, nir_instr_type_intrinsic); 496 instr->intrinsic = op; 497 498 if (nir_intrinsic_infos[op].has_dest) 499 dest_init(&instr->dest); 500 501 for (unsigned i = 0; i < num_srcs; i++) 502 src_init(&instr->src[i]); 503 504 return instr; 505 } 506 507 nir_call_instr * 508 nir_call_instr_create(nir_shader *shader, nir_function *callee) 509 { 510 nir_call_instr *instr = ralloc(shader, nir_call_instr); 511 instr_init(&instr->instr, nir_instr_type_call); 512 513 instr->callee = callee; 514 instr->num_params = callee->num_params; 515 instr->params = ralloc_array(instr, nir_deref_var *, instr->num_params); 516 instr->return_deref = NULL; 517 518 return instr; 519 } 520 521 nir_tex_instr * 522 nir_tex_instr_create(nir_shader *shader, unsigned num_srcs) 523 { 524 nir_tex_instr *instr = rzalloc(shader, nir_tex_instr); 525 instr_init(&instr->instr, nir_instr_type_tex); 526 527 dest_init(&instr->dest); 528 529 instr->num_srcs = num_srcs; 530 instr->src = ralloc_array(instr, nir_tex_src, num_srcs); 531 for (unsigned i = 0; i < num_srcs; i++) 532 src_init(&instr->src[i].src); 533 534 instr->texture_index = 0; 535 instr->texture_array_size = 0; 536 instr->texture = NULL; 537 instr->sampler_index = 0; 538 instr->sampler = NULL; 539 540 return instr; 541 } 542 543 void 544 nir_tex_instr_remove_src(nir_tex_instr *tex, unsigned src_idx) 545 { 546 assert(src_idx < tex->num_srcs); 547 548 /* First rewrite the source to NIR_SRC_INIT */ 549 nir_instr_rewrite_src(&tex->instr, &tex->src[src_idx].src, NIR_SRC_INIT); 550 551 /* Now, move all of the other sources down */ 552 for (unsigned i = src_idx + 1; i < tex->num_srcs; i++) { 553 tex->src[i-1].src_type = tex->src[i].src_type; 554 nir_instr_move_src(&tex->instr, &tex->src[i-1].src, &tex->src[i].src); 555 } 556 tex->num_srcs--; 557 } 558 559 nir_phi_instr * 560 nir_phi_instr_create(nir_shader *shader) 561 { 562 nir_phi_instr *instr = ralloc(shader, nir_phi_instr); 563 instr_init(&instr->instr, nir_instr_type_phi); 564 565 dest_init(&instr->dest); 566 exec_list_make_empty(&instr->srcs); 567 return instr; 568 } 569 570 nir_parallel_copy_instr * 571 nir_parallel_copy_instr_create(nir_shader *shader) 572 { 573 nir_parallel_copy_instr *instr = ralloc(shader, nir_parallel_copy_instr); 574 instr_init(&instr->instr, nir_instr_type_parallel_copy); 575 576 exec_list_make_empty(&instr->entries); 577 578 return instr; 579 } 580 581 nir_ssa_undef_instr * 582 nir_ssa_undef_instr_create(nir_shader *shader, 583 unsigned num_components, 584 unsigned bit_size) 585 { 586 nir_ssa_undef_instr *instr = ralloc(shader, nir_ssa_undef_instr); 587 instr_init(&instr->instr, nir_instr_type_ssa_undef); 588 589 nir_ssa_def_init(&instr->instr, &instr->def, num_components, bit_size, NULL); 590 591 return instr; 592 } 593 594 nir_deref_var * 595 nir_deref_var_create(void *mem_ctx, nir_variable *var) 596 { 597 nir_deref_var *deref = ralloc(mem_ctx, nir_deref_var); 598 deref->deref.deref_type = nir_deref_type_var; 599 deref->deref.child = NULL; 600 deref->deref.type = var->type; 601 deref->var = var; 602 return deref; 603 } 604 605 nir_deref_array * 606 nir_deref_array_create(void *mem_ctx) 607 { 608 nir_deref_array *deref = ralloc(mem_ctx, nir_deref_array); 609 deref->deref.deref_type = nir_deref_type_array; 610 deref->deref.child = NULL; 611 deref->deref_array_type = nir_deref_array_type_direct; 612 src_init(&deref->indirect); 613 deref->base_offset = 0; 614 return deref; 615 } 616 617 nir_deref_struct * 618 nir_deref_struct_create(void *mem_ctx, unsigned field_index) 619 { 620 nir_deref_struct *deref = ralloc(mem_ctx, nir_deref_struct); 621 deref->deref.deref_type = nir_deref_type_struct; 622 deref->deref.child = NULL; 623 deref->index = field_index; 624 return deref; 625 } 626 627 nir_deref_var * 628 nir_deref_var_clone(const nir_deref_var *deref, void *mem_ctx) 629 { 630 if (deref == NULL) 631 return NULL; 632 633 nir_deref_var *ret = nir_deref_var_create(mem_ctx, deref->var); 634 ret->deref.type = deref->deref.type; 635 if (deref->deref.child) 636 ret->deref.child = nir_deref_clone(deref->deref.child, ret); 637 return ret; 638 } 639 640 static nir_deref_array * 641 deref_array_clone(const nir_deref_array *deref, void *mem_ctx) 642 { 643 nir_deref_array *ret = nir_deref_array_create(mem_ctx); 644 ret->base_offset = deref->base_offset; 645 ret->deref_array_type = deref->deref_array_type; 646 if (deref->deref_array_type == nir_deref_array_type_indirect) { 647 nir_src_copy(&ret->indirect, &deref->indirect, mem_ctx); 648 } 649 ret->deref.type = deref->deref.type; 650 if (deref->deref.child) 651 ret->deref.child = nir_deref_clone(deref->deref.child, ret); 652 return ret; 653 } 654 655 static nir_deref_struct * 656 deref_struct_clone(const nir_deref_struct *deref, void *mem_ctx) 657 { 658 nir_deref_struct *ret = nir_deref_struct_create(mem_ctx, deref->index); 659 ret->deref.type = deref->deref.type; 660 if (deref->deref.child) 661 ret->deref.child = nir_deref_clone(deref->deref.child, ret); 662 return ret; 663 } 664 665 nir_deref * 666 nir_deref_clone(const nir_deref *deref, void *mem_ctx) 667 { 668 if (deref == NULL) 669 return NULL; 670 671 switch (deref->deref_type) { 672 case nir_deref_type_var: 673 return &nir_deref_var_clone(nir_deref_as_var(deref), mem_ctx)->deref; 674 case nir_deref_type_array: 675 return &deref_array_clone(nir_deref_as_array(deref), mem_ctx)->deref; 676 case nir_deref_type_struct: 677 return &deref_struct_clone(nir_deref_as_struct(deref), mem_ctx)->deref; 678 default: 679 unreachable("Invalid dereference type"); 680 } 681 682 return NULL; 683 } 684 685 /* This is the second step in the recursion. We've found the tail and made a 686 * copy. Now we need to iterate over all possible leaves and call the 687 * callback on each one. 688 */ 689 static bool 690 deref_foreach_leaf_build_recur(nir_deref_var *deref, nir_deref *tail, 691 nir_deref_foreach_leaf_cb cb, void *state) 692 { 693 unsigned length; 694 union { 695 nir_deref_array arr; 696 nir_deref_struct str; 697 } tmp; 698 699 assert(tail->child == NULL); 700 switch (glsl_get_base_type(tail->type)) { 701 case GLSL_TYPE_UINT: 702 case GLSL_TYPE_INT: 703 case GLSL_TYPE_FLOAT: 704 case GLSL_TYPE_DOUBLE: 705 case GLSL_TYPE_BOOL: 706 if (glsl_type_is_vector_or_scalar(tail->type)) 707 return cb(deref, state); 708 /* Fall Through */ 709 710 case GLSL_TYPE_ARRAY: 711 tmp.arr.deref.deref_type = nir_deref_type_array; 712 tmp.arr.deref.type = glsl_get_array_element(tail->type); 713 tmp.arr.deref_array_type = nir_deref_array_type_direct; 714 tmp.arr.indirect = NIR_SRC_INIT; 715 tail->child = &tmp.arr.deref; 716 717 length = glsl_get_length(tail->type); 718 for (unsigned i = 0; i < length; i++) { 719 tmp.arr.deref.child = NULL; 720 tmp.arr.base_offset = i; 721 if (!deref_foreach_leaf_build_recur(deref, &tmp.arr.deref, cb, state)) 722 return false; 723 } 724 return true; 725 726 case GLSL_TYPE_STRUCT: 727 tmp.str.deref.deref_type = nir_deref_type_struct; 728 tail->child = &tmp.str.deref; 729 730 length = glsl_get_length(tail->type); 731 for (unsigned i = 0; i < length; i++) { 732 tmp.arr.deref.child = NULL; 733 tmp.str.deref.type = glsl_get_struct_field(tail->type, i); 734 tmp.str.index = i; 735 if (!deref_foreach_leaf_build_recur(deref, &tmp.arr.deref, cb, state)) 736 return false; 737 } 738 return true; 739 740 default: 741 unreachable("Invalid type for dereference"); 742 } 743 } 744 745 /* This is the first step of the foreach_leaf recursion. In this step we are 746 * walking to the end of the deref chain and making a copy in the stack as we 747 * go. This is because we don't want to mutate the deref chain that was 748 * passed in by the caller. The downside is that this deref chain is on the 749 * stack and , if the caller wants to do anything with it, they will have to 750 * make their own copy because this one will go away. 751 */ 752 static bool 753 deref_foreach_leaf_copy_recur(nir_deref_var *deref, nir_deref *tail, 754 nir_deref_foreach_leaf_cb cb, void *state) 755 { 756 union { 757 nir_deref_array arr; 758 nir_deref_struct str; 759 } c; 760 761 if (tail->child) { 762 switch (tail->child->deref_type) { 763 case nir_deref_type_array: 764 c.arr = *nir_deref_as_array(tail->child); 765 tail->child = &c.arr.deref; 766 return deref_foreach_leaf_copy_recur(deref, &c.arr.deref, cb, state); 767 768 case nir_deref_type_struct: 769 c.str = *nir_deref_as_struct(tail->child); 770 tail->child = &c.str.deref; 771 return deref_foreach_leaf_copy_recur(deref, &c.str.deref, cb, state); 772 773 case nir_deref_type_var: 774 default: 775 unreachable("Invalid deref type for a child"); 776 } 777 } else { 778 /* We've gotten to the end of the original deref. Time to start 779 * building our own derefs. 780 */ 781 return deref_foreach_leaf_build_recur(deref, tail, cb, state); 782 } 783 } 784 785 /** 786 * This function iterates over all of the possible derefs that can be created 787 * with the given deref as the head. It then calls the provided callback with 788 * a full deref for each one. 789 * 790 * The deref passed to the callback will be allocated on the stack. You will 791 * need to make a copy if you want it to hang around. 792 */ 793 bool 794 nir_deref_foreach_leaf(nir_deref_var *deref, 795 nir_deref_foreach_leaf_cb cb, void *state) 796 { 797 nir_deref_var copy = *deref; 798 return deref_foreach_leaf_copy_recur(©, ©.deref, cb, state); 799 } 800 801 /* Returns a load_const instruction that represents the constant 802 * initializer for the given deref chain. The caller is responsible for 803 * ensuring that there actually is a constant initializer. 804 */ 805 nir_load_const_instr * 806 nir_deref_get_const_initializer_load(nir_shader *shader, nir_deref_var *deref) 807 { 808 nir_constant *constant = deref->var->constant_initializer; 809 assert(constant); 810 811 const nir_deref *tail = &deref->deref; 812 unsigned matrix_col = 0; 813 while (tail->child) { 814 switch (tail->child->deref_type) { 815 case nir_deref_type_array: { 816 nir_deref_array *arr = nir_deref_as_array(tail->child); 817 assert(arr->deref_array_type == nir_deref_array_type_direct); 818 if (glsl_type_is_matrix(tail->type)) { 819 assert(arr->deref.child == NULL); 820 matrix_col = arr->base_offset; 821 } else { 822 constant = constant->elements[arr->base_offset]; 823 } 824 break; 825 } 826 827 case nir_deref_type_struct: { 828 constant = constant->elements[nir_deref_as_struct(tail->child)->index]; 829 break; 830 } 831 832 default: 833 unreachable("Invalid deref child type"); 834 } 835 836 tail = tail->child; 837 } 838 839 unsigned bit_size = glsl_get_bit_size(tail->type); 840 nir_load_const_instr *load = 841 nir_load_const_instr_create(shader, glsl_get_vector_elements(tail->type), 842 bit_size); 843 844 switch (glsl_get_base_type(tail->type)) { 845 case GLSL_TYPE_FLOAT: 846 case GLSL_TYPE_INT: 847 case GLSL_TYPE_UINT: 848 case GLSL_TYPE_DOUBLE: 849 case GLSL_TYPE_BOOL: 850 load->value = constant->values[matrix_col]; 851 break; 852 default: 853 unreachable("Invalid immediate type"); 854 } 855 856 return load; 857 } 858 859 nir_function_impl * 860 nir_cf_node_get_function(nir_cf_node *node) 861 { 862 while (node->type != nir_cf_node_function) { 863 node = node->parent; 864 } 865 866 return nir_cf_node_as_function(node); 867 } 868 869 /* Reduces a cursor by trying to convert everything to after and trying to 870 * go up to block granularity when possible. 871 */ 872 static nir_cursor 873 reduce_cursor(nir_cursor cursor) 874 { 875 switch (cursor.option) { 876 case nir_cursor_before_block: 877 assert(nir_cf_node_prev(&cursor.block->cf_node) == NULL || 878 nir_cf_node_prev(&cursor.block->cf_node)->type != nir_cf_node_block); 879 if (exec_list_is_empty(&cursor.block->instr_list)) { 880 /* Empty block. After is as good as before. */ 881 cursor.option = nir_cursor_after_block; 882 } 883 return cursor; 884 885 case nir_cursor_after_block: 886 return cursor; 887 888 case nir_cursor_before_instr: { 889 nir_instr *prev_instr = nir_instr_prev(cursor.instr); 890 if (prev_instr) { 891 /* Before this instruction is after the previous */ 892 cursor.instr = prev_instr; 893 cursor.option = nir_cursor_after_instr; 894 } else { 895 /* No previous instruction. Switch to before block */ 896 cursor.block = cursor.instr->block; 897 cursor.option = nir_cursor_before_block; 898 } 899 return reduce_cursor(cursor); 900 } 901 902 case nir_cursor_after_instr: 903 if (nir_instr_next(cursor.instr) == NULL) { 904 /* This is the last instruction, switch to after block */ 905 cursor.option = nir_cursor_after_block; 906 cursor.block = cursor.instr->block; 907 } 908 return cursor; 909 910 default: 911 unreachable("Inavlid cursor option"); 912 } 913 } 914 915 bool 916 nir_cursors_equal(nir_cursor a, nir_cursor b) 917 { 918 /* Reduced cursors should be unique */ 919 a = reduce_cursor(a); 920 b = reduce_cursor(b); 921 922 return a.block == b.block && a.option == b.option; 923 } 924 925 static bool 926 add_use_cb(nir_src *src, void *state) 927 { 928 nir_instr *instr = state; 929 930 src->parent_instr = instr; 931 list_addtail(&src->use_link, 932 src->is_ssa ? &src->ssa->uses : &src->reg.reg->uses); 933 934 return true; 935 } 936 937 static bool 938 add_ssa_def_cb(nir_ssa_def *def, void *state) 939 { 940 nir_instr *instr = state; 941 942 if (instr->block && def->index == UINT_MAX) { 943 nir_function_impl *impl = 944 nir_cf_node_get_function(&instr->block->cf_node); 945 946 def->index = impl->ssa_alloc++; 947 } 948 949 return true; 950 } 951 952 static bool 953 add_reg_def_cb(nir_dest *dest, void *state) 954 { 955 nir_instr *instr = state; 956 957 if (!dest->is_ssa) { 958 dest->reg.parent_instr = instr; 959 list_addtail(&dest->reg.def_link, &dest->reg.reg->defs); 960 } 961 962 return true; 963 } 964 965 static void 966 add_defs_uses(nir_instr *instr) 967 { 968 nir_foreach_src(instr, add_use_cb, instr); 969 nir_foreach_dest(instr, add_reg_def_cb, instr); 970 nir_foreach_ssa_def(instr, add_ssa_def_cb, instr); 971 } 972 973 void 974 nir_instr_insert(nir_cursor cursor, nir_instr *instr) 975 { 976 switch (cursor.option) { 977 case nir_cursor_before_block: 978 /* Only allow inserting jumps into empty blocks. */ 979 if (instr->type == nir_instr_type_jump) 980 assert(exec_list_is_empty(&cursor.block->instr_list)); 981 982 instr->block = cursor.block; 983 add_defs_uses(instr); 984 exec_list_push_head(&cursor.block->instr_list, &instr->node); 985 break; 986 case nir_cursor_after_block: { 987 /* Inserting instructions after a jump is illegal. */ 988 nir_instr *last = nir_block_last_instr(cursor.block); 989 assert(last == NULL || last->type != nir_instr_type_jump); 990 (void) last; 991 992 instr->block = cursor.block; 993 add_defs_uses(instr); 994 exec_list_push_tail(&cursor.block->instr_list, &instr->node); 995 break; 996 } 997 case nir_cursor_before_instr: 998 assert(instr->type != nir_instr_type_jump); 999 instr->block = cursor.instr->block; 1000 add_defs_uses(instr); 1001 exec_node_insert_node_before(&cursor.instr->node, &instr->node); 1002 break; 1003 case nir_cursor_after_instr: 1004 /* Inserting instructions after a jump is illegal. */ 1005 assert(cursor.instr->type != nir_instr_type_jump); 1006 1007 /* Only allow inserting jumps at the end of the block. */ 1008 if (instr->type == nir_instr_type_jump) 1009 assert(cursor.instr == nir_block_last_instr(cursor.instr->block)); 1010 1011 instr->block = cursor.instr->block; 1012 add_defs_uses(instr); 1013 exec_node_insert_after(&cursor.instr->node, &instr->node); 1014 break; 1015 } 1016 1017 if (instr->type == nir_instr_type_jump) 1018 nir_handle_add_jump(instr->block); 1019 } 1020 1021 static bool 1022 src_is_valid(const nir_src *src) 1023 { 1024 return src->is_ssa ? (src->ssa != NULL) : (src->reg.reg != NULL); 1025 } 1026 1027 static bool 1028 remove_use_cb(nir_src *src, void *state) 1029 { 1030 (void) state; 1031 1032 if (src_is_valid(src)) 1033 list_del(&src->use_link); 1034 1035 return true; 1036 } 1037 1038 static bool 1039 remove_def_cb(nir_dest *dest, void *state) 1040 { 1041 (void) state; 1042 1043 if (!dest->is_ssa) 1044 list_del(&dest->reg.def_link); 1045 1046 return true; 1047 } 1048 1049 static void 1050 remove_defs_uses(nir_instr *instr) 1051 { 1052 nir_foreach_dest(instr, remove_def_cb, instr); 1053 nir_foreach_src(instr, remove_use_cb, instr); 1054 } 1055 1056 void nir_instr_remove(nir_instr *instr) 1057 { 1058 remove_defs_uses(instr); 1059 exec_node_remove(&instr->node); 1060 1061 if (instr->type == nir_instr_type_jump) { 1062 nir_jump_instr *jump_instr = nir_instr_as_jump(instr); 1063 nir_handle_remove_jump(instr->block, jump_instr->type); 1064 } 1065 } 1066 1067 /*@}*/ 1068 1069 void 1070 nir_index_local_regs(nir_function_impl *impl) 1071 { 1072 unsigned index = 0; 1073 foreach_list_typed(nir_register, reg, node, &impl->registers) { 1074 reg->index = index++; 1075 } 1076 impl->reg_alloc = index; 1077 } 1078 1079 void 1080 nir_index_global_regs(nir_shader *shader) 1081 { 1082 unsigned index = 0; 1083 foreach_list_typed(nir_register, reg, node, &shader->registers) { 1084 reg->index = index++; 1085 } 1086 shader->reg_alloc = index; 1087 } 1088 1089 static bool 1090 visit_alu_dest(nir_alu_instr *instr, nir_foreach_dest_cb cb, void *state) 1091 { 1092 return cb(&instr->dest.dest, state); 1093 } 1094 1095 static bool 1096 visit_intrinsic_dest(nir_intrinsic_instr *instr, nir_foreach_dest_cb cb, 1097 void *state) 1098 { 1099 if (nir_intrinsic_infos[instr->intrinsic].has_dest) 1100 return cb(&instr->dest, state); 1101 1102 return true; 1103 } 1104 1105 static bool 1106 visit_texture_dest(nir_tex_instr *instr, nir_foreach_dest_cb cb, 1107 void *state) 1108 { 1109 return cb(&instr->dest, state); 1110 } 1111 1112 static bool 1113 visit_phi_dest(nir_phi_instr *instr, nir_foreach_dest_cb cb, void *state) 1114 { 1115 return cb(&instr->dest, state); 1116 } 1117 1118 static bool 1119 visit_parallel_copy_dest(nir_parallel_copy_instr *instr, 1120 nir_foreach_dest_cb cb, void *state) 1121 { 1122 nir_foreach_parallel_copy_entry(entry, instr) { 1123 if (!cb(&entry->dest, state)) 1124 return false; 1125 } 1126 1127 return true; 1128 } 1129 1130 bool 1131 nir_foreach_dest(nir_instr *instr, nir_foreach_dest_cb cb, void *state) 1132 { 1133 switch (instr->type) { 1134 case nir_instr_type_alu: 1135 return visit_alu_dest(nir_instr_as_alu(instr), cb, state); 1136 case nir_instr_type_intrinsic: 1137 return visit_intrinsic_dest(nir_instr_as_intrinsic(instr), cb, state); 1138 case nir_instr_type_tex: 1139 return visit_texture_dest(nir_instr_as_tex(instr), cb, state); 1140 case nir_instr_type_phi: 1141 return visit_phi_dest(nir_instr_as_phi(instr), cb, state); 1142 case nir_instr_type_parallel_copy: 1143 return visit_parallel_copy_dest(nir_instr_as_parallel_copy(instr), 1144 cb, state); 1145 1146 case nir_instr_type_load_const: 1147 case nir_instr_type_ssa_undef: 1148 case nir_instr_type_call: 1149 case nir_instr_type_jump: 1150 break; 1151 1152 default: 1153 unreachable("Invalid instruction type"); 1154 break; 1155 } 1156 1157 return true; 1158 } 1159 1160 struct foreach_ssa_def_state { 1161 nir_foreach_ssa_def_cb cb; 1162 void *client_state; 1163 }; 1164 1165 static inline bool 1166 nir_ssa_def_visitor(nir_dest *dest, void *void_state) 1167 { 1168 struct foreach_ssa_def_state *state = void_state; 1169 1170 if (dest->is_ssa) 1171 return state->cb(&dest->ssa, state->client_state); 1172 else 1173 return true; 1174 } 1175 1176 bool 1177 nir_foreach_ssa_def(nir_instr *instr, nir_foreach_ssa_def_cb cb, void *state) 1178 { 1179 switch (instr->type) { 1180 case nir_instr_type_alu: 1181 case nir_instr_type_tex: 1182 case nir_instr_type_intrinsic: 1183 case nir_instr_type_phi: 1184 case nir_instr_type_parallel_copy: { 1185 struct foreach_ssa_def_state foreach_state = {cb, state}; 1186 return nir_foreach_dest(instr, nir_ssa_def_visitor, &foreach_state); 1187 } 1188 1189 case nir_instr_type_load_const: 1190 return cb(&nir_instr_as_load_const(instr)->def, state); 1191 case nir_instr_type_ssa_undef: 1192 return cb(&nir_instr_as_ssa_undef(instr)->def, state); 1193 case nir_instr_type_call: 1194 case nir_instr_type_jump: 1195 return true; 1196 default: 1197 unreachable("Invalid instruction type"); 1198 } 1199 } 1200 1201 static bool 1202 visit_src(nir_src *src, nir_foreach_src_cb cb, void *state) 1203 { 1204 if (!cb(src, state)) 1205 return false; 1206 if (!src->is_ssa && src->reg.indirect) 1207 return cb(src->reg.indirect, state); 1208 return true; 1209 } 1210 1211 static bool 1212 visit_deref_array_src(nir_deref_array *deref, nir_foreach_src_cb cb, 1213 void *state) 1214 { 1215 if (deref->deref_array_type == nir_deref_array_type_indirect) 1216 return visit_src(&deref->indirect, cb, state); 1217 return true; 1218 } 1219 1220 static bool 1221 visit_deref_src(nir_deref_var *deref, nir_foreach_src_cb cb, void *state) 1222 { 1223 nir_deref *cur = &deref->deref; 1224 while (cur != NULL) { 1225 if (cur->deref_type == nir_deref_type_array) { 1226 if (!visit_deref_array_src(nir_deref_as_array(cur), cb, state)) 1227 return false; 1228 } 1229 1230 cur = cur->child; 1231 } 1232 1233 return true; 1234 } 1235 1236 static bool 1237 visit_alu_src(nir_alu_instr *instr, nir_foreach_src_cb cb, void *state) 1238 { 1239 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) 1240 if (!visit_src(&instr->src[i].src, cb, state)) 1241 return false; 1242 1243 return true; 1244 } 1245 1246 static bool 1247 visit_tex_src(nir_tex_instr *instr, nir_foreach_src_cb cb, void *state) 1248 { 1249 for (unsigned i = 0; i < instr->num_srcs; i++) { 1250 if (!visit_src(&instr->src[i].src, cb, state)) 1251 return false; 1252 } 1253 1254 if (instr->texture != NULL) { 1255 if (!visit_deref_src(instr->texture, cb, state)) 1256 return false; 1257 } 1258 1259 if (instr->sampler != NULL) { 1260 if (!visit_deref_src(instr->sampler, cb, state)) 1261 return false; 1262 } 1263 1264 return true; 1265 } 1266 1267 static bool 1268 visit_intrinsic_src(nir_intrinsic_instr *instr, nir_foreach_src_cb cb, 1269 void *state) 1270 { 1271 unsigned num_srcs = nir_intrinsic_infos[instr->intrinsic].num_srcs; 1272 for (unsigned i = 0; i < num_srcs; i++) { 1273 if (!visit_src(&instr->src[i], cb, state)) 1274 return false; 1275 } 1276 1277 unsigned num_vars = 1278 nir_intrinsic_infos[instr->intrinsic].num_variables; 1279 for (unsigned i = 0; i < num_vars; i++) { 1280 if (!visit_deref_src(instr->variables[i], cb, state)) 1281 return false; 1282 } 1283 1284 return true; 1285 } 1286 1287 static bool 1288 visit_phi_src(nir_phi_instr *instr, nir_foreach_src_cb cb, void *state) 1289 { 1290 nir_foreach_phi_src(src, instr) { 1291 if (!visit_src(&src->src, cb, state)) 1292 return false; 1293 } 1294 1295 return true; 1296 } 1297 1298 static bool 1299 visit_parallel_copy_src(nir_parallel_copy_instr *instr, 1300 nir_foreach_src_cb cb, void *state) 1301 { 1302 nir_foreach_parallel_copy_entry(entry, instr) { 1303 if (!visit_src(&entry->src, cb, state)) 1304 return false; 1305 } 1306 1307 return true; 1308 } 1309 1310 typedef struct { 1311 void *state; 1312 nir_foreach_src_cb cb; 1313 } visit_dest_indirect_state; 1314 1315 static bool 1316 visit_dest_indirect(nir_dest *dest, void *_state) 1317 { 1318 visit_dest_indirect_state *state = (visit_dest_indirect_state *) _state; 1319 1320 if (!dest->is_ssa && dest->reg.indirect) 1321 return state->cb(dest->reg.indirect, state->state); 1322 1323 return true; 1324 } 1325 1326 bool 1327 nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state) 1328 { 1329 switch (instr->type) { 1330 case nir_instr_type_alu: 1331 if (!visit_alu_src(nir_instr_as_alu(instr), cb, state)) 1332 return false; 1333 break; 1334 case nir_instr_type_intrinsic: 1335 if (!visit_intrinsic_src(nir_instr_as_intrinsic(instr), cb, state)) 1336 return false; 1337 break; 1338 case nir_instr_type_tex: 1339 if (!visit_tex_src(nir_instr_as_tex(instr), cb, state)) 1340 return false; 1341 break; 1342 case nir_instr_type_call: 1343 /* Call instructions have no regular sources */ 1344 break; 1345 case nir_instr_type_load_const: 1346 /* Constant load instructions have no regular sources */ 1347 break; 1348 case nir_instr_type_phi: 1349 if (!visit_phi_src(nir_instr_as_phi(instr), cb, state)) 1350 return false; 1351 break; 1352 case nir_instr_type_parallel_copy: 1353 if (!visit_parallel_copy_src(nir_instr_as_parallel_copy(instr), 1354 cb, state)) 1355 return false; 1356 break; 1357 case nir_instr_type_jump: 1358 case nir_instr_type_ssa_undef: 1359 return true; 1360 1361 default: 1362 unreachable("Invalid instruction type"); 1363 break; 1364 } 1365 1366 visit_dest_indirect_state dest_state; 1367 dest_state.state = state; 1368 dest_state.cb = cb; 1369 return nir_foreach_dest(instr, visit_dest_indirect, &dest_state); 1370 } 1371 1372 nir_const_value * 1373 nir_src_as_const_value(nir_src src) 1374 { 1375 if (!src.is_ssa) 1376 return NULL; 1377 1378 if (src.ssa->parent_instr->type != nir_instr_type_load_const) 1379 return NULL; 1380 1381 nir_load_const_instr *load = nir_instr_as_load_const(src.ssa->parent_instr); 1382 1383 return &load->value; 1384 } 1385 1386 /** 1387 * Returns true if the source is known to be dynamically uniform. Otherwise it 1388 * returns false which means it may or may not be dynamically uniform but it 1389 * can't be determined. 1390 */ 1391 bool 1392 nir_src_is_dynamically_uniform(nir_src src) 1393 { 1394 if (!src.is_ssa) 1395 return false; 1396 1397 /* Constants are trivially dynamically uniform */ 1398 if (src.ssa->parent_instr->type == nir_instr_type_load_const) 1399 return true; 1400 1401 /* As are uniform variables */ 1402 if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) { 1403 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr); 1404 1405 if (intr->intrinsic == nir_intrinsic_load_uniform) 1406 return true; 1407 } 1408 1409 /* XXX: this could have many more tests, such as when a sampler function is 1410 * called with dynamically uniform arguments. 1411 */ 1412 return false; 1413 } 1414 1415 static void 1416 src_remove_all_uses(nir_src *src) 1417 { 1418 for (; src; src = src->is_ssa ? NULL : src->reg.indirect) { 1419 if (!src_is_valid(src)) 1420 continue; 1421 1422 list_del(&src->use_link); 1423 } 1424 } 1425 1426 static void 1427 src_add_all_uses(nir_src *src, nir_instr *parent_instr, nir_if *parent_if) 1428 { 1429 for (; src; src = src->is_ssa ? NULL : src->reg.indirect) { 1430 if (!src_is_valid(src)) 1431 continue; 1432 1433 if (parent_instr) { 1434 src->parent_instr = parent_instr; 1435 if (src->is_ssa) 1436 list_addtail(&src->use_link, &src->ssa->uses); 1437 else 1438 list_addtail(&src->use_link, &src->reg.reg->uses); 1439 } else { 1440 assert(parent_if); 1441 src->parent_if = parent_if; 1442 if (src->is_ssa) 1443 list_addtail(&src->use_link, &src->ssa->if_uses); 1444 else 1445 list_addtail(&src->use_link, &src->reg.reg->if_uses); 1446 } 1447 } 1448 } 1449 1450 void 1451 nir_instr_rewrite_src(nir_instr *instr, nir_src *src, nir_src new_src) 1452 { 1453 assert(!src_is_valid(src) || src->parent_instr == instr); 1454 1455 src_remove_all_uses(src); 1456 *src = new_src; 1457 src_add_all_uses(src, instr, NULL); 1458 } 1459 1460 void 1461 nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src) 1462 { 1463 assert(!src_is_valid(dest) || dest->parent_instr == dest_instr); 1464 1465 src_remove_all_uses(dest); 1466 src_remove_all_uses(src); 1467 *dest = *src; 1468 *src = NIR_SRC_INIT; 1469 src_add_all_uses(dest, dest_instr, NULL); 1470 } 1471 1472 void 1473 nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src) 1474 { 1475 nir_src *src = &if_stmt->condition; 1476 assert(!src_is_valid(src) || src->parent_if == if_stmt); 1477 1478 src_remove_all_uses(src); 1479 *src = new_src; 1480 src_add_all_uses(src, NULL, if_stmt); 1481 } 1482 1483 void 1484 nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest) 1485 { 1486 if (dest->is_ssa) { 1487 /* We can only overwrite an SSA destination if it has no uses. */ 1488 assert(list_empty(&dest->ssa.uses) && list_empty(&dest->ssa.if_uses)); 1489 } else { 1490 list_del(&dest->reg.def_link); 1491 if (dest->reg.indirect) 1492 src_remove_all_uses(dest->reg.indirect); 1493 } 1494 1495 /* We can't re-write with an SSA def */ 1496 assert(!new_dest.is_ssa); 1497 1498 nir_dest_copy(dest, &new_dest, instr); 1499 1500 dest->reg.parent_instr = instr; 1501 list_addtail(&dest->reg.def_link, &new_dest.reg.reg->defs); 1502 1503 if (dest->reg.indirect) 1504 src_add_all_uses(dest->reg.indirect, instr, NULL); 1505 } 1506 1507 /* note: does *not* take ownership of 'name' */ 1508 void 1509 nir_ssa_def_init(nir_instr *instr, nir_ssa_def *def, 1510 unsigned num_components, 1511 unsigned bit_size, const char *name) 1512 { 1513 def->name = ralloc_strdup(instr, name); 1514 def->parent_instr = instr; 1515 list_inithead(&def->uses); 1516 list_inithead(&def->if_uses); 1517 def->num_components = num_components; 1518 def->bit_size = bit_size; 1519 1520 if (instr->block) { 1521 nir_function_impl *impl = 1522 nir_cf_node_get_function(&instr->block->cf_node); 1523 1524 def->index = impl->ssa_alloc++; 1525 } else { 1526 def->index = UINT_MAX; 1527 } 1528 } 1529 1530 /* note: does *not* take ownership of 'name' */ 1531 void 1532 nir_ssa_dest_init(nir_instr *instr, nir_dest *dest, 1533 unsigned num_components, unsigned bit_size, 1534 const char *name) 1535 { 1536 dest->is_ssa = true; 1537 nir_ssa_def_init(instr, &dest->ssa, num_components, bit_size, name); 1538 } 1539 1540 void 1541 nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src) 1542 { 1543 assert(!new_src.is_ssa || def != new_src.ssa); 1544 1545 nir_foreach_use_safe(use_src, def) 1546 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src); 1547 1548 nir_foreach_if_use_safe(use_src, def) 1549 nir_if_rewrite_condition(use_src->parent_if, new_src); 1550 } 1551 1552 static bool 1553 is_instr_between(nir_instr *start, nir_instr *end, nir_instr *between) 1554 { 1555 assert(start->block == end->block); 1556 1557 if (between->block != start->block) 1558 return false; 1559 1560 /* Search backwards looking for "between" */ 1561 while (start != end) { 1562 if (between == end) 1563 return true; 1564 1565 end = nir_instr_prev(end); 1566 assert(end); 1567 } 1568 1569 return false; 1570 } 1571 1572 /* Replaces all uses of the given SSA def with the given source but only if 1573 * the use comes after the after_me instruction. This can be useful if you 1574 * are emitting code to fix up the result of some instruction: you can freely 1575 * use the result in that code and then call rewrite_uses_after and pass the 1576 * last fixup instruction as after_me and it will replace all of the uses you 1577 * want without touching the fixup code. 1578 * 1579 * This function assumes that after_me is in the same block as 1580 * def->parent_instr and that after_me comes after def->parent_instr. 1581 */ 1582 void 1583 nir_ssa_def_rewrite_uses_after(nir_ssa_def *def, nir_src new_src, 1584 nir_instr *after_me) 1585 { 1586 assert(!new_src.is_ssa || def != new_src.ssa); 1587 1588 nir_foreach_use_safe(use_src, def) { 1589 assert(use_src->parent_instr != def->parent_instr); 1590 /* Since def already dominates all of its uses, the only way a use can 1591 * not be dominated by after_me is if it is between def and after_me in 1592 * the instruction list. 1593 */ 1594 if (!is_instr_between(def->parent_instr, after_me, use_src->parent_instr)) 1595 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src); 1596 } 1597 1598 nir_foreach_if_use_safe(use_src, def) 1599 nir_if_rewrite_condition(use_src->parent_if, new_src); 1600 } 1601 1602 uint8_t 1603 nir_ssa_def_components_read(nir_ssa_def *def) 1604 { 1605 uint8_t read_mask = 0; 1606 nir_foreach_use(use, def) { 1607 if (use->parent_instr->type == nir_instr_type_alu) { 1608 nir_alu_instr *alu = nir_instr_as_alu(use->parent_instr); 1609 nir_alu_src *alu_src = exec_node_data(nir_alu_src, use, src); 1610 int src_idx = alu_src - &alu->src[0]; 1611 assert(src_idx >= 0 && src_idx < nir_op_infos[alu->op].num_inputs); 1612 1613 for (unsigned c = 0; c < 4; c++) { 1614 if (!nir_alu_instr_channel_used(alu, src_idx, c)) 1615 continue; 1616 1617 read_mask |= (1 << alu_src->swizzle[c]); 1618 } 1619 } else { 1620 return (1 << def->num_components) - 1; 1621 } 1622 } 1623 1624 return read_mask; 1625 } 1626 1627 nir_block * 1628 nir_block_cf_tree_next(nir_block *block) 1629 { 1630 if (block == NULL) { 1631 /* nir_foreach_block_safe() will call this function on a NULL block 1632 * after the last iteration, but it won't use the result so just return 1633 * NULL here. 1634 */ 1635 return NULL; 1636 } 1637 1638 nir_cf_node *cf_next = nir_cf_node_next(&block->cf_node); 1639 if (cf_next) 1640 return nir_cf_node_cf_tree_first(cf_next); 1641 1642 nir_cf_node *parent = block->cf_node.parent; 1643 1644 switch (parent->type) { 1645 case nir_cf_node_if: { 1646 /* Are we at the end of the if? Go to the beginning of the else */ 1647 nir_if *if_stmt = nir_cf_node_as_if(parent); 1648 if (block == nir_if_last_then_block(if_stmt)) 1649 return nir_if_first_else_block(if_stmt); 1650 1651 assert(block == nir_if_last_else_block(if_stmt)); 1652 /* fall through */ 1653 } 1654 1655 case nir_cf_node_loop: 1656 return nir_cf_node_as_block(nir_cf_node_next(parent)); 1657 1658 case nir_cf_node_function: 1659 return NULL; 1660 1661 default: 1662 unreachable("unknown cf node type"); 1663 } 1664 } 1665 1666 nir_block * 1667 nir_block_cf_tree_prev(nir_block *block) 1668 { 1669 if (block == NULL) { 1670 /* do this for consistency with nir_block_cf_tree_next() */ 1671 return NULL; 1672 } 1673 1674 nir_cf_node *cf_prev = nir_cf_node_prev(&block->cf_node); 1675 if (cf_prev) 1676 return nir_cf_node_cf_tree_last(cf_prev); 1677 1678 nir_cf_node *parent = block->cf_node.parent; 1679 1680 switch (parent->type) { 1681 case nir_cf_node_if: { 1682 /* Are we at the beginning of the else? Go to the end of the if */ 1683 nir_if *if_stmt = nir_cf_node_as_if(parent); 1684 if (block == nir_if_first_else_block(if_stmt)) 1685 return nir_if_last_then_block(if_stmt); 1686 1687 assert(block == nir_if_first_then_block(if_stmt)); 1688 /* fall through */ 1689 } 1690 1691 case nir_cf_node_loop: 1692 return nir_cf_node_as_block(nir_cf_node_prev(parent)); 1693 1694 case nir_cf_node_function: 1695 return NULL; 1696 1697 default: 1698 unreachable("unknown cf node type"); 1699 } 1700 } 1701 1702 nir_block *nir_cf_node_cf_tree_first(nir_cf_node *node) 1703 { 1704 switch (node->type) { 1705 case nir_cf_node_function: { 1706 nir_function_impl *impl = nir_cf_node_as_function(node); 1707 return nir_start_block(impl); 1708 } 1709 1710 case nir_cf_node_if: { 1711 nir_if *if_stmt = nir_cf_node_as_if(node); 1712 return nir_if_first_then_block(if_stmt); 1713 } 1714 1715 case nir_cf_node_loop: { 1716 nir_loop *loop = nir_cf_node_as_loop(node); 1717 return nir_loop_first_block(loop); 1718 } 1719 1720 case nir_cf_node_block: { 1721 return nir_cf_node_as_block(node); 1722 } 1723 1724 default: 1725 unreachable("unknown node type"); 1726 } 1727 } 1728 1729 nir_block *nir_cf_node_cf_tree_last(nir_cf_node *node) 1730 { 1731 switch (node->type) { 1732 case nir_cf_node_function: { 1733 nir_function_impl *impl = nir_cf_node_as_function(node); 1734 return nir_impl_last_block(impl); 1735 } 1736 1737 case nir_cf_node_if: { 1738 nir_if *if_stmt = nir_cf_node_as_if(node); 1739 return nir_if_last_else_block(if_stmt); 1740 } 1741 1742 case nir_cf_node_loop: { 1743 nir_loop *loop = nir_cf_node_as_loop(node); 1744 return nir_loop_last_block(loop); 1745 } 1746 1747 case nir_cf_node_block: { 1748 return nir_cf_node_as_block(node); 1749 } 1750 1751 default: 1752 unreachable("unknown node type"); 1753 } 1754 } 1755 1756 nir_block *nir_cf_node_cf_tree_next(nir_cf_node *node) 1757 { 1758 if (node->type == nir_cf_node_block) 1759 return nir_block_cf_tree_next(nir_cf_node_as_block(node)); 1760 else if (node->type == nir_cf_node_function) 1761 return NULL; 1762 else 1763 return nir_cf_node_as_block(nir_cf_node_next(node)); 1764 } 1765 1766 nir_if * 1767 nir_block_get_following_if(nir_block *block) 1768 { 1769 if (exec_node_is_tail_sentinel(&block->cf_node.node)) 1770 return NULL; 1771 1772 if (nir_cf_node_is_last(&block->cf_node)) 1773 return NULL; 1774 1775 nir_cf_node *next_node = nir_cf_node_next(&block->cf_node); 1776 1777 if (next_node->type != nir_cf_node_if) 1778 return NULL; 1779 1780 return nir_cf_node_as_if(next_node); 1781 } 1782 1783 nir_loop * 1784 nir_block_get_following_loop(nir_block *block) 1785 { 1786 if (exec_node_is_tail_sentinel(&block->cf_node.node)) 1787 return NULL; 1788 1789 if (nir_cf_node_is_last(&block->cf_node)) 1790 return NULL; 1791 1792 nir_cf_node *next_node = nir_cf_node_next(&block->cf_node); 1793 1794 if (next_node->type != nir_cf_node_loop) 1795 return NULL; 1796 1797 return nir_cf_node_as_loop(next_node); 1798 } 1799 1800 void 1801 nir_index_blocks(nir_function_impl *impl) 1802 { 1803 unsigned index = 0; 1804 1805 if (impl->valid_metadata & nir_metadata_block_index) 1806 return; 1807 1808 nir_foreach_block(block, impl) { 1809 block->index = index++; 1810 } 1811 1812 impl->num_blocks = index; 1813 } 1814 1815 static bool 1816 index_ssa_def_cb(nir_ssa_def *def, void *state) 1817 { 1818 unsigned *index = (unsigned *) state; 1819 def->index = (*index)++; 1820 1821 return true; 1822 } 1823 1824 /** 1825 * The indices are applied top-to-bottom which has the very nice property 1826 * that, if A dominates B, then A->index <= B->index. 1827 */ 1828 void 1829 nir_index_ssa_defs(nir_function_impl *impl) 1830 { 1831 unsigned index = 0; 1832 1833 nir_foreach_block(block, impl) { 1834 nir_foreach_instr(instr, block) 1835 nir_foreach_ssa_def(instr, index_ssa_def_cb, &index); 1836 } 1837 1838 impl->ssa_alloc = index; 1839 } 1840 1841 /** 1842 * The indices are applied top-to-bottom which has the very nice property 1843 * that, if A dominates B, then A->index <= B->index. 1844 */ 1845 unsigned 1846 nir_index_instrs(nir_function_impl *impl) 1847 { 1848 unsigned index = 0; 1849 1850 nir_foreach_block(block, impl) { 1851 nir_foreach_instr(instr, block) 1852 instr->index = index++; 1853 } 1854 1855 return index; 1856 } 1857 1858 nir_intrinsic_op 1859 nir_intrinsic_from_system_value(gl_system_value val) 1860 { 1861 switch (val) { 1862 case SYSTEM_VALUE_VERTEX_ID: 1863 return nir_intrinsic_load_vertex_id; 1864 case SYSTEM_VALUE_INSTANCE_ID: 1865 return nir_intrinsic_load_instance_id; 1866 case SYSTEM_VALUE_DRAW_ID: 1867 return nir_intrinsic_load_draw_id; 1868 case SYSTEM_VALUE_BASE_INSTANCE: 1869 return nir_intrinsic_load_base_instance; 1870 case SYSTEM_VALUE_VERTEX_ID_ZERO_BASE: 1871 return nir_intrinsic_load_vertex_id_zero_base; 1872 case SYSTEM_VALUE_BASE_VERTEX: 1873 return nir_intrinsic_load_base_vertex; 1874 case SYSTEM_VALUE_INVOCATION_ID: 1875 return nir_intrinsic_load_invocation_id; 1876 case SYSTEM_VALUE_FRONT_FACE: 1877 return nir_intrinsic_load_front_face; 1878 case SYSTEM_VALUE_SAMPLE_ID: 1879 return nir_intrinsic_load_sample_id; 1880 case SYSTEM_VALUE_SAMPLE_POS: 1881 return nir_intrinsic_load_sample_pos; 1882 case SYSTEM_VALUE_SAMPLE_MASK_IN: 1883 return nir_intrinsic_load_sample_mask_in; 1884 case SYSTEM_VALUE_LOCAL_INVOCATION_ID: 1885 return nir_intrinsic_load_local_invocation_id; 1886 case SYSTEM_VALUE_LOCAL_INVOCATION_INDEX: 1887 return nir_intrinsic_load_local_invocation_index; 1888 case SYSTEM_VALUE_WORK_GROUP_ID: 1889 return nir_intrinsic_load_work_group_id; 1890 case SYSTEM_VALUE_NUM_WORK_GROUPS: 1891 return nir_intrinsic_load_num_work_groups; 1892 case SYSTEM_VALUE_PRIMITIVE_ID: 1893 return nir_intrinsic_load_primitive_id; 1894 case SYSTEM_VALUE_TESS_COORD: 1895 return nir_intrinsic_load_tess_coord; 1896 case SYSTEM_VALUE_TESS_LEVEL_OUTER: 1897 return nir_intrinsic_load_tess_level_outer; 1898 case SYSTEM_VALUE_TESS_LEVEL_INNER: 1899 return nir_intrinsic_load_tess_level_inner; 1900 case SYSTEM_VALUE_VERTICES_IN: 1901 return nir_intrinsic_load_patch_vertices_in; 1902 case SYSTEM_VALUE_HELPER_INVOCATION: 1903 return nir_intrinsic_load_helper_invocation; 1904 default: 1905 unreachable("system value does not directly correspond to intrinsic"); 1906 } 1907 } 1908 1909 gl_system_value 1910 nir_system_value_from_intrinsic(nir_intrinsic_op intrin) 1911 { 1912 switch (intrin) { 1913 case nir_intrinsic_load_vertex_id: 1914 return SYSTEM_VALUE_VERTEX_ID; 1915 case nir_intrinsic_load_instance_id: 1916 return SYSTEM_VALUE_INSTANCE_ID; 1917 case nir_intrinsic_load_draw_id: 1918 return SYSTEM_VALUE_DRAW_ID; 1919 case nir_intrinsic_load_base_instance: 1920 return SYSTEM_VALUE_BASE_INSTANCE; 1921 case nir_intrinsic_load_vertex_id_zero_base: 1922 return SYSTEM_VALUE_VERTEX_ID_ZERO_BASE; 1923 case nir_intrinsic_load_base_vertex: 1924 return SYSTEM_VALUE_BASE_VERTEX; 1925 case nir_intrinsic_load_invocation_id: 1926 return SYSTEM_VALUE_INVOCATION_ID; 1927 case nir_intrinsic_load_front_face: 1928 return SYSTEM_VALUE_FRONT_FACE; 1929 case nir_intrinsic_load_sample_id: 1930 return SYSTEM_VALUE_SAMPLE_ID; 1931 case nir_intrinsic_load_sample_pos: 1932 return SYSTEM_VALUE_SAMPLE_POS; 1933 case nir_intrinsic_load_sample_mask_in: 1934 return SYSTEM_VALUE_SAMPLE_MASK_IN; 1935 case nir_intrinsic_load_local_invocation_id: 1936 return SYSTEM_VALUE_LOCAL_INVOCATION_ID; 1937 case nir_intrinsic_load_local_invocation_index: 1938 return SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 1939 case nir_intrinsic_load_num_work_groups: 1940 return SYSTEM_VALUE_NUM_WORK_GROUPS; 1941 case nir_intrinsic_load_work_group_id: 1942 return SYSTEM_VALUE_WORK_GROUP_ID; 1943 case nir_intrinsic_load_primitive_id: 1944 return SYSTEM_VALUE_PRIMITIVE_ID; 1945 case nir_intrinsic_load_tess_coord: 1946 return SYSTEM_VALUE_TESS_COORD; 1947 case nir_intrinsic_load_tess_level_outer: 1948 return SYSTEM_VALUE_TESS_LEVEL_OUTER; 1949 case nir_intrinsic_load_tess_level_inner: 1950 return SYSTEM_VALUE_TESS_LEVEL_INNER; 1951 case nir_intrinsic_load_patch_vertices_in: 1952 return SYSTEM_VALUE_VERTICES_IN; 1953 case nir_intrinsic_load_helper_invocation: 1954 return SYSTEM_VALUE_HELPER_INVOCATION; 1955 default: 1956 unreachable("intrinsic doesn't produce a system value"); 1957 } 1958 } 1959 1960 nir_op 1961 nir_type_conversion_op(nir_alu_type src, nir_alu_type dst) 1962 { 1963 nir_alu_type src_base_type = (nir_alu_type) nir_alu_type_get_base_type(src); 1964 nir_alu_type dst_base_type = (nir_alu_type) nir_alu_type_get_base_type(dst); 1965 unsigned src_bitsize = nir_alu_type_get_type_size(src); 1966 unsigned dst_bitsize = nir_alu_type_get_type_size(dst); 1967 1968 if (src_base_type == dst_base_type) { 1969 if (src_bitsize == dst_bitsize) 1970 return (src_base_type == nir_type_float) ? nir_op_fmov : nir_op_imov; 1971 1972 assert (src_base_type == nir_type_float); 1973 /* TODO: implement support for float16 */ 1974 assert(src_bitsize == 64 || dst_bitsize == 64); 1975 return (src_bitsize == 64) ? nir_op_d2f : nir_op_f2d; 1976 } 1977 1978 /* Different base type but same bit_size */ 1979 if (src_bitsize == dst_bitsize) { 1980 /* TODO: This does not include specific conversions between 1981 * signed or unsigned integer types of bit size different than 32 yet. 1982 */ 1983 assert(src_bitsize == 32); 1984 switch (src_base_type) { 1985 case nir_type_uint: 1986 return (dst_base_type == nir_type_float) ? nir_op_u2f : nir_op_imov; 1987 case nir_type_int: 1988 return (dst_base_type == nir_type_float) ? nir_op_i2f : nir_op_imov; 1989 case nir_type_bool: 1990 return (dst_base_type == nir_type_float) ? nir_op_b2f : nir_op_b2i; 1991 case nir_type_float: 1992 switch (dst_base_type) { 1993 case nir_type_uint: 1994 return nir_op_f2u; 1995 case nir_type_bool: 1996 return nir_op_f2b; 1997 default: 1998 return nir_op_f2i; 1999 }; 2000 default: 2001 unreachable("Invalid conversion"); 2002 }; 2003 } 2004 2005 /* Different bit_size and different base type */ 2006 /* TODO: Implement integer support for types with bit_size != 32 */ 2007 switch (src_base_type) { 2008 case nir_type_uint: 2009 assert(dst == nir_type_float64); 2010 return nir_op_u2d; 2011 case nir_type_int: 2012 assert(dst == nir_type_float64); 2013 return nir_op_i2d; 2014 case nir_type_bool: 2015 assert(dst == nir_type_float64); 2016 return nir_op_u2d; 2017 case nir_type_float: 2018 assert(src_bitsize == 32 || src_bitsize == 64); 2019 if (src_bitsize != 64) { 2020 assert(dst == nir_type_float64); 2021 return nir_op_f2d; 2022 } 2023 assert(dst_bitsize == 32); 2024 switch (dst_base_type) { 2025 case nir_type_uint: 2026 return nir_op_d2u; 2027 case nir_type_int: 2028 return nir_op_d2i; 2029 case nir_type_bool: 2030 return nir_op_d2b; 2031 case nir_type_float: 2032 return nir_op_d2f; 2033 default: 2034 unreachable("Invalid conversion"); 2035 }; 2036 default: 2037 unreachable("Invalid conversion"); 2038 }; 2039 } 2040
