1 /* 2 * Copyright 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Jason Ekstrand (jason (at) jlekstrand.net) 25 * 26 */ 27 28 #include "vtn_private.h" 29 #include "spirv_info.h" 30 31 static struct vtn_access_chain * 32 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old, 33 unsigned new_ids) 34 { 35 struct vtn_access_chain *chain; 36 37 unsigned new_len = old->length + new_ids; 38 /* TODO: don't use rzalloc */ 39 chain = rzalloc_size(b, sizeof(*chain) + new_len * sizeof(chain->link[0])); 40 41 chain->var = old->var; 42 chain->length = new_len; 43 44 for (unsigned i = 0; i < old->length; i++) 45 chain->link[i] = old->link[i]; 46 47 return chain; 48 } 49 50 static nir_ssa_def * 51 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link, 52 unsigned stride) 53 { 54 assert(stride > 0); 55 if (link.mode == vtn_access_mode_literal) { 56 return nir_imm_int(&b->nb, link.id * stride); 57 } else if (stride == 1) { 58 return vtn_ssa_value(b, link.id)->def; 59 } else { 60 return nir_imul(&b->nb, vtn_ssa_value(b, link.id)->def, 61 nir_imm_int(&b->nb, stride)); 62 } 63 } 64 65 static struct vtn_type * 66 vtn_access_chain_tail_type(struct vtn_builder *b, 67 struct vtn_access_chain *chain) 68 { 69 struct vtn_type *type = chain->var->type; 70 for (unsigned i = 0; i < chain->length; i++) { 71 if (glsl_type_is_struct(type->type)) { 72 assert(chain->link[i].mode == vtn_access_mode_literal); 73 type = type->members[chain->link[i].id]; 74 } else { 75 type = type->array_element; 76 } 77 } 78 return type; 79 } 80 81 /* Crawls a chain of array derefs and rewrites the types so that the 82 * lengths stay the same but the terminal type is the one given by 83 * tail_type. This is useful for split structures. 84 */ 85 static void 86 rewrite_deref_types(nir_deref *deref, const struct glsl_type *type) 87 { 88 deref->type = type; 89 if (deref->child) { 90 assert(deref->child->deref_type == nir_deref_type_array); 91 assert(glsl_type_is_array(deref->type)); 92 rewrite_deref_types(deref->child, glsl_get_array_element(type)); 93 } 94 } 95 96 nir_deref_var * 97 vtn_access_chain_to_deref(struct vtn_builder *b, struct vtn_access_chain *chain) 98 { 99 nir_deref_var *deref_var; 100 if (chain->var->var) { 101 deref_var = nir_deref_var_create(b, chain->var->var); 102 } else { 103 assert(chain->var->members); 104 /* Create the deref_var manually. It will get filled out later. */ 105 deref_var = rzalloc(b, nir_deref_var); 106 deref_var->deref.deref_type = nir_deref_type_var; 107 } 108 109 struct vtn_type *deref_type = chain->var->type; 110 nir_deref *tail = &deref_var->deref; 111 nir_variable **members = chain->var->members; 112 113 for (unsigned i = 0; i < chain->length; i++) { 114 enum glsl_base_type base_type = glsl_get_base_type(deref_type->type); 115 switch (base_type) { 116 case GLSL_TYPE_UINT: 117 case GLSL_TYPE_INT: 118 case GLSL_TYPE_FLOAT: 119 case GLSL_TYPE_DOUBLE: 120 case GLSL_TYPE_BOOL: 121 case GLSL_TYPE_ARRAY: { 122 deref_type = deref_type->array_element; 123 124 nir_deref_array *deref_arr = nir_deref_array_create(b); 125 deref_arr->deref.type = deref_type->type; 126 127 if (chain->link[i].mode == vtn_access_mode_literal) { 128 deref_arr->deref_array_type = nir_deref_array_type_direct; 129 deref_arr->base_offset = chain->link[i].id; 130 } else { 131 assert(chain->link[i].mode == vtn_access_mode_id); 132 deref_arr->deref_array_type = nir_deref_array_type_indirect; 133 deref_arr->base_offset = 0; 134 deref_arr->indirect = 135 nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def); 136 } 137 tail->child = &deref_arr->deref; 138 tail = tail->child; 139 break; 140 } 141 142 case GLSL_TYPE_STRUCT: { 143 assert(chain->link[i].mode == vtn_access_mode_literal); 144 unsigned idx = chain->link[i].id; 145 deref_type = deref_type->members[idx]; 146 if (members) { 147 /* This is a pre-split structure. */ 148 deref_var->var = members[idx]; 149 rewrite_deref_types(&deref_var->deref, members[idx]->type); 150 assert(tail->type == deref_type->type); 151 members = NULL; 152 } else { 153 nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx); 154 deref_struct->deref.type = deref_type->type; 155 tail->child = &deref_struct->deref; 156 tail = tail->child; 157 } 158 break; 159 } 160 default: 161 unreachable("Invalid type for deref"); 162 } 163 } 164 165 assert(members == NULL); 166 return deref_var; 167 } 168 169 static void 170 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref, 171 nir_deref *tail, struct vtn_ssa_value *inout) 172 { 173 /* The deref tail may contain a deref to select a component of a vector (in 174 * other words, it might not be an actual tail) so we have to save it away 175 * here since we overwrite it later. 176 */ 177 nir_deref *old_child = tail->child; 178 179 if (glsl_type_is_vector_or_scalar(tail->type)) { 180 /* Terminate the deref chain in case there is one more link to pick 181 * off a component of the vector. 182 */ 183 tail->child = NULL; 184 185 nir_intrinsic_op op = load ? nir_intrinsic_load_var : 186 nir_intrinsic_store_var; 187 188 nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op); 189 intrin->variables[0] = nir_deref_var_clone(deref, intrin); 190 intrin->num_components = glsl_get_vector_elements(tail->type); 191 192 if (load) { 193 nir_ssa_dest_init(&intrin->instr, &intrin->dest, 194 intrin->num_components, 195 glsl_get_bit_size(tail->type), 196 NULL); 197 inout->def = &intrin->dest.ssa; 198 } else { 199 nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1); 200 intrin->src[0] = nir_src_for_ssa(inout->def); 201 } 202 203 nir_builder_instr_insert(&b->nb, &intrin->instr); 204 } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY || 205 glsl_type_is_matrix(tail->type)) { 206 unsigned elems = glsl_get_length(tail->type); 207 nir_deref_array *deref_arr = nir_deref_array_create(b); 208 deref_arr->deref_array_type = nir_deref_array_type_direct; 209 deref_arr->deref.type = glsl_get_array_element(tail->type); 210 tail->child = &deref_arr->deref; 211 for (unsigned i = 0; i < elems; i++) { 212 deref_arr->base_offset = i; 213 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]); 214 } 215 } else { 216 assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT); 217 unsigned elems = glsl_get_length(tail->type); 218 nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0); 219 tail->child = &deref_struct->deref; 220 for (unsigned i = 0; i < elems; i++) { 221 deref_struct->index = i; 222 deref_struct->deref.type = glsl_get_struct_field(tail->type, i); 223 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]); 224 } 225 } 226 227 tail->child = old_child; 228 } 229 230 nir_deref_var * 231 vtn_nir_deref(struct vtn_builder *b, uint32_t id) 232 { 233 struct vtn_access_chain *chain = 234 vtn_value(b, id, vtn_value_type_access_chain)->access_chain; 235 236 return vtn_access_chain_to_deref(b, chain); 237 } 238 239 /* 240 * Gets the NIR-level deref tail, which may have as a child an array deref 241 * selecting which component due to OpAccessChain supporting per-component 242 * indexing in SPIR-V. 243 */ 244 static nir_deref * 245 get_deref_tail(nir_deref_var *deref) 246 { 247 nir_deref *cur = &deref->deref; 248 while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child) 249 cur = cur->child; 250 251 return cur; 252 } 253 254 struct vtn_ssa_value * 255 vtn_local_load(struct vtn_builder *b, nir_deref_var *src) 256 { 257 nir_deref *src_tail = get_deref_tail(src); 258 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type); 259 _vtn_local_load_store(b, true, src, src_tail, val); 260 261 if (src_tail->child) { 262 nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child); 263 assert(vec_deref->deref.child == NULL); 264 val->type = vec_deref->deref.type; 265 if (vec_deref->deref_array_type == nir_deref_array_type_direct) 266 val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset); 267 else 268 val->def = vtn_vector_extract_dynamic(b, val->def, 269 vec_deref->indirect.ssa); 270 } 271 272 return val; 273 } 274 275 void 276 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, 277 nir_deref_var *dest) 278 { 279 nir_deref *dest_tail = get_deref_tail(dest); 280 281 if (dest_tail->child) { 282 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); 283 _vtn_local_load_store(b, true, dest, dest_tail, val); 284 nir_deref_array *deref = nir_deref_as_array(dest_tail->child); 285 assert(deref->deref.child == NULL); 286 if (deref->deref_array_type == nir_deref_array_type_direct) 287 val->def = vtn_vector_insert(b, val->def, src->def, 288 deref->base_offset); 289 else 290 val->def = vtn_vector_insert_dynamic(b, val->def, src->def, 291 deref->indirect.ssa); 292 _vtn_local_load_store(b, false, dest, dest_tail, val); 293 } else { 294 _vtn_local_load_store(b, false, dest, dest_tail, src); 295 } 296 } 297 298 static nir_ssa_def * 299 get_vulkan_resource_index(struct vtn_builder *b, struct vtn_access_chain *chain, 300 struct vtn_type **type, unsigned *chain_idx) 301 { 302 /* Push constants have no explicit binding */ 303 if (chain->var->mode == vtn_variable_mode_push_constant) { 304 *chain_idx = 0; 305 *type = chain->var->type; 306 return NULL; 307 } 308 309 nir_ssa_def *array_index; 310 if (glsl_type_is_array(chain->var->type->type)) { 311 assert(chain->length > 0); 312 array_index = vtn_access_link_as_ssa(b, chain->link[0], 1); 313 *chain_idx = 1; 314 *type = chain->var->type->array_element; 315 } else { 316 array_index = nir_imm_int(&b->nb, 0); 317 *chain_idx = 0; 318 *type = chain->var->type; 319 } 320 321 nir_intrinsic_instr *instr = 322 nir_intrinsic_instr_create(b->nb.shader, 323 nir_intrinsic_vulkan_resource_index); 324 instr->src[0] = nir_src_for_ssa(array_index); 325 nir_intrinsic_set_desc_set(instr, chain->var->descriptor_set); 326 nir_intrinsic_set_binding(instr, chain->var->binding); 327 328 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 329 nir_builder_instr_insert(&b->nb, &instr->instr); 330 331 return &instr->dest.ssa; 332 } 333 334 nir_ssa_def * 335 vtn_access_chain_to_offset(struct vtn_builder *b, 336 struct vtn_access_chain *chain, 337 nir_ssa_def **index_out, struct vtn_type **type_out, 338 unsigned *end_idx_out, bool stop_at_matrix) 339 { 340 unsigned idx = 0; 341 struct vtn_type *type; 342 *index_out = get_vulkan_resource_index(b, chain, &type, &idx); 343 344 nir_ssa_def *offset = nir_imm_int(&b->nb, 0); 345 for (; idx < chain->length; idx++) { 346 enum glsl_base_type base_type = glsl_get_base_type(type->type); 347 switch (base_type) { 348 case GLSL_TYPE_UINT: 349 case GLSL_TYPE_INT: 350 case GLSL_TYPE_FLOAT: 351 case GLSL_TYPE_DOUBLE: 352 case GLSL_TYPE_BOOL: 353 /* Some users may not want matrix or vector derefs */ 354 if (stop_at_matrix) 355 goto end; 356 /* Fall through */ 357 358 case GLSL_TYPE_ARRAY: 359 offset = nir_iadd(&b->nb, offset, 360 vtn_access_link_as_ssa(b, chain->link[idx], 361 type->stride)); 362 363 type = type->array_element; 364 break; 365 366 case GLSL_TYPE_STRUCT: { 367 assert(chain->link[idx].mode == vtn_access_mode_literal); 368 unsigned member = chain->link[idx].id; 369 offset = nir_iadd(&b->nb, offset, 370 nir_imm_int(&b->nb, type->offsets[member])); 371 type = type->members[member]; 372 break; 373 } 374 375 default: 376 unreachable("Invalid type for deref"); 377 } 378 } 379 380 end: 381 *type_out = type; 382 if (end_idx_out) 383 *end_idx_out = idx; 384 385 return offset; 386 } 387 388 /* Tries to compute the size of an interface block based on the strides and 389 * offsets that are provided to us in the SPIR-V source. 390 */ 391 static unsigned 392 vtn_type_block_size(struct vtn_type *type) 393 { 394 enum glsl_base_type base_type = glsl_get_base_type(type->type); 395 switch (base_type) { 396 case GLSL_TYPE_UINT: 397 case GLSL_TYPE_INT: 398 case GLSL_TYPE_FLOAT: 399 case GLSL_TYPE_BOOL: 400 case GLSL_TYPE_DOUBLE: { 401 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) : 402 glsl_get_matrix_columns(type->type); 403 if (cols > 1) { 404 assert(type->stride > 0); 405 return type->stride * cols; 406 } else if (base_type == GLSL_TYPE_DOUBLE) { 407 return glsl_get_vector_elements(type->type) * 8; 408 } else { 409 return glsl_get_vector_elements(type->type) * 4; 410 } 411 } 412 413 case GLSL_TYPE_STRUCT: 414 case GLSL_TYPE_INTERFACE: { 415 unsigned size = 0; 416 unsigned num_fields = glsl_get_length(type->type); 417 for (unsigned f = 0; f < num_fields; f++) { 418 unsigned field_end = type->offsets[f] + 419 vtn_type_block_size(type->members[f]); 420 size = MAX2(size, field_end); 421 } 422 return size; 423 } 424 425 case GLSL_TYPE_ARRAY: 426 assert(type->stride > 0); 427 assert(glsl_get_length(type->type) > 0); 428 return type->stride * glsl_get_length(type->type); 429 430 default: 431 assert(!"Invalid block type"); 432 return 0; 433 } 434 } 435 436 static void 437 vtn_access_chain_get_offset_size(struct vtn_access_chain *chain, 438 unsigned *access_offset, 439 unsigned *access_size) 440 { 441 /* Only valid for push constants accesses now. */ 442 assert(chain->var->mode == vtn_variable_mode_push_constant); 443 444 struct vtn_type *type = chain->var->type; 445 446 *access_offset = 0; 447 448 for (unsigned i = 0; i < chain->length; i++) { 449 if (chain->link[i].mode != vtn_access_mode_literal) 450 break; 451 452 if (glsl_type_is_struct(type->type)) { 453 *access_offset += type->offsets[chain->link[i].id]; 454 type = type->members[chain->link[i].id]; 455 } else { 456 *access_offset += type->stride * chain->link[i].id; 457 type = type->array_element; 458 } 459 } 460 461 *access_size = vtn_type_block_size(type); 462 } 463 464 static void 465 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load, 466 nir_ssa_def *index, nir_ssa_def *offset, 467 unsigned access_offset, unsigned access_size, 468 struct vtn_ssa_value **inout, const struct glsl_type *type) 469 { 470 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op); 471 instr->num_components = glsl_get_vector_elements(type); 472 473 int src = 0; 474 if (!load) { 475 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1); 476 instr->src[src++] = nir_src_for_ssa((*inout)->def); 477 } 478 479 if (op == nir_intrinsic_load_push_constant) { 480 assert(access_offset % 4 == 0); 481 482 nir_intrinsic_set_base(instr, access_offset); 483 nir_intrinsic_set_range(instr, access_size); 484 } 485 486 if (index) 487 instr->src[src++] = nir_src_for_ssa(index); 488 489 if (op == nir_intrinsic_load_push_constant) { 490 /* We need to subtract the offset from where the intrinsic will load the 491 * data. */ 492 instr->src[src++] = 493 nir_src_for_ssa(nir_isub(&b->nb, offset, 494 nir_imm_int(&b->nb, access_offset))); 495 } else { 496 instr->src[src++] = nir_src_for_ssa(offset); 497 } 498 499 if (load) { 500 nir_ssa_dest_init(&instr->instr, &instr->dest, 501 instr->num_components, 502 glsl_get_bit_size(type), NULL); 503 (*inout)->def = &instr->dest.ssa; 504 } 505 506 nir_builder_instr_insert(&b->nb, &instr->instr); 507 508 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL) 509 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0)); 510 } 511 512 static void 513 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load, 514 nir_ssa_def *index, nir_ssa_def *offset, 515 unsigned access_offset, unsigned access_size, 516 struct vtn_access_chain *chain, unsigned chain_idx, 517 struct vtn_type *type, struct vtn_ssa_value **inout) 518 { 519 if (chain && chain_idx >= chain->length) 520 chain = NULL; 521 522 if (load && chain == NULL && *inout == NULL) 523 *inout = vtn_create_ssa_value(b, type->type); 524 525 enum glsl_base_type base_type = glsl_get_base_type(type->type); 526 switch (base_type) { 527 case GLSL_TYPE_UINT: 528 case GLSL_TYPE_INT: 529 case GLSL_TYPE_FLOAT: 530 case GLSL_TYPE_DOUBLE: 531 case GLSL_TYPE_BOOL: 532 /* This is where things get interesting. At this point, we've hit 533 * a vector, a scalar, or a matrix. 534 */ 535 if (glsl_type_is_matrix(type->type)) { 536 if (chain == NULL) { 537 /* Loading the whole matrix */ 538 struct vtn_ssa_value *transpose; 539 unsigned num_ops, vec_width; 540 if (type->row_major) { 541 num_ops = glsl_get_vector_elements(type->type); 542 vec_width = glsl_get_matrix_columns(type->type); 543 if (load) { 544 const struct glsl_type *transpose_type = 545 glsl_matrix_type(base_type, vec_width, num_ops); 546 *inout = vtn_create_ssa_value(b, transpose_type); 547 } else { 548 transpose = vtn_ssa_transpose(b, *inout); 549 inout = &transpose; 550 } 551 } else { 552 num_ops = glsl_get_matrix_columns(type->type); 553 vec_width = glsl_get_vector_elements(type->type); 554 } 555 556 for (unsigned i = 0; i < num_ops; i++) { 557 nir_ssa_def *elem_offset = 558 nir_iadd(&b->nb, offset, 559 nir_imm_int(&b->nb, i * type->stride)); 560 _vtn_load_store_tail(b, op, load, index, elem_offset, 561 access_offset, access_size, 562 &(*inout)->elems[i], 563 glsl_vector_type(base_type, vec_width)); 564 } 565 566 if (load && type->row_major) 567 *inout = vtn_ssa_transpose(b, *inout); 568 } else if (type->row_major) { 569 /* Row-major but with an access chiain. */ 570 nir_ssa_def *col_offset = 571 vtn_access_link_as_ssa(b, chain->link[chain_idx], 572 type->array_element->stride); 573 offset = nir_iadd(&b->nb, offset, col_offset); 574 575 if (chain_idx + 1 < chain->length) { 576 /* Picking off a single element */ 577 nir_ssa_def *row_offset = 578 vtn_access_link_as_ssa(b, chain->link[chain_idx + 1], 579 type->stride); 580 offset = nir_iadd(&b->nb, offset, row_offset); 581 if (load) 582 *inout = vtn_create_ssa_value(b, glsl_scalar_type(base_type)); 583 _vtn_load_store_tail(b, op, load, index, offset, 584 access_offset, access_size, 585 inout, glsl_scalar_type(base_type)); 586 } else { 587 /* Grabbing a column; picking one element off each row */ 588 unsigned num_comps = glsl_get_vector_elements(type->type); 589 const struct glsl_type *column_type = 590 glsl_get_column_type(type->type); 591 592 nir_ssa_def *comps[4]; 593 for (unsigned i = 0; i < num_comps; i++) { 594 nir_ssa_def *elem_offset = 595 nir_iadd(&b->nb, offset, 596 nir_imm_int(&b->nb, i * type->stride)); 597 598 struct vtn_ssa_value *comp, temp_val; 599 if (!load) { 600 temp_val.def = nir_channel(&b->nb, (*inout)->def, i); 601 temp_val.type = glsl_scalar_type(base_type); 602 } 603 comp = &temp_val; 604 _vtn_load_store_tail(b, op, load, index, elem_offset, 605 access_offset, access_size, 606 &comp, glsl_scalar_type(base_type)); 607 comps[i] = comp->def; 608 } 609 610 if (load) { 611 if (*inout == NULL) 612 *inout = vtn_create_ssa_value(b, column_type); 613 614 (*inout)->def = nir_vec(&b->nb, comps, num_comps); 615 } 616 } 617 } else { 618 /* Column-major with a deref. Fall through to array case. */ 619 nir_ssa_def *col_offset = 620 vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride); 621 offset = nir_iadd(&b->nb, offset, col_offset); 622 623 _vtn_block_load_store(b, op, load, index, offset, 624 access_offset, access_size, 625 chain, chain_idx + 1, 626 type->array_element, inout); 627 } 628 } else if (chain == NULL) { 629 /* Single whole vector */ 630 assert(glsl_type_is_vector_or_scalar(type->type)); 631 _vtn_load_store_tail(b, op, load, index, offset, 632 access_offset, access_size, 633 inout, type->type); 634 } else { 635 /* Single component of a vector. Fall through to array case. */ 636 nir_ssa_def *elem_offset = 637 vtn_access_link_as_ssa(b, chain->link[chain_idx], type->stride); 638 offset = nir_iadd(&b->nb, offset, elem_offset); 639 640 _vtn_block_load_store(b, op, load, index, offset, 641 access_offset, access_size, 642 NULL, 0, 643 type->array_element, inout); 644 } 645 return; 646 647 case GLSL_TYPE_ARRAY: { 648 unsigned elems = glsl_get_length(type->type); 649 for (unsigned i = 0; i < elems; i++) { 650 nir_ssa_def *elem_off = 651 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride)); 652 _vtn_block_load_store(b, op, load, index, elem_off, 653 access_offset, access_size, 654 NULL, 0, 655 type->array_element, &(*inout)->elems[i]); 656 } 657 return; 658 } 659 660 case GLSL_TYPE_STRUCT: { 661 unsigned elems = glsl_get_length(type->type); 662 for (unsigned i = 0; i < elems; i++) { 663 nir_ssa_def *elem_off = 664 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i])); 665 _vtn_block_load_store(b, op, load, index, elem_off, 666 access_offset, access_size, 667 NULL, 0, 668 type->members[i], &(*inout)->elems[i]); 669 } 670 return; 671 } 672 673 default: 674 unreachable("Invalid block member type"); 675 } 676 } 677 678 static struct vtn_ssa_value * 679 vtn_block_load(struct vtn_builder *b, struct vtn_access_chain *src) 680 { 681 nir_intrinsic_op op; 682 unsigned access_offset = 0, access_size = 0; 683 switch (src->var->mode) { 684 case vtn_variable_mode_ubo: 685 op = nir_intrinsic_load_ubo; 686 break; 687 case vtn_variable_mode_ssbo: 688 op = nir_intrinsic_load_ssbo; 689 break; 690 case vtn_variable_mode_push_constant: 691 op = nir_intrinsic_load_push_constant; 692 vtn_access_chain_get_offset_size(src, &access_offset, &access_size); 693 break; 694 default: 695 assert(!"Invalid block variable mode"); 696 } 697 698 nir_ssa_def *offset, *index = NULL; 699 struct vtn_type *type; 700 unsigned chain_idx; 701 offset = vtn_access_chain_to_offset(b, src, &index, &type, &chain_idx, true); 702 703 struct vtn_ssa_value *value = NULL; 704 _vtn_block_load_store(b, op, true, index, offset, 705 access_offset, access_size, 706 src, chain_idx, type, &value); 707 return value; 708 } 709 710 static void 711 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src, 712 struct vtn_access_chain *dst) 713 { 714 nir_ssa_def *offset, *index = NULL; 715 struct vtn_type *type; 716 unsigned chain_idx; 717 offset = vtn_access_chain_to_offset(b, dst, &index, &type, &chain_idx, true); 718 719 _vtn_block_load_store(b, nir_intrinsic_store_ssbo, false, index, offset, 720 0, 0, dst, chain_idx, type, &src); 721 } 722 723 static bool 724 vtn_variable_is_external_block(struct vtn_variable *var) 725 { 726 return var->mode == vtn_variable_mode_ssbo || 727 var->mode == vtn_variable_mode_ubo || 728 var->mode == vtn_variable_mode_push_constant; 729 } 730 731 static void 732 _vtn_variable_load_store(struct vtn_builder *b, bool load, 733 struct vtn_access_chain *chain, 734 struct vtn_type *tail_type, 735 struct vtn_ssa_value **inout) 736 { 737 enum glsl_base_type base_type = glsl_get_base_type(tail_type->type); 738 switch (base_type) { 739 case GLSL_TYPE_UINT: 740 case GLSL_TYPE_INT: 741 case GLSL_TYPE_FLOAT: 742 case GLSL_TYPE_BOOL: 743 case GLSL_TYPE_DOUBLE: 744 /* At this point, we have a scalar, vector, or matrix so we know that 745 * there cannot be any structure splitting still in the way. By 746 * stopping at the matrix level rather than the vector level, we 747 * ensure that matrices get loaded in the optimal way even if they 748 * are storred row-major in a UBO. 749 */ 750 if (load) { 751 *inout = vtn_local_load(b, vtn_access_chain_to_deref(b, chain)); 752 } else { 753 vtn_local_store(b, *inout, vtn_access_chain_to_deref(b, chain)); 754 } 755 return; 756 757 case GLSL_TYPE_ARRAY: 758 case GLSL_TYPE_STRUCT: { 759 struct vtn_access_chain *new_chain = 760 vtn_access_chain_extend(b, chain, 1); 761 new_chain->link[chain->length].mode = vtn_access_mode_literal; 762 unsigned elems = glsl_get_length(tail_type->type); 763 if (load) { 764 assert(*inout == NULL); 765 *inout = rzalloc(b, struct vtn_ssa_value); 766 (*inout)->type = tail_type->type; 767 (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems); 768 } 769 for (unsigned i = 0; i < elems; i++) { 770 new_chain->link[chain->length].id = i; 771 struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ? 772 tail_type->array_element : tail_type->members[i]; 773 _vtn_variable_load_store(b, load, new_chain, elem_type, 774 &(*inout)->elems[i]); 775 } 776 return; 777 } 778 779 default: 780 unreachable("Invalid access chain type"); 781 } 782 } 783 784 struct vtn_ssa_value * 785 vtn_variable_load(struct vtn_builder *b, struct vtn_access_chain *src) 786 { 787 if (vtn_variable_is_external_block(src->var)) { 788 return vtn_block_load(b, src); 789 } else { 790 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src); 791 struct vtn_ssa_value *val = NULL; 792 _vtn_variable_load_store(b, true, src, tail_type, &val); 793 return val; 794 } 795 } 796 797 void 798 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src, 799 struct vtn_access_chain *dest) 800 { 801 if (vtn_variable_is_external_block(dest->var)) { 802 assert(dest->var->mode == vtn_variable_mode_ssbo); 803 vtn_block_store(b, src, dest); 804 } else { 805 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, dest); 806 _vtn_variable_load_store(b, false, dest, tail_type, &src); 807 } 808 } 809 810 static void 811 _vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest, 812 struct vtn_access_chain *src, struct vtn_type *tail_type) 813 { 814 enum glsl_base_type base_type = glsl_get_base_type(tail_type->type); 815 switch (base_type) { 816 case GLSL_TYPE_UINT: 817 case GLSL_TYPE_INT: 818 case GLSL_TYPE_FLOAT: 819 case GLSL_TYPE_DOUBLE: 820 case GLSL_TYPE_BOOL: 821 /* At this point, we have a scalar, vector, or matrix so we know that 822 * there cannot be any structure splitting still in the way. By 823 * stopping at the matrix level rather than the vector level, we 824 * ensure that matrices get loaded in the optimal way even if they 825 * are storred row-major in a UBO. 826 */ 827 vtn_variable_store(b, vtn_variable_load(b, src), dest); 828 return; 829 830 case GLSL_TYPE_ARRAY: 831 case GLSL_TYPE_STRUCT: { 832 struct vtn_access_chain *new_src, *new_dest; 833 new_src = vtn_access_chain_extend(b, src, 1); 834 new_dest = vtn_access_chain_extend(b, dest, 1); 835 new_src->link[src->length].mode = vtn_access_mode_literal; 836 new_dest->link[dest->length].mode = vtn_access_mode_literal; 837 unsigned elems = glsl_get_length(tail_type->type); 838 for (unsigned i = 0; i < elems; i++) { 839 new_src->link[src->length].id = i; 840 new_dest->link[dest->length].id = i; 841 struct vtn_type *elem_type = base_type == GLSL_TYPE_ARRAY ? 842 tail_type->array_element : tail_type->members[i]; 843 _vtn_variable_copy(b, new_dest, new_src, elem_type); 844 } 845 return; 846 } 847 848 default: 849 unreachable("Invalid access chain type"); 850 } 851 } 852 853 static void 854 vtn_variable_copy(struct vtn_builder *b, struct vtn_access_chain *dest, 855 struct vtn_access_chain *src) 856 { 857 struct vtn_type *tail_type = vtn_access_chain_tail_type(b, src); 858 assert(vtn_access_chain_tail_type(b, dest)->type == tail_type->type); 859 860 /* TODO: At some point, we should add a special-case for when we can 861 * just emit a copy_var intrinsic. 862 */ 863 _vtn_variable_copy(b, dest, src, tail_type); 864 } 865 866 static void 867 set_mode_system_value(nir_variable_mode *mode) 868 { 869 assert(*mode == nir_var_system_value || *mode == nir_var_shader_in); 870 *mode = nir_var_system_value; 871 } 872 873 static void 874 vtn_get_builtin_location(struct vtn_builder *b, 875 SpvBuiltIn builtin, int *location, 876 nir_variable_mode *mode) 877 { 878 switch (builtin) { 879 case SpvBuiltInPosition: 880 *location = VARYING_SLOT_POS; 881 break; 882 case SpvBuiltInPointSize: 883 *location = VARYING_SLOT_PSIZ; 884 break; 885 case SpvBuiltInClipDistance: 886 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */ 887 break; 888 case SpvBuiltInCullDistance: 889 *location = VARYING_SLOT_CULL_DIST0; 890 break; 891 case SpvBuiltInVertexIndex: 892 *location = SYSTEM_VALUE_VERTEX_ID; 893 set_mode_system_value(mode); 894 break; 895 case SpvBuiltInVertexId: 896 /* Vulkan defines VertexID to be zero-based and reserves the new 897 * builtin keyword VertexIndex to indicate the non-zero-based value. 898 */ 899 *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE; 900 set_mode_system_value(mode); 901 break; 902 case SpvBuiltInInstanceIndex: 903 *location = SYSTEM_VALUE_INSTANCE_INDEX; 904 set_mode_system_value(mode); 905 break; 906 case SpvBuiltInInstanceId: 907 *location = SYSTEM_VALUE_INSTANCE_ID; 908 set_mode_system_value(mode); 909 break; 910 case SpvBuiltInPrimitiveId: 911 if (b->shader->stage == MESA_SHADER_FRAGMENT) { 912 assert(*mode == nir_var_shader_in); 913 *location = VARYING_SLOT_PRIMITIVE_ID; 914 } else if (*mode == nir_var_shader_out) { 915 *location = VARYING_SLOT_PRIMITIVE_ID; 916 } else { 917 *location = SYSTEM_VALUE_PRIMITIVE_ID; 918 set_mode_system_value(mode); 919 } 920 break; 921 case SpvBuiltInInvocationId: 922 *location = SYSTEM_VALUE_INVOCATION_ID; 923 set_mode_system_value(mode); 924 break; 925 case SpvBuiltInLayer: 926 *location = VARYING_SLOT_LAYER; 927 if (b->shader->stage == MESA_SHADER_FRAGMENT) 928 *mode = nir_var_shader_in; 929 else if (b->shader->stage == MESA_SHADER_GEOMETRY) 930 *mode = nir_var_shader_out; 931 else 932 unreachable("invalid stage for SpvBuiltInLayer"); 933 break; 934 case SpvBuiltInViewportIndex: 935 *location = VARYING_SLOT_VIEWPORT; 936 if (b->shader->stage == MESA_SHADER_GEOMETRY) 937 *mode = nir_var_shader_out; 938 else if (b->shader->stage == MESA_SHADER_FRAGMENT) 939 *mode = nir_var_shader_in; 940 else 941 unreachable("invalid stage for SpvBuiltInViewportIndex"); 942 break; 943 case SpvBuiltInTessLevelOuter: 944 *location = VARYING_SLOT_TESS_LEVEL_OUTER; 945 break; 946 case SpvBuiltInTessLevelInner: 947 *location = VARYING_SLOT_TESS_LEVEL_INNER; 948 break; 949 case SpvBuiltInTessCoord: 950 *location = SYSTEM_VALUE_TESS_COORD; 951 set_mode_system_value(mode); 952 break; 953 case SpvBuiltInPatchVertices: 954 *location = SYSTEM_VALUE_VERTICES_IN; 955 set_mode_system_value(mode); 956 break; 957 case SpvBuiltInFragCoord: 958 *location = VARYING_SLOT_POS; 959 assert(*mode == nir_var_shader_in); 960 break; 961 case SpvBuiltInPointCoord: 962 *location = VARYING_SLOT_PNTC; 963 assert(*mode == nir_var_shader_in); 964 break; 965 case SpvBuiltInFrontFacing: 966 *location = SYSTEM_VALUE_FRONT_FACE; 967 set_mode_system_value(mode); 968 break; 969 case SpvBuiltInSampleId: 970 *location = SYSTEM_VALUE_SAMPLE_ID; 971 set_mode_system_value(mode); 972 break; 973 case SpvBuiltInSamplePosition: 974 *location = SYSTEM_VALUE_SAMPLE_POS; 975 set_mode_system_value(mode); 976 break; 977 case SpvBuiltInSampleMask: 978 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; /* XXX out? */ 979 set_mode_system_value(mode); 980 break; 981 case SpvBuiltInFragDepth: 982 *location = FRAG_RESULT_DEPTH; 983 assert(*mode == nir_var_shader_out); 984 break; 985 case SpvBuiltInNumWorkgroups: 986 *location = SYSTEM_VALUE_NUM_WORK_GROUPS; 987 set_mode_system_value(mode); 988 break; 989 case SpvBuiltInWorkgroupSize: 990 /* This should already be handled */ 991 unreachable("unsupported builtin"); 992 break; 993 case SpvBuiltInWorkgroupId: 994 *location = SYSTEM_VALUE_WORK_GROUP_ID; 995 set_mode_system_value(mode); 996 break; 997 case SpvBuiltInLocalInvocationId: 998 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID; 999 set_mode_system_value(mode); 1000 break; 1001 case SpvBuiltInLocalInvocationIndex: 1002 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 1003 set_mode_system_value(mode); 1004 break; 1005 case SpvBuiltInGlobalInvocationId: 1006 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID; 1007 set_mode_system_value(mode); 1008 break; 1009 case SpvBuiltInHelperInvocation: 1010 default: 1011 unreachable("unsupported builtin"); 1012 } 1013 } 1014 1015 static void 1016 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var, 1017 const struct vtn_decoration *dec) 1018 { 1019 switch (dec->decoration) { 1020 case SpvDecorationRelaxedPrecision: 1021 break; /* FIXME: Do nothing with this for now. */ 1022 case SpvDecorationNoPerspective: 1023 nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE; 1024 break; 1025 case SpvDecorationFlat: 1026 nir_var->data.interpolation = INTERP_MODE_FLAT; 1027 break; 1028 case SpvDecorationCentroid: 1029 nir_var->data.centroid = true; 1030 break; 1031 case SpvDecorationSample: 1032 nir_var->data.sample = true; 1033 break; 1034 case SpvDecorationInvariant: 1035 nir_var->data.invariant = true; 1036 break; 1037 case SpvDecorationConstant: 1038 assert(nir_var->constant_initializer != NULL); 1039 nir_var->data.read_only = true; 1040 break; 1041 case SpvDecorationNonWritable: 1042 nir_var->data.read_only = true; 1043 break; 1044 case SpvDecorationComponent: 1045 nir_var->data.location_frac = dec->literals[0]; 1046 break; 1047 case SpvDecorationIndex: 1048 nir_var->data.index = dec->literals[0]; 1049 break; 1050 case SpvDecorationBuiltIn: { 1051 SpvBuiltIn builtin = dec->literals[0]; 1052 1053 if (builtin == SpvBuiltInWorkgroupSize) { 1054 /* This shouldn't be a builtin. It's actually a constant. */ 1055 nir_var->data.mode = nir_var_global; 1056 nir_var->data.read_only = true; 1057 1058 nir_constant *c = rzalloc(nir_var, nir_constant); 1059 c->values[0].u32[0] = b->shader->info->cs.local_size[0]; 1060 c->values[0].u32[1] = b->shader->info->cs.local_size[1]; 1061 c->values[0].u32[2] = b->shader->info->cs.local_size[2]; 1062 nir_var->constant_initializer = c; 1063 break; 1064 } 1065 1066 nir_variable_mode mode = nir_var->data.mode; 1067 vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode); 1068 nir_var->data.mode = mode; 1069 1070 switch (builtin) { 1071 case SpvBuiltInTessLevelOuter: 1072 case SpvBuiltInTessLevelInner: 1073 nir_var->data.compact = true; 1074 break; 1075 case SpvBuiltInSamplePosition: 1076 nir_var->data.origin_upper_left = b->origin_upper_left; 1077 /* fallthrough */ 1078 case SpvBuiltInFragCoord: 1079 nir_var->data.pixel_center_integer = b->pixel_center_integer; 1080 break; 1081 default: 1082 break; 1083 } 1084 } 1085 1086 case SpvDecorationSpecId: 1087 case SpvDecorationRowMajor: 1088 case SpvDecorationColMajor: 1089 case SpvDecorationMatrixStride: 1090 case SpvDecorationRestrict: 1091 case SpvDecorationAliased: 1092 case SpvDecorationVolatile: 1093 case SpvDecorationCoherent: 1094 case SpvDecorationNonReadable: 1095 case SpvDecorationUniform: 1096 case SpvDecorationStream: 1097 case SpvDecorationOffset: 1098 case SpvDecorationLinkageAttributes: 1099 break; /* Do nothing with these here */ 1100 1101 case SpvDecorationPatch: 1102 nir_var->data.patch = true; 1103 break; 1104 1105 case SpvDecorationLocation: 1106 unreachable("Handled above"); 1107 1108 case SpvDecorationBlock: 1109 case SpvDecorationBufferBlock: 1110 case SpvDecorationArrayStride: 1111 case SpvDecorationGLSLShared: 1112 case SpvDecorationGLSLPacked: 1113 break; /* These can apply to a type but we don't care about them */ 1114 1115 case SpvDecorationBinding: 1116 case SpvDecorationDescriptorSet: 1117 case SpvDecorationNoContraction: 1118 case SpvDecorationInputAttachmentIndex: 1119 vtn_warn("Decoration not allowed for variable or structure member: %s", 1120 spirv_decoration_to_string(dec->decoration)); 1121 break; 1122 1123 case SpvDecorationXfbBuffer: 1124 case SpvDecorationXfbStride: 1125 vtn_warn("Vulkan does not have transform feedback: %s", 1126 spirv_decoration_to_string(dec->decoration)); 1127 break; 1128 1129 case SpvDecorationCPacked: 1130 case SpvDecorationSaturatedConversion: 1131 case SpvDecorationFuncParamAttr: 1132 case SpvDecorationFPRoundingMode: 1133 case SpvDecorationFPFastMathMode: 1134 case SpvDecorationAlignment: 1135 vtn_warn("Decoraiton only allowed for CL-style kernels: %s", 1136 spirv_decoration_to_string(dec->decoration)); 1137 break; 1138 } 1139 } 1140 1141 static void 1142 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1143 const struct vtn_decoration *dec, void *out_is_patch) 1144 { 1145 if (dec->decoration == SpvDecorationPatch) { 1146 *((bool *) out_is_patch) = true; 1147 } 1148 } 1149 1150 static void 1151 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1152 const struct vtn_decoration *dec, void *void_var) 1153 { 1154 struct vtn_variable *vtn_var = void_var; 1155 1156 /* Handle decorations that apply to a vtn_variable as a whole */ 1157 switch (dec->decoration) { 1158 case SpvDecorationBinding: 1159 vtn_var->binding = dec->literals[0]; 1160 return; 1161 case SpvDecorationDescriptorSet: 1162 vtn_var->descriptor_set = dec->literals[0]; 1163 return; 1164 case SpvDecorationInputAttachmentIndex: 1165 vtn_var->input_attachment_index = dec->literals[0]; 1166 return; 1167 case SpvDecorationPatch: 1168 vtn_var->patch = true; 1169 break; 1170 default: 1171 break; 1172 } 1173 1174 if (val->value_type == vtn_value_type_access_chain) { 1175 assert(val->access_chain->length == 0); 1176 assert(val->access_chain->var == void_var); 1177 assert(member == -1); 1178 } else { 1179 assert(val->value_type == vtn_value_type_type); 1180 } 1181 1182 /* Location is odd. If applied to a split structure, we have to walk the 1183 * whole thing and accumulate the location. It's easier to handle as a 1184 * special case. 1185 */ 1186 if (dec->decoration == SpvDecorationLocation) { 1187 unsigned location = dec->literals[0]; 1188 bool is_vertex_input; 1189 if (b->shader->stage == MESA_SHADER_FRAGMENT && 1190 vtn_var->mode == vtn_variable_mode_output) { 1191 is_vertex_input = false; 1192 location += FRAG_RESULT_DATA0; 1193 } else if (b->shader->stage == MESA_SHADER_VERTEX && 1194 vtn_var->mode == vtn_variable_mode_input) { 1195 is_vertex_input = true; 1196 location += VERT_ATTRIB_GENERIC0; 1197 } else if (vtn_var->mode == vtn_variable_mode_input || 1198 vtn_var->mode == vtn_variable_mode_output) { 1199 is_vertex_input = false; 1200 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0; 1201 } else { 1202 vtn_warn("Location must be on input or output variable"); 1203 return; 1204 } 1205 1206 if (vtn_var->var) { 1207 /* This handles the member and lone variable cases */ 1208 vtn_var->var->data.location = location; 1209 } else { 1210 /* This handles the structure member case */ 1211 assert(vtn_var->members); 1212 unsigned length = 1213 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1214 for (unsigned i = 0; i < length; i++) { 1215 vtn_var->members[i]->data.location = location; 1216 location += 1217 glsl_count_attribute_slots(vtn_var->members[i]->interface_type, 1218 is_vertex_input); 1219 } 1220 } 1221 return; 1222 } else { 1223 if (vtn_var->var) { 1224 assert(member <= 0); 1225 apply_var_decoration(b, vtn_var->var, dec); 1226 } else if (vtn_var->members) { 1227 if (member >= 0) { 1228 assert(vtn_var->members); 1229 apply_var_decoration(b, vtn_var->members[member], dec); 1230 } else { 1231 unsigned length = 1232 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1233 for (unsigned i = 0; i < length; i++) 1234 apply_var_decoration(b, vtn_var->members[i], dec); 1235 } 1236 } else { 1237 /* A few variables, those with external storage, have no actual 1238 * nir_variables associated with them. Fortunately, all decorations 1239 * we care about for those variables are on the type only. 1240 */ 1241 assert(vtn_var->mode == vtn_variable_mode_ubo || 1242 vtn_var->mode == vtn_variable_mode_ssbo || 1243 vtn_var->mode == vtn_variable_mode_push_constant); 1244 } 1245 } 1246 } 1247 1248 static bool 1249 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage) 1250 { 1251 if (var->patch || !glsl_type_is_array(var->type->type)) 1252 return false; 1253 1254 if (var->mode == vtn_variable_mode_input) { 1255 return stage == MESA_SHADER_TESS_CTRL || 1256 stage == MESA_SHADER_TESS_EVAL || 1257 stage == MESA_SHADER_GEOMETRY; 1258 } 1259 1260 if (var->mode == vtn_variable_mode_output) 1261 return stage == MESA_SHADER_TESS_CTRL; 1262 1263 return false; 1264 } 1265 1266 void 1267 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, 1268 const uint32_t *w, unsigned count) 1269 { 1270 switch (opcode) { 1271 case SpvOpUndef: { 1272 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); 1273 val->type = vtn_value(b, w[1], vtn_value_type_type)->type; 1274 break; 1275 } 1276 1277 case SpvOpVariable: { 1278 struct vtn_variable *var = rzalloc(b, struct vtn_variable); 1279 var->type = vtn_value(b, w[1], vtn_value_type_type)->type; 1280 1281 var->chain.var = var; 1282 var->chain.length = 0; 1283 1284 struct vtn_value *val = 1285 vtn_push_value(b, w[2], vtn_value_type_access_chain); 1286 val->access_chain = &var->chain; 1287 1288 struct vtn_type *without_array = var->type; 1289 while(glsl_type_is_array(without_array->type)) 1290 without_array = without_array->array_element; 1291 1292 nir_variable_mode nir_mode; 1293 switch ((SpvStorageClass)w[3]) { 1294 case SpvStorageClassUniform: 1295 case SpvStorageClassUniformConstant: 1296 if (without_array->block) { 1297 var->mode = vtn_variable_mode_ubo; 1298 b->shader->info->num_ubos++; 1299 } else if (without_array->buffer_block) { 1300 var->mode = vtn_variable_mode_ssbo; 1301 b->shader->info->num_ssbos++; 1302 } else if (glsl_type_is_image(without_array->type)) { 1303 var->mode = vtn_variable_mode_image; 1304 nir_mode = nir_var_uniform; 1305 b->shader->info->num_images++; 1306 } else if (glsl_type_is_sampler(without_array->type)) { 1307 var->mode = vtn_variable_mode_sampler; 1308 nir_mode = nir_var_uniform; 1309 b->shader->info->num_textures++; 1310 } else { 1311 assert(!"Invalid uniform variable type"); 1312 } 1313 break; 1314 case SpvStorageClassPushConstant: 1315 var->mode = vtn_variable_mode_push_constant; 1316 assert(b->shader->num_uniforms == 0); 1317 b->shader->num_uniforms = vtn_type_block_size(var->type); 1318 break; 1319 case SpvStorageClassInput: 1320 var->mode = vtn_variable_mode_input; 1321 nir_mode = nir_var_shader_in; 1322 break; 1323 case SpvStorageClassOutput: 1324 var->mode = vtn_variable_mode_output; 1325 nir_mode = nir_var_shader_out; 1326 break; 1327 case SpvStorageClassPrivate: 1328 var->mode = vtn_variable_mode_global; 1329 nir_mode = nir_var_global; 1330 break; 1331 case SpvStorageClassFunction: 1332 var->mode = vtn_variable_mode_local; 1333 nir_mode = nir_var_local; 1334 break; 1335 case SpvStorageClassWorkgroup: 1336 var->mode = vtn_variable_mode_workgroup; 1337 nir_mode = nir_var_shared; 1338 break; 1339 case SpvStorageClassCrossWorkgroup: 1340 case SpvStorageClassGeneric: 1341 case SpvStorageClassAtomicCounter: 1342 default: 1343 unreachable("Unhandled variable storage class"); 1344 } 1345 1346 switch (var->mode) { 1347 case vtn_variable_mode_local: 1348 case vtn_variable_mode_global: 1349 case vtn_variable_mode_image: 1350 case vtn_variable_mode_sampler: 1351 case vtn_variable_mode_workgroup: 1352 /* For these, we create the variable normally */ 1353 var->var = rzalloc(b->shader, nir_variable); 1354 var->var->name = ralloc_strdup(var->var, val->name); 1355 var->var->type = var->type->type; 1356 var->var->data.mode = nir_mode; 1357 1358 switch (var->mode) { 1359 case vtn_variable_mode_image: 1360 case vtn_variable_mode_sampler: 1361 var->var->interface_type = without_array->type; 1362 break; 1363 default: 1364 var->var->interface_type = NULL; 1365 break; 1366 } 1367 break; 1368 1369 case vtn_variable_mode_input: 1370 case vtn_variable_mode_output: { 1371 /* In order to know whether or not we're a per-vertex inout, we need 1372 * the patch qualifier. This means walking the variable decorations 1373 * early before we actually create any variables. Not a big deal. 1374 * 1375 * GLSLang really likes to place decorations in the most interior 1376 * thing it possibly can. In particular, if you have a struct, it 1377 * will place the patch decorations on the struct members. This 1378 * should be handled by the variable splitting below just fine. 1379 * 1380 * If you have an array-of-struct, things get even more weird as it 1381 * will place the patch decorations on the struct even though it's 1382 * inside an array and some of the members being patch and others not 1383 * makes no sense whatsoever. Since the only sensible thing is for 1384 * it to be all or nothing, we'll call it patch if any of the members 1385 * are declared patch. 1386 */ 1387 var->patch = false; 1388 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch); 1389 if (glsl_type_is_array(var->type->type) && 1390 glsl_type_is_struct(without_array->type)) { 1391 vtn_foreach_decoration(b, without_array->val, 1392 var_is_patch_cb, &var->patch); 1393 } 1394 1395 /* For inputs and outputs, we immediately split structures. This 1396 * is for a couple of reasons. For one, builtins may all come in 1397 * a struct and we really want those split out into separate 1398 * variables. For another, interpolation qualifiers can be 1399 * applied to members of the top-level struct ane we need to be 1400 * able to preserve that information. 1401 */ 1402 1403 int array_length = -1; 1404 struct vtn_type *interface_type = var->type; 1405 if (is_per_vertex_inout(var, b->shader->stage)) { 1406 /* In Geometry shaders (and some tessellation), inputs come 1407 * in per-vertex arrays. However, some builtins come in 1408 * non-per-vertex, hence the need for the is_array check. In 1409 * any case, there are no non-builtin arrays allowed so this 1410 * check should be sufficient. 1411 */ 1412 interface_type = var->type->array_element; 1413 array_length = glsl_get_length(var->type->type); 1414 } 1415 1416 if (glsl_type_is_struct(interface_type->type)) { 1417 /* It's a struct. Split it. */ 1418 unsigned num_members = glsl_get_length(interface_type->type); 1419 var->members = ralloc_array(b, nir_variable *, num_members); 1420 1421 for (unsigned i = 0; i < num_members; i++) { 1422 const struct glsl_type *mtype = interface_type->members[i]->type; 1423 if (array_length >= 0) 1424 mtype = glsl_array_type(mtype, array_length); 1425 1426 var->members[i] = rzalloc(b->shader, nir_variable); 1427 var->members[i]->name = 1428 ralloc_asprintf(var->members[i], "%s.%d", val->name, i); 1429 var->members[i]->type = mtype; 1430 var->members[i]->interface_type = 1431 interface_type->members[i]->type; 1432 var->members[i]->data.mode = nir_mode; 1433 var->members[i]->data.patch = var->patch; 1434 } 1435 } else { 1436 var->var = rzalloc(b->shader, nir_variable); 1437 var->var->name = ralloc_strdup(var->var, val->name); 1438 var->var->type = var->type->type; 1439 var->var->interface_type = interface_type->type; 1440 var->var->data.mode = nir_mode; 1441 var->var->data.patch = var->patch; 1442 } 1443 1444 /* For inputs and outputs, we need to grab locations and builtin 1445 * information from the interface type. 1446 */ 1447 vtn_foreach_decoration(b, interface_type->val, var_decoration_cb, var); 1448 break; 1449 1450 case vtn_variable_mode_param: 1451 unreachable("Not created through OpVariable"); 1452 } 1453 1454 case vtn_variable_mode_ubo: 1455 case vtn_variable_mode_ssbo: 1456 case vtn_variable_mode_push_constant: 1457 /* These don't need actual variables. */ 1458 break; 1459 } 1460 1461 if (count > 4) { 1462 assert(count == 5); 1463 nir_constant *constant = 1464 vtn_value(b, w[4], vtn_value_type_constant)->constant; 1465 var->var->constant_initializer = 1466 nir_constant_clone(constant, var->var); 1467 } 1468 1469 vtn_foreach_decoration(b, val, var_decoration_cb, var); 1470 1471 if (var->mode == vtn_variable_mode_image || 1472 var->mode == vtn_variable_mode_sampler) { 1473 /* XXX: We still need the binding information in the nir_variable 1474 * for these. We should fix that. 1475 */ 1476 var->var->data.binding = var->binding; 1477 var->var->data.descriptor_set = var->descriptor_set; 1478 var->var->data.index = var->input_attachment_index; 1479 1480 if (var->mode == vtn_variable_mode_image) 1481 var->var->data.image.format = without_array->image_format; 1482 } 1483 1484 if (var->mode == vtn_variable_mode_local) { 1485 assert(var->members == NULL && var->var != NULL); 1486 nir_function_impl_add_variable(b->impl, var->var); 1487 } else if (var->var) { 1488 nir_shader_add_variable(b->shader, var->var); 1489 } else if (var->members) { 1490 unsigned count = glsl_get_length(without_array->type); 1491 for (unsigned i = 0; i < count; i++) { 1492 assert(var->members[i]->data.mode != nir_var_local); 1493 nir_shader_add_variable(b->shader, var->members[i]); 1494 } 1495 } else { 1496 assert(var->mode == vtn_variable_mode_ubo || 1497 var->mode == vtn_variable_mode_ssbo || 1498 var->mode == vtn_variable_mode_push_constant); 1499 } 1500 break; 1501 } 1502 1503 case SpvOpAccessChain: 1504 case SpvOpInBoundsAccessChain: { 1505 struct vtn_access_chain *base, *chain; 1506 struct vtn_value *base_val = vtn_untyped_value(b, w[3]); 1507 if (base_val->value_type == vtn_value_type_sampled_image) { 1508 /* This is rather insane. SPIR-V allows you to use OpSampledImage 1509 * to combine an array of images with a single sampler to get an 1510 * array of sampled images that all share the same sampler. 1511 * Fortunately, this means that we can more-or-less ignore the 1512 * sampler when crawling the access chain, but it does leave us 1513 * with this rather awkward little special-case. 1514 */ 1515 base = base_val->sampled_image->image; 1516 } else { 1517 assert(base_val->value_type == vtn_value_type_access_chain); 1518 base = base_val->access_chain; 1519 } 1520 1521 chain = vtn_access_chain_extend(b, base, count - 4); 1522 1523 unsigned idx = base->length; 1524 for (int i = 4; i < count; i++) { 1525 struct vtn_value *link_val = vtn_untyped_value(b, w[i]); 1526 if (link_val->value_type == vtn_value_type_constant) { 1527 chain->link[idx].mode = vtn_access_mode_literal; 1528 chain->link[idx].id = link_val->constant->values[0].u32[0]; 1529 } else { 1530 chain->link[idx].mode = vtn_access_mode_id; 1531 chain->link[idx].id = w[i]; 1532 } 1533 idx++; 1534 } 1535 1536 if (base_val->value_type == vtn_value_type_sampled_image) { 1537 struct vtn_value *val = 1538 vtn_push_value(b, w[2], vtn_value_type_sampled_image); 1539 val->sampled_image = ralloc(b, struct vtn_sampled_image); 1540 val->sampled_image->image = chain; 1541 val->sampled_image->sampler = base_val->sampled_image->sampler; 1542 } else { 1543 struct vtn_value *val = 1544 vtn_push_value(b, w[2], vtn_value_type_access_chain); 1545 val->access_chain = chain; 1546 } 1547 break; 1548 } 1549 1550 case SpvOpCopyMemory: { 1551 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_access_chain); 1552 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_access_chain); 1553 1554 vtn_variable_copy(b, dest->access_chain, src->access_chain); 1555 break; 1556 } 1557 1558 case SpvOpLoad: { 1559 struct vtn_access_chain *src = 1560 vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain; 1561 1562 if (src->var->mode == vtn_variable_mode_image || 1563 src->var->mode == vtn_variable_mode_sampler) { 1564 vtn_push_value(b, w[2], vtn_value_type_access_chain)->access_chain = src; 1565 return; 1566 } 1567 1568 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); 1569 val->ssa = vtn_variable_load(b, src); 1570 break; 1571 } 1572 1573 case SpvOpStore: { 1574 struct vtn_access_chain *dest = 1575 vtn_value(b, w[1], vtn_value_type_access_chain)->access_chain; 1576 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); 1577 vtn_variable_store(b, src, dest); 1578 break; 1579 } 1580 1581 case SpvOpArrayLength: { 1582 struct vtn_access_chain *chain = 1583 vtn_value(b, w[3], vtn_value_type_access_chain)->access_chain; 1584 1585 const uint32_t offset = chain->var->type->offsets[w[4]]; 1586 const uint32_t stride = chain->var->type->members[w[4]]->stride; 1587 1588 unsigned chain_idx; 1589 struct vtn_type *type; 1590 nir_ssa_def *index = 1591 get_vulkan_resource_index(b, chain, &type, &chain_idx); 1592 1593 nir_intrinsic_instr *instr = 1594 nir_intrinsic_instr_create(b->nb.shader, 1595 nir_intrinsic_get_buffer_size); 1596 instr->src[0] = nir_src_for_ssa(index); 1597 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 1598 nir_builder_instr_insert(&b->nb, &instr->instr); 1599 nir_ssa_def *buf_size = &instr->dest.ssa; 1600 1601 /* array_length = max(buffer_size - offset, 0) / stride */ 1602 nir_ssa_def *array_length = 1603 nir_idiv(&b->nb, 1604 nir_imax(&b->nb, 1605 nir_isub(&b->nb, 1606 buf_size, 1607 nir_imm_int(&b->nb, offset)), 1608 nir_imm_int(&b->nb, 0u)), 1609 nir_imm_int(&b->nb, stride)); 1610 1611 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); 1612 val->ssa = vtn_create_ssa_value(b, glsl_uint_type()); 1613 val->ssa->def = array_length; 1614 break; 1615 } 1616 1617 case SpvOpCopyMemorySized: 1618 default: 1619 unreachable("Unhandled opcode"); 1620 } 1621 } 1622
