1 /* 2 * Copyright 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Jason Ekstrand (jason (at) jlekstrand.net) 25 * 26 */ 27 28 #include "vtn_private.h" 29 #include "spirv_info.h" 30 31 static struct vtn_access_chain * 32 vtn_access_chain_create(struct vtn_builder *b, unsigned length) 33 { 34 struct vtn_access_chain *chain; 35 36 /* Subtract 1 from the length since there's already one built in */ 37 size_t size = sizeof(*chain) + 38 (MAX2(length, 1) - 1) * sizeof(chain->link[0]); 39 chain = rzalloc_size(b, size); 40 chain->length = length; 41 42 return chain; 43 } 44 45 static struct vtn_access_chain * 46 vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old, 47 unsigned new_ids) 48 { 49 struct vtn_access_chain *chain; 50 51 unsigned old_len = old ? old->length : 0; 52 chain = vtn_access_chain_create(b, old_len + new_ids); 53 54 for (unsigned i = 0; i < old_len; i++) 55 chain->link[i] = old->link[i]; 56 57 return chain; 58 } 59 60 static bool 61 vtn_pointer_uses_ssa_offset(struct vtn_builder *b, 62 struct vtn_pointer *ptr) 63 { 64 return ptr->mode == vtn_variable_mode_ubo || 65 ptr->mode == vtn_variable_mode_ssbo || 66 (ptr->mode == vtn_variable_mode_workgroup && 67 b->options->lower_workgroup_access_to_offsets); 68 } 69 70 static bool 71 vtn_pointer_is_external_block(struct vtn_builder *b, 72 struct vtn_pointer *ptr) 73 { 74 return ptr->mode == vtn_variable_mode_ssbo || 75 ptr->mode == vtn_variable_mode_ubo || 76 ptr->mode == vtn_variable_mode_push_constant || 77 (ptr->mode == vtn_variable_mode_workgroup && 78 b->options->lower_workgroup_access_to_offsets); 79 } 80 81 /* Dereference the given base pointer by the access chain */ 82 static struct vtn_pointer * 83 vtn_access_chain_pointer_dereference(struct vtn_builder *b, 84 struct vtn_pointer *base, 85 struct vtn_access_chain *deref_chain) 86 { 87 struct vtn_access_chain *chain = 88 vtn_access_chain_extend(b, base->chain, deref_chain->length); 89 struct vtn_type *type = base->type; 90 91 /* OpPtrAccessChain is only allowed on things which support variable 92 * pointers. For everything else, the client is expected to just pass us 93 * the right access chain. 94 */ 95 vtn_assert(!deref_chain->ptr_as_array); 96 97 unsigned start = base->chain ? base->chain->length : 0; 98 for (unsigned i = 0; i < deref_chain->length; i++) { 99 chain->link[start + i] = deref_chain->link[i]; 100 101 if (glsl_type_is_struct(type->type)) { 102 vtn_assert(deref_chain->link[i].mode == vtn_access_mode_literal); 103 type = type->members[deref_chain->link[i].id]; 104 } else { 105 type = type->array_element; 106 } 107 } 108 109 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 110 ptr->mode = base->mode; 111 ptr->type = type; 112 ptr->var = base->var; 113 ptr->chain = chain; 114 115 return ptr; 116 } 117 118 static nir_ssa_def * 119 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link, 120 unsigned stride) 121 { 122 vtn_assert(stride > 0); 123 if (link.mode == vtn_access_mode_literal) { 124 return nir_imm_int(&b->nb, link.id * stride); 125 } else if (stride == 1) { 126 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def; 127 if (ssa->bit_size != 32) 128 ssa = nir_u2u32(&b->nb, ssa); 129 return ssa; 130 } else { 131 nir_ssa_def *src0 = vtn_ssa_value(b, link.id)->def; 132 if (src0->bit_size != 32) 133 src0 = nir_u2u32(&b->nb, src0); 134 return nir_imul(&b->nb, src0, nir_imm_int(&b->nb, stride)); 135 } 136 } 137 138 static nir_ssa_def * 139 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var, 140 nir_ssa_def *desc_array_index) 141 { 142 if (!desc_array_index) { 143 vtn_assert(glsl_type_is_struct(var->type->type)); 144 desc_array_index = nir_imm_int(&b->nb, 0); 145 } 146 147 nir_intrinsic_instr *instr = 148 nir_intrinsic_instr_create(b->nb.shader, 149 nir_intrinsic_vulkan_resource_index); 150 instr->src[0] = nir_src_for_ssa(desc_array_index); 151 nir_intrinsic_set_desc_set(instr, var->descriptor_set); 152 nir_intrinsic_set_binding(instr, var->binding); 153 154 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 155 nir_builder_instr_insert(&b->nb, &instr->instr); 156 157 return &instr->dest.ssa; 158 } 159 160 static nir_ssa_def * 161 vtn_resource_reindex(struct vtn_builder *b, nir_ssa_def *base_index, 162 nir_ssa_def *offset_index) 163 { 164 nir_intrinsic_instr *instr = 165 nir_intrinsic_instr_create(b->nb.shader, 166 nir_intrinsic_vulkan_resource_reindex); 167 instr->src[0] = nir_src_for_ssa(base_index); 168 instr->src[1] = nir_src_for_ssa(offset_index); 169 170 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 171 nir_builder_instr_insert(&b->nb, &instr->instr); 172 173 return &instr->dest.ssa; 174 } 175 176 static struct vtn_pointer * 177 vtn_ssa_offset_pointer_dereference(struct vtn_builder *b, 178 struct vtn_pointer *base, 179 struct vtn_access_chain *deref_chain) 180 { 181 nir_ssa_def *block_index = base->block_index; 182 nir_ssa_def *offset = base->offset; 183 struct vtn_type *type = base->type; 184 185 unsigned idx = 0; 186 if (base->mode == vtn_variable_mode_ubo || 187 base->mode == vtn_variable_mode_ssbo) { 188 if (!block_index) { 189 vtn_assert(base->var && base->type); 190 nir_ssa_def *desc_arr_idx; 191 if (glsl_type_is_array(type->type)) { 192 if (deref_chain->length >= 1) { 193 desc_arr_idx = 194 vtn_access_link_as_ssa(b, deref_chain->link[0], 1); 195 idx++; 196 /* This consumes a level of type */ 197 type = type->array_element; 198 } else { 199 /* This is annoying. We've been asked for a pointer to the 200 * array of UBOs/SSBOs and not a specifc buffer. Return a 201 * pointer with a descriptor index of 0 and we'll have to do 202 * a reindex later to adjust it to the right thing. 203 */ 204 desc_arr_idx = nir_imm_int(&b->nb, 0); 205 } 206 } else if (deref_chain->ptr_as_array) { 207 /* You can't have a zero-length OpPtrAccessChain */ 208 vtn_assert(deref_chain->length >= 1); 209 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[0], 1); 210 } else { 211 /* We have a regular non-array SSBO. */ 212 desc_arr_idx = NULL; 213 } 214 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx); 215 } else if (deref_chain->ptr_as_array && 216 type->base_type == vtn_base_type_struct && type->block) { 217 /* We are doing an OpPtrAccessChain on a pointer to a struct that is 218 * decorated block. This is an interesting corner in the SPIR-V 219 * spec. One interpretation would be that they client is clearly 220 * trying to treat that block as if it's an implicit array of blocks 221 * repeated in the buffer. However, the SPIR-V spec for the 222 * OpPtrAccessChain says: 223 * 224 * "Base is treated as the address of the first element of an 225 * array, and the Element elements address is computed to be the 226 * base for the Indexes, as per OpAccessChain." 227 * 228 * Taken literally, that would mean that your struct type is supposed 229 * to be treated as an array of such a struct and, since it's 230 * decorated block, that means an array of blocks which corresponds 231 * to an array descriptor. Therefore, we need to do a reindex 232 * operation to add the index from the first link in the access chain 233 * to the index we recieved. 234 * 235 * The downside to this interpretation (there always is one) is that 236 * this might be somewhat surprising behavior to apps if they expect 237 * the implicit array behavior described above. 238 */ 239 vtn_assert(deref_chain->length >= 1); 240 nir_ssa_def *offset_index = 241 vtn_access_link_as_ssa(b, deref_chain->link[0], 1); 242 idx++; 243 244 block_index = vtn_resource_reindex(b, block_index, offset_index); 245 } 246 } 247 248 if (!offset) { 249 if (base->mode == vtn_variable_mode_workgroup) { 250 /* SLM doesn't need nor have a block index */ 251 vtn_assert(!block_index); 252 253 /* We need the variable for the base offset */ 254 vtn_assert(base->var); 255 256 /* We need ptr_type for size and alignment */ 257 vtn_assert(base->ptr_type); 258 259 /* Assign location on first use so that we don't end up bloating SLM 260 * address space for variables which are never statically used. 261 */ 262 if (base->var->shared_location < 0) { 263 vtn_assert(base->ptr_type->length > 0 && base->ptr_type->align > 0); 264 b->shader->num_shared = vtn_align_u32(b->shader->num_shared, 265 base->ptr_type->align); 266 base->var->shared_location = b->shader->num_shared; 267 b->shader->num_shared += base->ptr_type->length; 268 } 269 270 offset = nir_imm_int(&b->nb, base->var->shared_location); 271 } else { 272 /* The code above should have ensured a block_index when needed. */ 273 vtn_assert(block_index); 274 275 /* Start off with at the start of the buffer. */ 276 offset = nir_imm_int(&b->nb, 0); 277 } 278 } 279 280 if (deref_chain->ptr_as_array && idx == 0) { 281 /* We need ptr_type for the stride */ 282 vtn_assert(base->ptr_type); 283 284 /* We need at least one element in the chain */ 285 vtn_assert(deref_chain->length >= 1); 286 287 nir_ssa_def *elem_offset = 288 vtn_access_link_as_ssa(b, deref_chain->link[idx], 289 base->ptr_type->stride); 290 offset = nir_iadd(&b->nb, offset, elem_offset); 291 idx++; 292 } 293 294 for (; idx < deref_chain->length; idx++) { 295 switch (glsl_get_base_type(type->type)) { 296 case GLSL_TYPE_UINT: 297 case GLSL_TYPE_INT: 298 case GLSL_TYPE_UINT16: 299 case GLSL_TYPE_INT16: 300 case GLSL_TYPE_UINT64: 301 case GLSL_TYPE_INT64: 302 case GLSL_TYPE_FLOAT: 303 case GLSL_TYPE_FLOAT16: 304 case GLSL_TYPE_DOUBLE: 305 case GLSL_TYPE_BOOL: 306 case GLSL_TYPE_ARRAY: { 307 nir_ssa_def *elem_offset = 308 vtn_access_link_as_ssa(b, deref_chain->link[idx], type->stride); 309 offset = nir_iadd(&b->nb, offset, elem_offset); 310 type = type->array_element; 311 break; 312 } 313 314 case GLSL_TYPE_STRUCT: { 315 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal); 316 unsigned member = deref_chain->link[idx].id; 317 nir_ssa_def *mem_offset = nir_imm_int(&b->nb, type->offsets[member]); 318 offset = nir_iadd(&b->nb, offset, mem_offset); 319 type = type->members[member]; 320 break; 321 } 322 323 default: 324 vtn_fail("Invalid type for deref"); 325 } 326 } 327 328 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 329 ptr->mode = base->mode; 330 ptr->type = type; 331 ptr->block_index = block_index; 332 ptr->offset = offset; 333 334 return ptr; 335 } 336 337 /* Dereference the given base pointer by the access chain */ 338 static struct vtn_pointer * 339 vtn_pointer_dereference(struct vtn_builder *b, 340 struct vtn_pointer *base, 341 struct vtn_access_chain *deref_chain) 342 { 343 if (vtn_pointer_uses_ssa_offset(b, base)) { 344 return vtn_ssa_offset_pointer_dereference(b, base, deref_chain); 345 } else { 346 return vtn_access_chain_pointer_dereference(b, base, deref_chain); 347 } 348 } 349 350 /* Crawls a chain of array derefs and rewrites the types so that the 351 * lengths stay the same but the terminal type is the one given by 352 * tail_type. This is useful for split structures. 353 */ 354 static void 355 rewrite_deref_types(struct vtn_builder *b, nir_deref *deref, 356 const struct glsl_type *type) 357 { 358 deref->type = type; 359 if (deref->child) { 360 vtn_assert(deref->child->deref_type == nir_deref_type_array); 361 vtn_assert(glsl_type_is_array(deref->type)); 362 rewrite_deref_types(b, deref->child, glsl_get_array_element(type)); 363 } 364 } 365 366 struct vtn_pointer * 367 vtn_pointer_for_variable(struct vtn_builder *b, 368 struct vtn_variable *var, struct vtn_type *ptr_type) 369 { 370 struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer); 371 372 pointer->mode = var->mode; 373 pointer->type = var->type; 374 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 375 vtn_assert(ptr_type->deref->type == var->type->type); 376 pointer->ptr_type = ptr_type; 377 pointer->var = var; 378 379 return pointer; 380 } 381 382 nir_deref_var * 383 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr) 384 { 385 /* Do on-the-fly copy propagation for samplers. */ 386 if (ptr->var->copy_prop_sampler) 387 return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler); 388 389 nir_deref_var *deref_var; 390 if (ptr->var->var) { 391 deref_var = nir_deref_var_create(b, ptr->var->var); 392 /* Raw variable access */ 393 if (!ptr->chain) 394 return deref_var; 395 } else { 396 vtn_assert(ptr->var->members); 397 /* Create the deref_var manually. It will get filled out later. */ 398 deref_var = rzalloc(b, nir_deref_var); 399 deref_var->deref.deref_type = nir_deref_type_var; 400 } 401 402 struct vtn_access_chain *chain = ptr->chain; 403 vtn_assert(chain); 404 405 struct vtn_type *deref_type = ptr->var->type; 406 nir_deref *tail = &deref_var->deref; 407 nir_variable **members = ptr->var->members; 408 409 for (unsigned i = 0; i < chain->length; i++) { 410 enum glsl_base_type base_type = glsl_get_base_type(deref_type->type); 411 switch (base_type) { 412 case GLSL_TYPE_UINT: 413 case GLSL_TYPE_INT: 414 case GLSL_TYPE_UINT16: 415 case GLSL_TYPE_INT16: 416 case GLSL_TYPE_UINT64: 417 case GLSL_TYPE_INT64: 418 case GLSL_TYPE_FLOAT: 419 case GLSL_TYPE_FLOAT16: 420 case GLSL_TYPE_DOUBLE: 421 case GLSL_TYPE_BOOL: 422 case GLSL_TYPE_ARRAY: { 423 deref_type = deref_type->array_element; 424 425 nir_deref_array *deref_arr = nir_deref_array_create(b); 426 deref_arr->deref.type = deref_type->type; 427 428 if (chain->link[i].mode == vtn_access_mode_literal) { 429 deref_arr->deref_array_type = nir_deref_array_type_direct; 430 deref_arr->base_offset = chain->link[i].id; 431 } else { 432 vtn_assert(chain->link[i].mode == vtn_access_mode_id); 433 deref_arr->deref_array_type = nir_deref_array_type_indirect; 434 deref_arr->base_offset = 0; 435 deref_arr->indirect = 436 nir_src_for_ssa(vtn_ssa_value(b, chain->link[i].id)->def); 437 } 438 tail->child = &deref_arr->deref; 439 tail = tail->child; 440 break; 441 } 442 443 case GLSL_TYPE_STRUCT: { 444 vtn_assert(chain->link[i].mode == vtn_access_mode_literal); 445 unsigned idx = chain->link[i].id; 446 deref_type = deref_type->members[idx]; 447 if (members) { 448 /* This is a pre-split structure. */ 449 deref_var->var = members[idx]; 450 rewrite_deref_types(b, &deref_var->deref, members[idx]->type); 451 vtn_assert(tail->type == deref_type->type); 452 members = NULL; 453 } else { 454 nir_deref_struct *deref_struct = nir_deref_struct_create(b, idx); 455 deref_struct->deref.type = deref_type->type; 456 tail->child = &deref_struct->deref; 457 tail = tail->child; 458 } 459 break; 460 } 461 default: 462 vtn_fail("Invalid type for deref"); 463 } 464 } 465 466 vtn_assert(members == NULL); 467 return deref_var; 468 } 469 470 static void 471 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_var *deref, 472 nir_deref *tail, struct vtn_ssa_value *inout) 473 { 474 /* The deref tail may contain a deref to select a component of a vector (in 475 * other words, it might not be an actual tail) so we have to save it away 476 * here since we overwrite it later. 477 */ 478 nir_deref *old_child = tail->child; 479 480 if (glsl_type_is_vector_or_scalar(tail->type)) { 481 /* Terminate the deref chain in case there is one more link to pick 482 * off a component of the vector. 483 */ 484 tail->child = NULL; 485 486 nir_intrinsic_op op = load ? nir_intrinsic_load_var : 487 nir_intrinsic_store_var; 488 489 nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(b->shader, op); 490 intrin->variables[0] = nir_deref_var_clone(deref, intrin); 491 intrin->num_components = glsl_get_vector_elements(tail->type); 492 493 if (load) { 494 nir_ssa_dest_init(&intrin->instr, &intrin->dest, 495 intrin->num_components, 496 glsl_get_bit_size(tail->type), 497 NULL); 498 inout->def = &intrin->dest.ssa; 499 } else { 500 nir_intrinsic_set_write_mask(intrin, (1 << intrin->num_components) - 1); 501 intrin->src[0] = nir_src_for_ssa(inout->def); 502 } 503 504 nir_builder_instr_insert(&b->nb, &intrin->instr); 505 } else if (glsl_get_base_type(tail->type) == GLSL_TYPE_ARRAY || 506 glsl_type_is_matrix(tail->type)) { 507 unsigned elems = glsl_get_length(tail->type); 508 nir_deref_array *deref_arr = nir_deref_array_create(b); 509 deref_arr->deref_array_type = nir_deref_array_type_direct; 510 deref_arr->deref.type = glsl_get_array_element(tail->type); 511 tail->child = &deref_arr->deref; 512 for (unsigned i = 0; i < elems; i++) { 513 deref_arr->base_offset = i; 514 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]); 515 } 516 } else { 517 vtn_assert(glsl_get_base_type(tail->type) == GLSL_TYPE_STRUCT); 518 unsigned elems = glsl_get_length(tail->type); 519 nir_deref_struct *deref_struct = nir_deref_struct_create(b, 0); 520 tail->child = &deref_struct->deref; 521 for (unsigned i = 0; i < elems; i++) { 522 deref_struct->index = i; 523 deref_struct->deref.type = glsl_get_struct_field(tail->type, i); 524 _vtn_local_load_store(b, load, deref, tail->child, inout->elems[i]); 525 } 526 } 527 528 tail->child = old_child; 529 } 530 531 nir_deref_var * 532 vtn_nir_deref(struct vtn_builder *b, uint32_t id) 533 { 534 struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer; 535 return vtn_pointer_to_deref(b, ptr); 536 } 537 538 /* 539 * Gets the NIR-level deref tail, which may have as a child an array deref 540 * selecting which component due to OpAccessChain supporting per-component 541 * indexing in SPIR-V. 542 */ 543 static nir_deref * 544 get_deref_tail(nir_deref_var *deref) 545 { 546 nir_deref *cur = &deref->deref; 547 while (!glsl_type_is_vector_or_scalar(cur->type) && cur->child) 548 cur = cur->child; 549 550 return cur; 551 } 552 553 struct vtn_ssa_value * 554 vtn_local_load(struct vtn_builder *b, nir_deref_var *src) 555 { 556 nir_deref *src_tail = get_deref_tail(src); 557 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type); 558 _vtn_local_load_store(b, true, src, src_tail, val); 559 560 if (src_tail->child) { 561 nir_deref_array *vec_deref = nir_deref_as_array(src_tail->child); 562 vtn_assert(vec_deref->deref.child == NULL); 563 val->type = vec_deref->deref.type; 564 if (vec_deref->deref_array_type == nir_deref_array_type_direct) 565 val->def = vtn_vector_extract(b, val->def, vec_deref->base_offset); 566 else 567 val->def = vtn_vector_extract_dynamic(b, val->def, 568 vec_deref->indirect.ssa); 569 } 570 571 return val; 572 } 573 574 void 575 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, 576 nir_deref_var *dest) 577 { 578 nir_deref *dest_tail = get_deref_tail(dest); 579 580 if (dest_tail->child) { 581 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); 582 _vtn_local_load_store(b, true, dest, dest_tail, val); 583 nir_deref_array *deref = nir_deref_as_array(dest_tail->child); 584 vtn_assert(deref->deref.child == NULL); 585 if (deref->deref_array_type == nir_deref_array_type_direct) 586 val->def = vtn_vector_insert(b, val->def, src->def, 587 deref->base_offset); 588 else 589 val->def = vtn_vector_insert_dynamic(b, val->def, src->def, 590 deref->indirect.ssa); 591 _vtn_local_load_store(b, false, dest, dest_tail, val); 592 } else { 593 _vtn_local_load_store(b, false, dest, dest_tail, src); 594 } 595 } 596 597 nir_ssa_def * 598 vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr, 599 nir_ssa_def **index_out, unsigned *end_idx_out) 600 { 601 if (vtn_pointer_uses_ssa_offset(b, ptr)) { 602 if (!ptr->offset) { 603 struct vtn_access_chain chain = { 604 .length = 0, 605 }; 606 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 607 } 608 *index_out = ptr->block_index; 609 return ptr->offset; 610 } 611 612 vtn_assert(ptr->mode == vtn_variable_mode_push_constant); 613 *index_out = NULL; 614 615 unsigned idx = 0; 616 struct vtn_type *type = ptr->var->type; 617 nir_ssa_def *offset = nir_imm_int(&b->nb, 0); 618 619 if (ptr->chain) { 620 for (; idx < ptr->chain->length; idx++) { 621 enum glsl_base_type base_type = glsl_get_base_type(type->type); 622 switch (base_type) { 623 case GLSL_TYPE_UINT: 624 case GLSL_TYPE_INT: 625 case GLSL_TYPE_UINT16: 626 case GLSL_TYPE_INT16: 627 case GLSL_TYPE_UINT64: 628 case GLSL_TYPE_INT64: 629 case GLSL_TYPE_FLOAT: 630 case GLSL_TYPE_FLOAT16: 631 case GLSL_TYPE_DOUBLE: 632 case GLSL_TYPE_BOOL: 633 case GLSL_TYPE_ARRAY: 634 offset = nir_iadd(&b->nb, offset, 635 vtn_access_link_as_ssa(b, ptr->chain->link[idx], 636 type->stride)); 637 638 type = type->array_element; 639 break; 640 641 case GLSL_TYPE_STRUCT: { 642 vtn_assert(ptr->chain->link[idx].mode == vtn_access_mode_literal); 643 unsigned member = ptr->chain->link[idx].id; 644 offset = nir_iadd(&b->nb, offset, 645 nir_imm_int(&b->nb, type->offsets[member])); 646 type = type->members[member]; 647 break; 648 } 649 650 default: 651 vtn_fail("Invalid type for deref"); 652 } 653 } 654 } 655 656 vtn_assert(type == ptr->type); 657 if (end_idx_out) 658 *end_idx_out = idx; 659 660 return offset; 661 } 662 663 /* Tries to compute the size of an interface block based on the strides and 664 * offsets that are provided to us in the SPIR-V source. 665 */ 666 static unsigned 667 vtn_type_block_size(struct vtn_builder *b, struct vtn_type *type) 668 { 669 enum glsl_base_type base_type = glsl_get_base_type(type->type); 670 switch (base_type) { 671 case GLSL_TYPE_UINT: 672 case GLSL_TYPE_INT: 673 case GLSL_TYPE_UINT16: 674 case GLSL_TYPE_INT16: 675 case GLSL_TYPE_UINT64: 676 case GLSL_TYPE_INT64: 677 case GLSL_TYPE_FLOAT: 678 case GLSL_TYPE_FLOAT16: 679 case GLSL_TYPE_BOOL: 680 case GLSL_TYPE_DOUBLE: { 681 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) : 682 glsl_get_matrix_columns(type->type); 683 if (cols > 1) { 684 vtn_assert(type->stride > 0); 685 return type->stride * cols; 686 } else if (base_type == GLSL_TYPE_DOUBLE || 687 base_type == GLSL_TYPE_UINT64 || 688 base_type == GLSL_TYPE_INT64) { 689 return glsl_get_vector_elements(type->type) * 8; 690 } else { 691 return glsl_get_vector_elements(type->type) * 4; 692 } 693 } 694 695 case GLSL_TYPE_STRUCT: 696 case GLSL_TYPE_INTERFACE: { 697 unsigned size = 0; 698 unsigned num_fields = glsl_get_length(type->type); 699 for (unsigned f = 0; f < num_fields; f++) { 700 unsigned field_end = type->offsets[f] + 701 vtn_type_block_size(b, type->members[f]); 702 size = MAX2(size, field_end); 703 } 704 return size; 705 } 706 707 case GLSL_TYPE_ARRAY: 708 vtn_assert(type->stride > 0); 709 vtn_assert(glsl_get_length(type->type) > 0); 710 return type->stride * glsl_get_length(type->type); 711 712 default: 713 vtn_fail("Invalid block type"); 714 return 0; 715 } 716 } 717 718 static void 719 vtn_access_chain_get_offset_size(struct vtn_builder *b, 720 struct vtn_access_chain *chain, 721 struct vtn_type *type, 722 unsigned *access_offset, 723 unsigned *access_size) 724 { 725 *access_offset = 0; 726 727 for (unsigned i = 0; i < chain->length; i++) { 728 if (chain->link[i].mode != vtn_access_mode_literal) 729 break; 730 731 if (glsl_type_is_struct(type->type)) { 732 *access_offset += type->offsets[chain->link[i].id]; 733 type = type->members[chain->link[i].id]; 734 } else { 735 *access_offset += type->stride * chain->link[i].id; 736 type = type->array_element; 737 } 738 } 739 740 *access_size = vtn_type_block_size(b, type); 741 } 742 743 static void 744 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load, 745 nir_ssa_def *index, nir_ssa_def *offset, 746 unsigned access_offset, unsigned access_size, 747 struct vtn_ssa_value **inout, const struct glsl_type *type) 748 { 749 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op); 750 instr->num_components = glsl_get_vector_elements(type); 751 752 int src = 0; 753 if (!load) { 754 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1); 755 instr->src[src++] = nir_src_for_ssa((*inout)->def); 756 } 757 758 if (op == nir_intrinsic_load_push_constant) { 759 vtn_assert(access_offset % 4 == 0); 760 761 nir_intrinsic_set_base(instr, access_offset); 762 nir_intrinsic_set_range(instr, access_size); 763 } 764 765 if (index) 766 instr->src[src++] = nir_src_for_ssa(index); 767 768 if (op == nir_intrinsic_load_push_constant) { 769 /* We need to subtract the offset from where the intrinsic will load the 770 * data. */ 771 instr->src[src++] = 772 nir_src_for_ssa(nir_isub(&b->nb, offset, 773 nir_imm_int(&b->nb, access_offset))); 774 } else { 775 instr->src[src++] = nir_src_for_ssa(offset); 776 } 777 778 if (load) { 779 nir_ssa_dest_init(&instr->instr, &instr->dest, 780 instr->num_components, 781 glsl_get_bit_size(type), NULL); 782 (*inout)->def = &instr->dest.ssa; 783 } 784 785 nir_builder_instr_insert(&b->nb, &instr->instr); 786 787 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL) 788 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0)); 789 } 790 791 static void 792 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load, 793 nir_ssa_def *index, nir_ssa_def *offset, 794 unsigned access_offset, unsigned access_size, 795 struct vtn_access_chain *chain, unsigned chain_idx, 796 struct vtn_type *type, struct vtn_ssa_value **inout) 797 { 798 if (chain && chain_idx >= chain->length) 799 chain = NULL; 800 801 if (load && chain == NULL && *inout == NULL) 802 *inout = vtn_create_ssa_value(b, type->type); 803 804 enum glsl_base_type base_type = glsl_get_base_type(type->type); 805 switch (base_type) { 806 case GLSL_TYPE_UINT: 807 case GLSL_TYPE_INT: 808 case GLSL_TYPE_UINT16: 809 case GLSL_TYPE_INT16: 810 case GLSL_TYPE_UINT64: 811 case GLSL_TYPE_INT64: 812 case GLSL_TYPE_FLOAT: 813 case GLSL_TYPE_FLOAT16: 814 case GLSL_TYPE_DOUBLE: 815 case GLSL_TYPE_BOOL: 816 /* This is where things get interesting. At this point, we've hit 817 * a vector, a scalar, or a matrix. 818 */ 819 if (glsl_type_is_matrix(type->type)) { 820 /* Loading the whole matrix */ 821 struct vtn_ssa_value *transpose; 822 unsigned num_ops, vec_width, col_stride; 823 if (type->row_major) { 824 num_ops = glsl_get_vector_elements(type->type); 825 vec_width = glsl_get_matrix_columns(type->type); 826 col_stride = type->array_element->stride; 827 if (load) { 828 const struct glsl_type *transpose_type = 829 glsl_matrix_type(base_type, vec_width, num_ops); 830 *inout = vtn_create_ssa_value(b, transpose_type); 831 } else { 832 transpose = vtn_ssa_transpose(b, *inout); 833 inout = &transpose; 834 } 835 } else { 836 num_ops = glsl_get_matrix_columns(type->type); 837 vec_width = glsl_get_vector_elements(type->type); 838 col_stride = type->stride; 839 } 840 841 for (unsigned i = 0; i < num_ops; i++) { 842 nir_ssa_def *elem_offset = 843 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * col_stride)); 844 _vtn_load_store_tail(b, op, load, index, elem_offset, 845 access_offset, access_size, 846 &(*inout)->elems[i], 847 glsl_vector_type(base_type, vec_width)); 848 } 849 850 if (load && type->row_major) 851 *inout = vtn_ssa_transpose(b, *inout); 852 } else { 853 unsigned elems = glsl_get_vector_elements(type->type); 854 unsigned type_size = glsl_get_bit_size(type->type) / 8; 855 if (elems == 1 || type->stride == type_size) { 856 /* This is a tightly-packed normal scalar or vector load */ 857 vtn_assert(glsl_type_is_vector_or_scalar(type->type)); 858 _vtn_load_store_tail(b, op, load, index, offset, 859 access_offset, access_size, 860 inout, type->type); 861 } else { 862 /* This is a strided load. We have to load N things separately. 863 * This is the single column of a row-major matrix case. 864 */ 865 vtn_assert(type->stride > type_size); 866 vtn_assert(type->stride % type_size == 0); 867 868 nir_ssa_def *per_comp[4]; 869 for (unsigned i = 0; i < elems; i++) { 870 nir_ssa_def *elem_offset = 871 nir_iadd(&b->nb, offset, 872 nir_imm_int(&b->nb, i * type->stride)); 873 struct vtn_ssa_value *comp, temp_val; 874 if (!load) { 875 temp_val.def = nir_channel(&b->nb, (*inout)->def, i); 876 temp_val.type = glsl_scalar_type(base_type); 877 } 878 comp = &temp_val; 879 _vtn_load_store_tail(b, op, load, index, elem_offset, 880 access_offset, access_size, 881 &comp, glsl_scalar_type(base_type)); 882 per_comp[i] = comp->def; 883 } 884 885 if (load) { 886 if (*inout == NULL) 887 *inout = vtn_create_ssa_value(b, type->type); 888 (*inout)->def = nir_vec(&b->nb, per_comp, elems); 889 } 890 } 891 } 892 return; 893 894 case GLSL_TYPE_ARRAY: { 895 unsigned elems = glsl_get_length(type->type); 896 for (unsigned i = 0; i < elems; i++) { 897 nir_ssa_def *elem_off = 898 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride)); 899 _vtn_block_load_store(b, op, load, index, elem_off, 900 access_offset, access_size, 901 NULL, 0, 902 type->array_element, &(*inout)->elems[i]); 903 } 904 return; 905 } 906 907 case GLSL_TYPE_STRUCT: { 908 unsigned elems = glsl_get_length(type->type); 909 for (unsigned i = 0; i < elems; i++) { 910 nir_ssa_def *elem_off = 911 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i])); 912 _vtn_block_load_store(b, op, load, index, elem_off, 913 access_offset, access_size, 914 NULL, 0, 915 type->members[i], &(*inout)->elems[i]); 916 } 917 return; 918 } 919 920 default: 921 vtn_fail("Invalid block member type"); 922 } 923 } 924 925 static struct vtn_ssa_value * 926 vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src) 927 { 928 nir_intrinsic_op op; 929 unsigned access_offset = 0, access_size = 0; 930 switch (src->mode) { 931 case vtn_variable_mode_ubo: 932 op = nir_intrinsic_load_ubo; 933 break; 934 case vtn_variable_mode_ssbo: 935 op = nir_intrinsic_load_ssbo; 936 break; 937 case vtn_variable_mode_push_constant: 938 op = nir_intrinsic_load_push_constant; 939 vtn_access_chain_get_offset_size(b, src->chain, src->var->type, 940 &access_offset, &access_size); 941 break; 942 case vtn_variable_mode_workgroup: 943 op = nir_intrinsic_load_shared; 944 break; 945 default: 946 vtn_fail("Invalid block variable mode"); 947 } 948 949 nir_ssa_def *offset, *index = NULL; 950 unsigned chain_idx; 951 offset = vtn_pointer_to_offset(b, src, &index, &chain_idx); 952 953 struct vtn_ssa_value *value = NULL; 954 _vtn_block_load_store(b, op, true, index, offset, 955 access_offset, access_size, 956 src->chain, chain_idx, src->type, &value); 957 return value; 958 } 959 960 static void 961 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src, 962 struct vtn_pointer *dst) 963 { 964 nir_intrinsic_op op; 965 switch (dst->mode) { 966 case vtn_variable_mode_ssbo: 967 op = nir_intrinsic_store_ssbo; 968 break; 969 case vtn_variable_mode_workgroup: 970 op = nir_intrinsic_store_shared; 971 break; 972 default: 973 vtn_fail("Invalid block variable mode"); 974 } 975 976 nir_ssa_def *offset, *index = NULL; 977 unsigned chain_idx; 978 offset = vtn_pointer_to_offset(b, dst, &index, &chain_idx); 979 980 _vtn_block_load_store(b, op, false, index, offset, 981 0, 0, dst->chain, chain_idx, dst->type, &src); 982 } 983 984 static void 985 _vtn_variable_load_store(struct vtn_builder *b, bool load, 986 struct vtn_pointer *ptr, 987 struct vtn_ssa_value **inout) 988 { 989 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type); 990 switch (base_type) { 991 case GLSL_TYPE_UINT: 992 case GLSL_TYPE_INT: 993 case GLSL_TYPE_UINT16: 994 case GLSL_TYPE_INT16: 995 case GLSL_TYPE_UINT64: 996 case GLSL_TYPE_INT64: 997 case GLSL_TYPE_FLOAT: 998 case GLSL_TYPE_FLOAT16: 999 case GLSL_TYPE_BOOL: 1000 case GLSL_TYPE_DOUBLE: 1001 /* At this point, we have a scalar, vector, or matrix so we know that 1002 * there cannot be any structure splitting still in the way. By 1003 * stopping at the matrix level rather than the vector level, we 1004 * ensure that matrices get loaded in the optimal way even if they 1005 * are storred row-major in a UBO. 1006 */ 1007 if (load) { 1008 *inout = vtn_local_load(b, vtn_pointer_to_deref(b, ptr)); 1009 } else { 1010 vtn_local_store(b, *inout, vtn_pointer_to_deref(b, ptr)); 1011 } 1012 return; 1013 1014 case GLSL_TYPE_ARRAY: 1015 case GLSL_TYPE_STRUCT: { 1016 unsigned elems = glsl_get_length(ptr->type->type); 1017 if (load) { 1018 vtn_assert(*inout == NULL); 1019 *inout = rzalloc(b, struct vtn_ssa_value); 1020 (*inout)->type = ptr->type->type; 1021 (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems); 1022 } 1023 1024 struct vtn_access_chain chain = { 1025 .length = 1, 1026 .link = { 1027 { .mode = vtn_access_mode_literal, }, 1028 } 1029 }; 1030 for (unsigned i = 0; i < elems; i++) { 1031 chain.link[0].id = i; 1032 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain); 1033 _vtn_variable_load_store(b, load, elem, &(*inout)->elems[i]); 1034 } 1035 return; 1036 } 1037 1038 default: 1039 vtn_fail("Invalid access chain type"); 1040 } 1041 } 1042 1043 struct vtn_ssa_value * 1044 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src) 1045 { 1046 if (vtn_pointer_is_external_block(b, src)) { 1047 return vtn_block_load(b, src); 1048 } else { 1049 struct vtn_ssa_value *val = NULL; 1050 _vtn_variable_load_store(b, true, src, &val); 1051 return val; 1052 } 1053 } 1054 1055 void 1056 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src, 1057 struct vtn_pointer *dest) 1058 { 1059 if (vtn_pointer_is_external_block(b, dest)) { 1060 vtn_assert(dest->mode == vtn_variable_mode_ssbo || 1061 dest->mode == vtn_variable_mode_workgroup); 1062 vtn_block_store(b, src, dest); 1063 } else { 1064 _vtn_variable_load_store(b, false, dest, &src); 1065 } 1066 } 1067 1068 static void 1069 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 1070 struct vtn_pointer *src) 1071 { 1072 vtn_assert(src->type->type == dest->type->type); 1073 enum glsl_base_type base_type = glsl_get_base_type(src->type->type); 1074 switch (base_type) { 1075 case GLSL_TYPE_UINT: 1076 case GLSL_TYPE_INT: 1077 case GLSL_TYPE_UINT16: 1078 case GLSL_TYPE_INT16: 1079 case GLSL_TYPE_UINT64: 1080 case GLSL_TYPE_INT64: 1081 case GLSL_TYPE_FLOAT: 1082 case GLSL_TYPE_FLOAT16: 1083 case GLSL_TYPE_DOUBLE: 1084 case GLSL_TYPE_BOOL: 1085 /* At this point, we have a scalar, vector, or matrix so we know that 1086 * there cannot be any structure splitting still in the way. By 1087 * stopping at the matrix level rather than the vector level, we 1088 * ensure that matrices get loaded in the optimal way even if they 1089 * are storred row-major in a UBO. 1090 */ 1091 vtn_variable_store(b, vtn_variable_load(b, src), dest); 1092 return; 1093 1094 case GLSL_TYPE_ARRAY: 1095 case GLSL_TYPE_STRUCT: { 1096 struct vtn_access_chain chain = { 1097 .length = 1, 1098 .link = { 1099 { .mode = vtn_access_mode_literal, }, 1100 } 1101 }; 1102 unsigned elems = glsl_get_length(src->type->type); 1103 for (unsigned i = 0; i < elems; i++) { 1104 chain.link[0].id = i; 1105 struct vtn_pointer *src_elem = 1106 vtn_pointer_dereference(b, src, &chain); 1107 struct vtn_pointer *dest_elem = 1108 vtn_pointer_dereference(b, dest, &chain); 1109 1110 _vtn_variable_copy(b, dest_elem, src_elem); 1111 } 1112 return; 1113 } 1114 1115 default: 1116 vtn_fail("Invalid access chain type"); 1117 } 1118 } 1119 1120 static void 1121 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 1122 struct vtn_pointer *src) 1123 { 1124 /* TODO: At some point, we should add a special-case for when we can 1125 * just emit a copy_var intrinsic. 1126 */ 1127 _vtn_variable_copy(b, dest, src); 1128 } 1129 1130 static void 1131 set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode) 1132 { 1133 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in); 1134 *mode = nir_var_system_value; 1135 } 1136 1137 static void 1138 vtn_get_builtin_location(struct vtn_builder *b, 1139 SpvBuiltIn builtin, int *location, 1140 nir_variable_mode *mode) 1141 { 1142 switch (builtin) { 1143 case SpvBuiltInPosition: 1144 *location = VARYING_SLOT_POS; 1145 break; 1146 case SpvBuiltInPointSize: 1147 *location = VARYING_SLOT_PSIZ; 1148 break; 1149 case SpvBuiltInClipDistance: 1150 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */ 1151 break; 1152 case SpvBuiltInCullDistance: 1153 *location = VARYING_SLOT_CULL_DIST0; 1154 break; 1155 case SpvBuiltInVertexIndex: 1156 *location = SYSTEM_VALUE_VERTEX_ID; 1157 set_mode_system_value(b, mode); 1158 break; 1159 case SpvBuiltInVertexId: 1160 /* Vulkan defines VertexID to be zero-based and reserves the new 1161 * builtin keyword VertexIndex to indicate the non-zero-based value. 1162 */ 1163 *location = SYSTEM_VALUE_VERTEX_ID_ZERO_BASE; 1164 set_mode_system_value(b, mode); 1165 break; 1166 case SpvBuiltInInstanceIndex: 1167 *location = SYSTEM_VALUE_INSTANCE_INDEX; 1168 set_mode_system_value(b, mode); 1169 break; 1170 case SpvBuiltInInstanceId: 1171 *location = SYSTEM_VALUE_INSTANCE_ID; 1172 set_mode_system_value(b, mode); 1173 break; 1174 case SpvBuiltInPrimitiveId: 1175 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) { 1176 vtn_assert(*mode == nir_var_shader_in); 1177 *location = VARYING_SLOT_PRIMITIVE_ID; 1178 } else if (*mode == nir_var_shader_out) { 1179 *location = VARYING_SLOT_PRIMITIVE_ID; 1180 } else { 1181 *location = SYSTEM_VALUE_PRIMITIVE_ID; 1182 set_mode_system_value(b, mode); 1183 } 1184 break; 1185 case SpvBuiltInInvocationId: 1186 *location = SYSTEM_VALUE_INVOCATION_ID; 1187 set_mode_system_value(b, mode); 1188 break; 1189 case SpvBuiltInLayer: 1190 *location = VARYING_SLOT_LAYER; 1191 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 1192 *mode = nir_var_shader_in; 1193 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 1194 *mode = nir_var_shader_out; 1195 else 1196 vtn_fail("invalid stage for SpvBuiltInLayer"); 1197 break; 1198 case SpvBuiltInViewportIndex: 1199 *location = VARYING_SLOT_VIEWPORT; 1200 if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 1201 *mode = nir_var_shader_out; 1202 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 1203 *mode = nir_var_shader_in; 1204 else 1205 vtn_fail("invalid stage for SpvBuiltInViewportIndex"); 1206 break; 1207 case SpvBuiltInTessLevelOuter: 1208 *location = VARYING_SLOT_TESS_LEVEL_OUTER; 1209 break; 1210 case SpvBuiltInTessLevelInner: 1211 *location = VARYING_SLOT_TESS_LEVEL_INNER; 1212 break; 1213 case SpvBuiltInTessCoord: 1214 *location = SYSTEM_VALUE_TESS_COORD; 1215 set_mode_system_value(b, mode); 1216 break; 1217 case SpvBuiltInPatchVertices: 1218 *location = SYSTEM_VALUE_VERTICES_IN; 1219 set_mode_system_value(b, mode); 1220 break; 1221 case SpvBuiltInFragCoord: 1222 *location = VARYING_SLOT_POS; 1223 vtn_assert(*mode == nir_var_shader_in); 1224 break; 1225 case SpvBuiltInPointCoord: 1226 *location = VARYING_SLOT_PNTC; 1227 vtn_assert(*mode == nir_var_shader_in); 1228 break; 1229 case SpvBuiltInFrontFacing: 1230 *location = SYSTEM_VALUE_FRONT_FACE; 1231 set_mode_system_value(b, mode); 1232 break; 1233 case SpvBuiltInSampleId: 1234 *location = SYSTEM_VALUE_SAMPLE_ID; 1235 set_mode_system_value(b, mode); 1236 break; 1237 case SpvBuiltInSamplePosition: 1238 *location = SYSTEM_VALUE_SAMPLE_POS; 1239 set_mode_system_value(b, mode); 1240 break; 1241 case SpvBuiltInSampleMask: 1242 if (*mode == nir_var_shader_out) { 1243 *location = FRAG_RESULT_SAMPLE_MASK; 1244 } else { 1245 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; 1246 set_mode_system_value(b, mode); 1247 } 1248 break; 1249 case SpvBuiltInFragDepth: 1250 *location = FRAG_RESULT_DEPTH; 1251 vtn_assert(*mode == nir_var_shader_out); 1252 break; 1253 case SpvBuiltInHelperInvocation: 1254 *location = SYSTEM_VALUE_HELPER_INVOCATION; 1255 set_mode_system_value(b, mode); 1256 break; 1257 case SpvBuiltInNumWorkgroups: 1258 *location = SYSTEM_VALUE_NUM_WORK_GROUPS; 1259 set_mode_system_value(b, mode); 1260 break; 1261 case SpvBuiltInWorkgroupSize: 1262 /* This should already be handled */ 1263 vtn_fail("unsupported builtin"); 1264 break; 1265 case SpvBuiltInWorkgroupId: 1266 *location = SYSTEM_VALUE_WORK_GROUP_ID; 1267 set_mode_system_value(b, mode); 1268 break; 1269 case SpvBuiltInLocalInvocationId: 1270 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID; 1271 set_mode_system_value(b, mode); 1272 break; 1273 case SpvBuiltInLocalInvocationIndex: 1274 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 1275 set_mode_system_value(b, mode); 1276 break; 1277 case SpvBuiltInGlobalInvocationId: 1278 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID; 1279 set_mode_system_value(b, mode); 1280 break; 1281 case SpvBuiltInBaseVertex: 1282 *location = SYSTEM_VALUE_BASE_VERTEX; 1283 set_mode_system_value(b, mode); 1284 break; 1285 case SpvBuiltInBaseInstance: 1286 *location = SYSTEM_VALUE_BASE_INSTANCE; 1287 set_mode_system_value(b, mode); 1288 break; 1289 case SpvBuiltInDrawIndex: 1290 *location = SYSTEM_VALUE_DRAW_ID; 1291 set_mode_system_value(b, mode); 1292 break; 1293 case SpvBuiltInViewIndex: 1294 *location = SYSTEM_VALUE_VIEW_INDEX; 1295 set_mode_system_value(b, mode); 1296 break; 1297 default: 1298 vtn_fail("unsupported builtin"); 1299 } 1300 } 1301 1302 static void 1303 apply_var_decoration(struct vtn_builder *b, nir_variable *nir_var, 1304 const struct vtn_decoration *dec) 1305 { 1306 switch (dec->decoration) { 1307 case SpvDecorationRelaxedPrecision: 1308 break; /* FIXME: Do nothing with this for now. */ 1309 case SpvDecorationNoPerspective: 1310 nir_var->data.interpolation = INTERP_MODE_NOPERSPECTIVE; 1311 break; 1312 case SpvDecorationFlat: 1313 nir_var->data.interpolation = INTERP_MODE_FLAT; 1314 break; 1315 case SpvDecorationCentroid: 1316 nir_var->data.centroid = true; 1317 break; 1318 case SpvDecorationSample: 1319 nir_var->data.sample = true; 1320 break; 1321 case SpvDecorationInvariant: 1322 nir_var->data.invariant = true; 1323 break; 1324 case SpvDecorationConstant: 1325 vtn_assert(nir_var->constant_initializer != NULL); 1326 nir_var->data.read_only = true; 1327 break; 1328 case SpvDecorationNonReadable: 1329 nir_var->data.image.write_only = true; 1330 break; 1331 case SpvDecorationNonWritable: 1332 nir_var->data.read_only = true; 1333 nir_var->data.image.read_only = true; 1334 break; 1335 case SpvDecorationComponent: 1336 nir_var->data.location_frac = dec->literals[0]; 1337 break; 1338 case SpvDecorationIndex: 1339 nir_var->data.index = dec->literals[0]; 1340 break; 1341 case SpvDecorationBuiltIn: { 1342 SpvBuiltIn builtin = dec->literals[0]; 1343 1344 if (builtin == SpvBuiltInWorkgroupSize) { 1345 /* This shouldn't be a builtin. It's actually a constant. */ 1346 nir_var->data.mode = nir_var_global; 1347 nir_var->data.read_only = true; 1348 1349 nir_constant *c = rzalloc(nir_var, nir_constant); 1350 c->values[0].u32[0] = b->shader->info.cs.local_size[0]; 1351 c->values[0].u32[1] = b->shader->info.cs.local_size[1]; 1352 c->values[0].u32[2] = b->shader->info.cs.local_size[2]; 1353 nir_var->constant_initializer = c; 1354 break; 1355 } 1356 1357 nir_variable_mode mode = nir_var->data.mode; 1358 vtn_get_builtin_location(b, builtin, &nir_var->data.location, &mode); 1359 nir_var->data.mode = mode; 1360 1361 switch (builtin) { 1362 case SpvBuiltInTessLevelOuter: 1363 case SpvBuiltInTessLevelInner: 1364 nir_var->data.compact = true; 1365 break; 1366 case SpvBuiltInSamplePosition: 1367 nir_var->data.origin_upper_left = b->origin_upper_left; 1368 /* fallthrough */ 1369 case SpvBuiltInFragCoord: 1370 nir_var->data.pixel_center_integer = b->pixel_center_integer; 1371 break; 1372 default: 1373 break; 1374 } 1375 } 1376 1377 case SpvDecorationSpecId: 1378 case SpvDecorationRowMajor: 1379 case SpvDecorationColMajor: 1380 case SpvDecorationMatrixStride: 1381 case SpvDecorationRestrict: 1382 case SpvDecorationAliased: 1383 case SpvDecorationVolatile: 1384 case SpvDecorationCoherent: 1385 case SpvDecorationUniform: 1386 case SpvDecorationStream: 1387 case SpvDecorationOffset: 1388 case SpvDecorationLinkageAttributes: 1389 break; /* Do nothing with these here */ 1390 1391 case SpvDecorationPatch: 1392 nir_var->data.patch = true; 1393 break; 1394 1395 case SpvDecorationLocation: 1396 vtn_fail("Handled above"); 1397 1398 case SpvDecorationBlock: 1399 case SpvDecorationBufferBlock: 1400 case SpvDecorationArrayStride: 1401 case SpvDecorationGLSLShared: 1402 case SpvDecorationGLSLPacked: 1403 break; /* These can apply to a type but we don't care about them */ 1404 1405 case SpvDecorationBinding: 1406 case SpvDecorationDescriptorSet: 1407 case SpvDecorationNoContraction: 1408 case SpvDecorationInputAttachmentIndex: 1409 vtn_warn("Decoration not allowed for variable or structure member: %s", 1410 spirv_decoration_to_string(dec->decoration)); 1411 break; 1412 1413 case SpvDecorationXfbBuffer: 1414 case SpvDecorationXfbStride: 1415 vtn_warn("Vulkan does not have transform feedback: %s", 1416 spirv_decoration_to_string(dec->decoration)); 1417 break; 1418 1419 case SpvDecorationCPacked: 1420 case SpvDecorationSaturatedConversion: 1421 case SpvDecorationFuncParamAttr: 1422 case SpvDecorationFPRoundingMode: 1423 case SpvDecorationFPFastMathMode: 1424 case SpvDecorationAlignment: 1425 vtn_warn("Decoration only allowed for CL-style kernels: %s", 1426 spirv_decoration_to_string(dec->decoration)); 1427 break; 1428 1429 default: 1430 vtn_fail("Unhandled decoration"); 1431 } 1432 } 1433 1434 static void 1435 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1436 const struct vtn_decoration *dec, void *out_is_patch) 1437 { 1438 if (dec->decoration == SpvDecorationPatch) { 1439 *((bool *) out_is_patch) = true; 1440 } 1441 } 1442 1443 static void 1444 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1445 const struct vtn_decoration *dec, void *void_var) 1446 { 1447 struct vtn_variable *vtn_var = void_var; 1448 1449 /* Handle decorations that apply to a vtn_variable as a whole */ 1450 switch (dec->decoration) { 1451 case SpvDecorationBinding: 1452 vtn_var->binding = dec->literals[0]; 1453 return; 1454 case SpvDecorationDescriptorSet: 1455 vtn_var->descriptor_set = dec->literals[0]; 1456 return; 1457 case SpvDecorationInputAttachmentIndex: 1458 vtn_var->input_attachment_index = dec->literals[0]; 1459 return; 1460 case SpvDecorationPatch: 1461 vtn_var->patch = true; 1462 break; 1463 default: 1464 break; 1465 } 1466 1467 if (val->value_type == vtn_value_type_pointer) { 1468 assert(val->pointer->var == void_var); 1469 assert(val->pointer->chain == NULL); 1470 assert(member == -1); 1471 } else { 1472 assert(val->value_type == vtn_value_type_type); 1473 } 1474 1475 /* Location is odd. If applied to a split structure, we have to walk the 1476 * whole thing and accumulate the location. It's easier to handle as a 1477 * special case. 1478 */ 1479 if (dec->decoration == SpvDecorationLocation) { 1480 unsigned location = dec->literals[0]; 1481 bool is_vertex_input; 1482 if (b->shader->info.stage == MESA_SHADER_FRAGMENT && 1483 vtn_var->mode == vtn_variable_mode_output) { 1484 is_vertex_input = false; 1485 location += FRAG_RESULT_DATA0; 1486 } else if (b->shader->info.stage == MESA_SHADER_VERTEX && 1487 vtn_var->mode == vtn_variable_mode_input) { 1488 is_vertex_input = true; 1489 location += VERT_ATTRIB_GENERIC0; 1490 } else if (vtn_var->mode == vtn_variable_mode_input || 1491 vtn_var->mode == vtn_variable_mode_output) { 1492 is_vertex_input = false; 1493 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0; 1494 } else { 1495 vtn_warn("Location must be on input or output variable"); 1496 return; 1497 } 1498 1499 if (vtn_var->var) { 1500 /* This handles the member and lone variable cases */ 1501 vtn_var->var->data.location = location; 1502 } else { 1503 /* This handles the structure member case */ 1504 assert(vtn_var->members); 1505 unsigned length = 1506 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1507 for (unsigned i = 0; i < length; i++) { 1508 vtn_var->members[i]->data.location = location; 1509 location += 1510 glsl_count_attribute_slots(vtn_var->members[i]->interface_type, 1511 is_vertex_input); 1512 } 1513 } 1514 return; 1515 } else { 1516 if (vtn_var->var) { 1517 assert(member == -1); 1518 apply_var_decoration(b, vtn_var->var, dec); 1519 } else if (vtn_var->members) { 1520 if (member >= 0) { 1521 /* Member decorations must come from a type */ 1522 assert(val->value_type == vtn_value_type_type); 1523 apply_var_decoration(b, vtn_var->members[member], dec); 1524 } else { 1525 unsigned length = 1526 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1527 for (unsigned i = 0; i < length; i++) 1528 apply_var_decoration(b, vtn_var->members[i], dec); 1529 } 1530 } else { 1531 /* A few variables, those with external storage, have no actual 1532 * nir_variables associated with them. Fortunately, all decorations 1533 * we care about for those variables are on the type only. 1534 */ 1535 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo || 1536 vtn_var->mode == vtn_variable_mode_ssbo || 1537 vtn_var->mode == vtn_variable_mode_push_constant || 1538 (vtn_var->mode == vtn_variable_mode_workgroup && 1539 b->options->lower_workgroup_access_to_offsets)); 1540 } 1541 } 1542 } 1543 1544 static enum vtn_variable_mode 1545 vtn_storage_class_to_mode(struct vtn_builder *b, 1546 SpvStorageClass class, 1547 struct vtn_type *interface_type, 1548 nir_variable_mode *nir_mode_out) 1549 { 1550 enum vtn_variable_mode mode; 1551 nir_variable_mode nir_mode; 1552 switch (class) { 1553 case SpvStorageClassUniform: 1554 if (interface_type->block) { 1555 mode = vtn_variable_mode_ubo; 1556 nir_mode = 0; 1557 } else if (interface_type->buffer_block) { 1558 mode = vtn_variable_mode_ssbo; 1559 nir_mode = 0; 1560 } else { 1561 vtn_fail("Invalid uniform variable type"); 1562 } 1563 break; 1564 case SpvStorageClassStorageBuffer: 1565 mode = vtn_variable_mode_ssbo; 1566 nir_mode = 0; 1567 break; 1568 case SpvStorageClassUniformConstant: 1569 if (glsl_type_is_image(interface_type->type)) { 1570 mode = vtn_variable_mode_image; 1571 nir_mode = nir_var_uniform; 1572 } else if (glsl_type_is_sampler(interface_type->type)) { 1573 mode = vtn_variable_mode_sampler; 1574 nir_mode = nir_var_uniform; 1575 } else { 1576 vtn_fail("Invalid uniform constant variable type"); 1577 } 1578 break; 1579 case SpvStorageClassPushConstant: 1580 mode = vtn_variable_mode_push_constant; 1581 nir_mode = nir_var_uniform; 1582 break; 1583 case SpvStorageClassInput: 1584 mode = vtn_variable_mode_input; 1585 nir_mode = nir_var_shader_in; 1586 break; 1587 case SpvStorageClassOutput: 1588 mode = vtn_variable_mode_output; 1589 nir_mode = nir_var_shader_out; 1590 break; 1591 case SpvStorageClassPrivate: 1592 mode = vtn_variable_mode_global; 1593 nir_mode = nir_var_global; 1594 break; 1595 case SpvStorageClassFunction: 1596 mode = vtn_variable_mode_local; 1597 nir_mode = nir_var_local; 1598 break; 1599 case SpvStorageClassWorkgroup: 1600 mode = vtn_variable_mode_workgroup; 1601 nir_mode = nir_var_shared; 1602 break; 1603 case SpvStorageClassCrossWorkgroup: 1604 case SpvStorageClassGeneric: 1605 case SpvStorageClassAtomicCounter: 1606 default: 1607 vtn_fail("Unhandled variable storage class"); 1608 } 1609 1610 if (nir_mode_out) 1611 *nir_mode_out = nir_mode; 1612 1613 return mode; 1614 } 1615 1616 nir_ssa_def * 1617 vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr) 1618 { 1619 /* This pointer needs to have a pointer type with actual storage */ 1620 vtn_assert(ptr->ptr_type); 1621 vtn_assert(ptr->ptr_type->type); 1622 1623 if (!ptr->offset) { 1624 /* If we don't have an offset then we must be a pointer to the variable 1625 * itself. 1626 */ 1627 vtn_assert(!ptr->offset && !ptr->block_index); 1628 1629 struct vtn_access_chain chain = { 1630 .length = 0, 1631 }; 1632 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 1633 } 1634 1635 vtn_assert(ptr->offset); 1636 if (ptr->block_index) { 1637 vtn_assert(ptr->mode == vtn_variable_mode_ubo || 1638 ptr->mode == vtn_variable_mode_ssbo); 1639 return nir_vec2(&b->nb, ptr->block_index, ptr->offset); 1640 } else { 1641 vtn_assert(ptr->mode == vtn_variable_mode_workgroup); 1642 return ptr->offset; 1643 } 1644 } 1645 1646 struct vtn_pointer * 1647 vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, 1648 struct vtn_type *ptr_type) 1649 { 1650 vtn_assert(ssa->num_components <= 2 && ssa->bit_size == 32); 1651 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1652 vtn_assert(ptr_type->deref->base_type != vtn_base_type_pointer); 1653 /* This pointer type needs to have actual storage */ 1654 vtn_assert(ptr_type->type); 1655 1656 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 1657 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class, 1658 ptr_type, NULL); 1659 ptr->type = ptr_type->deref; 1660 ptr->ptr_type = ptr_type; 1661 1662 if (ssa->num_components > 1) { 1663 vtn_assert(ssa->num_components == 2); 1664 vtn_assert(ptr->mode == vtn_variable_mode_ubo || 1665 ptr->mode == vtn_variable_mode_ssbo); 1666 ptr->block_index = nir_channel(&b->nb, ssa, 0); 1667 ptr->offset = nir_channel(&b->nb, ssa, 1); 1668 } else { 1669 vtn_assert(ssa->num_components == 1); 1670 vtn_assert(ptr->mode == vtn_variable_mode_workgroup); 1671 ptr->block_index = NULL; 1672 ptr->offset = ssa; 1673 } 1674 1675 return ptr; 1676 } 1677 1678 static bool 1679 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage) 1680 { 1681 if (var->patch || !glsl_type_is_array(var->type->type)) 1682 return false; 1683 1684 if (var->mode == vtn_variable_mode_input) { 1685 return stage == MESA_SHADER_TESS_CTRL || 1686 stage == MESA_SHADER_TESS_EVAL || 1687 stage == MESA_SHADER_GEOMETRY; 1688 } 1689 1690 if (var->mode == vtn_variable_mode_output) 1691 return stage == MESA_SHADER_TESS_CTRL; 1692 1693 return false; 1694 } 1695 1696 static void 1697 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, 1698 struct vtn_type *ptr_type, SpvStorageClass storage_class, 1699 nir_constant *initializer) 1700 { 1701 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1702 struct vtn_type *type = ptr_type->deref; 1703 1704 struct vtn_type *without_array = type; 1705 while(glsl_type_is_array(without_array->type)) 1706 without_array = without_array->array_element; 1707 1708 enum vtn_variable_mode mode; 1709 nir_variable_mode nir_mode; 1710 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode); 1711 1712 switch (mode) { 1713 case vtn_variable_mode_ubo: 1714 b->shader->info.num_ubos++; 1715 break; 1716 case vtn_variable_mode_ssbo: 1717 b->shader->info.num_ssbos++; 1718 break; 1719 case vtn_variable_mode_image: 1720 b->shader->info.num_images++; 1721 break; 1722 case vtn_variable_mode_sampler: 1723 b->shader->info.num_textures++; 1724 break; 1725 case vtn_variable_mode_push_constant: 1726 b->shader->num_uniforms = vtn_type_block_size(b, type); 1727 break; 1728 default: 1729 /* No tallying is needed */ 1730 break; 1731 } 1732 1733 struct vtn_variable *var = rzalloc(b, struct vtn_variable); 1734 var->type = type; 1735 var->mode = mode; 1736 1737 vtn_assert(val->value_type == vtn_value_type_pointer); 1738 val->pointer = vtn_pointer_for_variable(b, var, ptr_type); 1739 1740 switch (var->mode) { 1741 case vtn_variable_mode_local: 1742 case vtn_variable_mode_global: 1743 case vtn_variable_mode_image: 1744 case vtn_variable_mode_sampler: 1745 /* For these, we create the variable normally */ 1746 var->var = rzalloc(b->shader, nir_variable); 1747 var->var->name = ralloc_strdup(var->var, val->name); 1748 var->var->type = var->type->type; 1749 var->var->data.mode = nir_mode; 1750 1751 switch (var->mode) { 1752 case vtn_variable_mode_image: 1753 case vtn_variable_mode_sampler: 1754 var->var->interface_type = without_array->type; 1755 break; 1756 default: 1757 var->var->interface_type = NULL; 1758 break; 1759 } 1760 break; 1761 1762 case vtn_variable_mode_workgroup: 1763 if (b->options->lower_workgroup_access_to_offsets) { 1764 var->shared_location = -1; 1765 } else { 1766 /* Create the variable normally */ 1767 var->var = rzalloc(b->shader, nir_variable); 1768 var->var->name = ralloc_strdup(var->var, val->name); 1769 var->var->type = var->type->type; 1770 var->var->data.mode = nir_var_shared; 1771 } 1772 break; 1773 1774 case vtn_variable_mode_input: 1775 case vtn_variable_mode_output: { 1776 /* In order to know whether or not we're a per-vertex inout, we need 1777 * the patch qualifier. This means walking the variable decorations 1778 * early before we actually create any variables. Not a big deal. 1779 * 1780 * GLSLang really likes to place decorations in the most interior 1781 * thing it possibly can. In particular, if you have a struct, it 1782 * will place the patch decorations on the struct members. This 1783 * should be handled by the variable splitting below just fine. 1784 * 1785 * If you have an array-of-struct, things get even more weird as it 1786 * will place the patch decorations on the struct even though it's 1787 * inside an array and some of the members being patch and others not 1788 * makes no sense whatsoever. Since the only sensible thing is for 1789 * it to be all or nothing, we'll call it patch if any of the members 1790 * are declared patch. 1791 */ 1792 var->patch = false; 1793 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch); 1794 if (glsl_type_is_array(var->type->type) && 1795 glsl_type_is_struct(without_array->type)) { 1796 vtn_foreach_decoration(b, vtn_value(b, without_array->id, 1797 vtn_value_type_type), 1798 var_is_patch_cb, &var->patch); 1799 } 1800 1801 /* For inputs and outputs, we immediately split structures. This 1802 * is for a couple of reasons. For one, builtins may all come in 1803 * a struct and we really want those split out into separate 1804 * variables. For another, interpolation qualifiers can be 1805 * applied to members of the top-level struct ane we need to be 1806 * able to preserve that information. 1807 */ 1808 1809 int array_length = -1; 1810 struct vtn_type *interface_type = var->type; 1811 if (is_per_vertex_inout(var, b->shader->info.stage)) { 1812 /* In Geometry shaders (and some tessellation), inputs come 1813 * in per-vertex arrays. However, some builtins come in 1814 * non-per-vertex, hence the need for the is_array check. In 1815 * any case, there are no non-builtin arrays allowed so this 1816 * check should be sufficient. 1817 */ 1818 interface_type = var->type->array_element; 1819 array_length = glsl_get_length(var->type->type); 1820 } 1821 1822 if (glsl_type_is_struct(interface_type->type)) { 1823 /* It's a struct. Split it. */ 1824 unsigned num_members = glsl_get_length(interface_type->type); 1825 var->members = ralloc_array(b, nir_variable *, num_members); 1826 1827 for (unsigned i = 0; i < num_members; i++) { 1828 const struct glsl_type *mtype = interface_type->members[i]->type; 1829 if (array_length >= 0) 1830 mtype = glsl_array_type(mtype, array_length); 1831 1832 var->members[i] = rzalloc(b->shader, nir_variable); 1833 var->members[i]->name = 1834 ralloc_asprintf(var->members[i], "%s.%d", val->name, i); 1835 var->members[i]->type = mtype; 1836 var->members[i]->interface_type = 1837 interface_type->members[i]->type; 1838 var->members[i]->data.mode = nir_mode; 1839 var->members[i]->data.patch = var->patch; 1840 } 1841 } else { 1842 var->var = rzalloc(b->shader, nir_variable); 1843 var->var->name = ralloc_strdup(var->var, val->name); 1844 var->var->type = var->type->type; 1845 var->var->interface_type = interface_type->type; 1846 var->var->data.mode = nir_mode; 1847 var->var->data.patch = var->patch; 1848 } 1849 1850 /* For inputs and outputs, we need to grab locations and builtin 1851 * information from the interface type. 1852 */ 1853 vtn_foreach_decoration(b, vtn_value(b, interface_type->id, 1854 vtn_value_type_type), 1855 var_decoration_cb, var); 1856 break; 1857 } 1858 1859 case vtn_variable_mode_param: 1860 vtn_fail("Not created through OpVariable"); 1861 1862 case vtn_variable_mode_ubo: 1863 case vtn_variable_mode_ssbo: 1864 case vtn_variable_mode_push_constant: 1865 /* These don't need actual variables. */ 1866 break; 1867 } 1868 1869 if (initializer) { 1870 var->var->constant_initializer = 1871 nir_constant_clone(initializer, var->var); 1872 } 1873 1874 vtn_foreach_decoration(b, val, var_decoration_cb, var); 1875 1876 if (var->mode == vtn_variable_mode_image || 1877 var->mode == vtn_variable_mode_sampler) { 1878 /* XXX: We still need the binding information in the nir_variable 1879 * for these. We should fix that. 1880 */ 1881 var->var->data.binding = var->binding; 1882 var->var->data.descriptor_set = var->descriptor_set; 1883 var->var->data.index = var->input_attachment_index; 1884 1885 if (var->mode == vtn_variable_mode_image) 1886 var->var->data.image.format = without_array->image_format; 1887 } 1888 1889 if (var->mode == vtn_variable_mode_local) { 1890 vtn_assert(var->members == NULL && var->var != NULL); 1891 nir_function_impl_add_variable(b->nb.impl, var->var); 1892 } else if (var->var) { 1893 nir_shader_add_variable(b->shader, var->var); 1894 } else if (var->members) { 1895 unsigned count = glsl_get_length(without_array->type); 1896 for (unsigned i = 0; i < count; i++) { 1897 vtn_assert(var->members[i]->data.mode != nir_var_local); 1898 nir_shader_add_variable(b->shader, var->members[i]); 1899 } 1900 } else { 1901 vtn_assert(vtn_pointer_is_external_block(b, val->pointer)); 1902 } 1903 } 1904 1905 static void 1906 vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode, 1907 struct vtn_type *dst_type, 1908 struct vtn_type *src_type) 1909 { 1910 if (dst_type->id == src_type->id) 1911 return; 1912 1913 if (vtn_types_compatible(b, dst_type, src_type)) { 1914 /* Early versions of GLSLang would re-emit types unnecessarily and you 1915 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have 1916 * mismatched source and destination types. 1917 * 1918 * https://github.com/KhronosGroup/glslang/issues/304 1919 * https://github.com/KhronosGroup/glslang/issues/307 1920 * https://bugs.freedesktop.org/show_bug.cgi?id=104338 1921 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 1922 */ 1923 vtn_warn("Source and destination types of %s do not have the same " 1924 "ID (but are compatible): %u vs %u", 1925 spirv_op_to_string(opcode), dst_type->id, src_type->id); 1926 return; 1927 } 1928 1929 vtn_fail("Source and destination types of %s do not match: %s vs. %s", 1930 spirv_op_to_string(opcode), 1931 glsl_get_type_name(dst_type->type), 1932 glsl_get_type_name(src_type->type)); 1933 } 1934 1935 void 1936 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, 1937 const uint32_t *w, unsigned count) 1938 { 1939 switch (opcode) { 1940 case SpvOpUndef: { 1941 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); 1942 val->type = vtn_value(b, w[1], vtn_value_type_type)->type; 1943 break; 1944 } 1945 1946 case SpvOpVariable: { 1947 struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; 1948 1949 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 1950 1951 SpvStorageClass storage_class = w[3]; 1952 nir_constant *initializer = NULL; 1953 if (count > 4) 1954 initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant; 1955 1956 vtn_create_variable(b, val, ptr_type, storage_class, initializer); 1957 break; 1958 } 1959 1960 case SpvOpAccessChain: 1961 case SpvOpPtrAccessChain: 1962 case SpvOpInBoundsAccessChain: { 1963 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4); 1964 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain); 1965 1966 unsigned idx = 0; 1967 for (int i = 4; i < count; i++) { 1968 struct vtn_value *link_val = vtn_untyped_value(b, w[i]); 1969 if (link_val->value_type == vtn_value_type_constant) { 1970 chain->link[idx].mode = vtn_access_mode_literal; 1971 chain->link[idx].id = link_val->constant->values[0].u32[0]; 1972 } else { 1973 chain->link[idx].mode = vtn_access_mode_id; 1974 chain->link[idx].id = w[i]; 1975 1976 } 1977 idx++; 1978 } 1979 1980 struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; 1981 struct vtn_value *base_val = vtn_untyped_value(b, w[3]); 1982 if (base_val->value_type == vtn_value_type_sampled_image) { 1983 /* This is rather insane. SPIR-V allows you to use OpSampledImage 1984 * to combine an array of images with a single sampler to get an 1985 * array of sampled images that all share the same sampler. 1986 * Fortunately, this means that we can more-or-less ignore the 1987 * sampler when crawling the access chain, but it does leave us 1988 * with this rather awkward little special-case. 1989 */ 1990 struct vtn_value *val = 1991 vtn_push_value(b, w[2], vtn_value_type_sampled_image); 1992 val->sampled_image = ralloc(b, struct vtn_sampled_image); 1993 val->sampled_image->type = base_val->sampled_image->type; 1994 val->sampled_image->image = 1995 vtn_pointer_dereference(b, base_val->sampled_image->image, chain); 1996 val->sampled_image->sampler = base_val->sampled_image->sampler; 1997 } else { 1998 vtn_assert(base_val->value_type == vtn_value_type_pointer); 1999 struct vtn_value *val = 2000 vtn_push_value(b, w[2], vtn_value_type_pointer); 2001 val->pointer = vtn_pointer_dereference(b, base_val->pointer, chain); 2002 val->pointer->ptr_type = ptr_type; 2003 } 2004 break; 2005 } 2006 2007 case SpvOpCopyMemory: { 2008 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer); 2009 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer); 2010 2011 vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref); 2012 2013 vtn_variable_copy(b, dest->pointer, src->pointer); 2014 break; 2015 } 2016 2017 case SpvOpLoad: { 2018 struct vtn_type *res_type = 2019 vtn_value(b, w[1], vtn_value_type_type)->type; 2020 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer); 2021 struct vtn_pointer *src = src_val->pointer; 2022 2023 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref); 2024 2025 if (src->mode == vtn_variable_mode_image || 2026 src->mode == vtn_variable_mode_sampler) { 2027 vtn_push_value(b, w[2], vtn_value_type_pointer)->pointer = src; 2028 return; 2029 } 2030 2031 vtn_push_ssa(b, w[2], res_type, vtn_variable_load(b, src)); 2032 break; 2033 } 2034 2035 case SpvOpStore: { 2036 struct vtn_value *dest_val = vtn_value(b, w[1], vtn_value_type_pointer); 2037 struct vtn_pointer *dest = dest_val->pointer; 2038 struct vtn_value *src_val = vtn_untyped_value(b, w[2]); 2039 2040 /* OpStore requires us to actually have a storage type */ 2041 vtn_fail_if(dest->type->type == NULL, 2042 "Invalid destination type for OpStore"); 2043 2044 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL && 2045 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) { 2046 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but 2047 * would then store them to a local variable as bool. Work around 2048 * the issue by doing an implicit conversion. 2049 * 2050 * https://github.com/KhronosGroup/glslang/issues/170 2051 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2052 */ 2053 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type " 2054 "OpTypeBool. Doing an implicit conversion to work around " 2055 "the problem."); 2056 struct vtn_ssa_value *bool_ssa = 2057 vtn_create_ssa_value(b, dest->type->type); 2058 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def); 2059 vtn_variable_store(b, bool_ssa, dest); 2060 break; 2061 } 2062 2063 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type); 2064 2065 if (glsl_type_is_sampler(dest->type->type)) { 2066 vtn_warn("OpStore of a sampler detected. Doing on-the-fly copy " 2067 "propagation to workaround the problem."); 2068 vtn_assert(dest->var->copy_prop_sampler == NULL); 2069 dest->var->copy_prop_sampler = 2070 vtn_value(b, w[2], vtn_value_type_pointer)->pointer; 2071 break; 2072 } 2073 2074 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); 2075 vtn_variable_store(b, src, dest); 2076 break; 2077 } 2078 2079 case SpvOpArrayLength: { 2080 struct vtn_pointer *ptr = 2081 vtn_value(b, w[3], vtn_value_type_pointer)->pointer; 2082 2083 const uint32_t offset = ptr->var->type->offsets[w[4]]; 2084 const uint32_t stride = ptr->var->type->members[w[4]]->stride; 2085 2086 if (!ptr->block_index) { 2087 struct vtn_access_chain chain = { 2088 .length = 0, 2089 }; 2090 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 2091 vtn_assert(ptr->block_index); 2092 } 2093 2094 nir_intrinsic_instr *instr = 2095 nir_intrinsic_instr_create(b->nb.shader, 2096 nir_intrinsic_get_buffer_size); 2097 instr->src[0] = nir_src_for_ssa(ptr->block_index); 2098 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 2099 nir_builder_instr_insert(&b->nb, &instr->instr); 2100 nir_ssa_def *buf_size = &instr->dest.ssa; 2101 2102 /* array_length = max(buffer_size - offset, 0) / stride */ 2103 nir_ssa_def *array_length = 2104 nir_idiv(&b->nb, 2105 nir_imax(&b->nb, 2106 nir_isub(&b->nb, 2107 buf_size, 2108 nir_imm_int(&b->nb, offset)), 2109 nir_imm_int(&b->nb, 0u)), 2110 nir_imm_int(&b->nb, stride)); 2111 2112 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); 2113 val->ssa = vtn_create_ssa_value(b, glsl_uint_type()); 2114 val->ssa->def = array_length; 2115 break; 2116 } 2117 2118 case SpvOpCopyMemorySized: 2119 default: 2120 vtn_fail("Unhandled opcode"); 2121 } 2122 } 2123
