1 /* 2 * Copyright 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include "anv_nir.h" 25 #include "program/prog_parameter.h" 26 #include "nir/nir_builder.h" 27 28 struct apply_pipeline_layout_state { 29 nir_shader *shader; 30 nir_builder builder; 31 32 struct anv_pipeline_layout *layout; 33 bool add_bounds_checks; 34 35 struct { 36 BITSET_WORD *used; 37 uint8_t *surface_offsets; 38 uint8_t *sampler_offsets; 39 uint8_t *image_offsets; 40 } set[MAX_SETS]; 41 }; 42 43 static void 44 add_binding(struct apply_pipeline_layout_state *state, 45 uint32_t set, uint32_t binding) 46 { 47 BITSET_SET(state->set[set].used, binding); 48 } 49 50 static void 51 add_var_binding(struct apply_pipeline_layout_state *state, nir_variable *var) 52 { 53 add_binding(state, var->data.descriptor_set, var->data.binding); 54 } 55 56 static void 57 get_used_bindings_block(nir_block *block, 58 struct apply_pipeline_layout_state *state) 59 { 60 nir_foreach_instr_safe(instr, block) { 61 switch (instr->type) { 62 case nir_instr_type_intrinsic: { 63 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); 64 switch (intrin->intrinsic) { 65 case nir_intrinsic_vulkan_resource_index: 66 add_binding(state, nir_intrinsic_desc_set(intrin), 67 nir_intrinsic_binding(intrin)); 68 break; 69 70 case nir_intrinsic_image_load: 71 case nir_intrinsic_image_store: 72 case nir_intrinsic_image_atomic_add: 73 case nir_intrinsic_image_atomic_min: 74 case nir_intrinsic_image_atomic_max: 75 case nir_intrinsic_image_atomic_and: 76 case nir_intrinsic_image_atomic_or: 77 case nir_intrinsic_image_atomic_xor: 78 case nir_intrinsic_image_atomic_exchange: 79 case nir_intrinsic_image_atomic_comp_swap: 80 case nir_intrinsic_image_size: 81 case nir_intrinsic_image_samples: 82 add_var_binding(state, intrin->variables[0]->var); 83 break; 84 85 default: 86 break; 87 } 88 break; 89 } 90 case nir_instr_type_tex: { 91 nir_tex_instr *tex = nir_instr_as_tex(instr); 92 assert(tex->texture); 93 add_var_binding(state, tex->texture->var); 94 if (tex->sampler) 95 add_var_binding(state, tex->sampler->var); 96 break; 97 } 98 default: 99 continue; 100 } 101 } 102 } 103 104 static void 105 lower_res_index_intrinsic(nir_intrinsic_instr *intrin, 106 struct apply_pipeline_layout_state *state) 107 { 108 nir_builder *b = &state->builder; 109 110 b->cursor = nir_before_instr(&intrin->instr); 111 112 uint32_t set = nir_intrinsic_desc_set(intrin); 113 uint32_t binding = nir_intrinsic_binding(intrin); 114 115 uint32_t surface_index = state->set[set].surface_offsets[binding]; 116 uint32_t array_size = 117 state->layout->set[set].layout->binding[binding].array_size; 118 119 nir_ssa_def *block_index = nir_ssa_for_src(b, intrin->src[0], 1); 120 121 if (state->add_bounds_checks) 122 block_index = nir_umin(b, block_index, nir_imm_int(b, array_size - 1)); 123 124 block_index = nir_iadd(b, nir_imm_int(b, surface_index), block_index); 125 126 assert(intrin->dest.is_ssa); 127 nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(block_index)); 128 nir_instr_remove(&intrin->instr); 129 } 130 131 static void 132 lower_tex_deref(nir_tex_instr *tex, nir_deref_var *deref, 133 unsigned *const_index, unsigned array_size, 134 nir_tex_src_type src_type, 135 struct apply_pipeline_layout_state *state) 136 { 137 nir_builder *b = &state->builder; 138 139 if (deref->deref.child) { 140 assert(deref->deref.child->deref_type == nir_deref_type_array); 141 nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child); 142 143 if (deref_array->deref_array_type == nir_deref_array_type_indirect) { 144 nir_ssa_def *index = 145 nir_iadd(b, nir_imm_int(b, deref_array->base_offset), 146 nir_ssa_for_src(b, deref_array->indirect, 1)); 147 148 if (state->add_bounds_checks) 149 index = nir_umin(b, index, nir_imm_int(b, array_size - 1)); 150 151 nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src, 152 tex->num_srcs + 1); 153 154 for (unsigned i = 0; i < tex->num_srcs; i++) { 155 new_srcs[i].src_type = tex->src[i].src_type; 156 nir_instr_move_src(&tex->instr, &new_srcs[i].src, &tex->src[i].src); 157 } 158 159 ralloc_free(tex->src); 160 tex->src = new_srcs; 161 162 /* Now we can go ahead and move the source over to being a 163 * first-class texture source. 164 */ 165 tex->src[tex->num_srcs].src_type = src_type; 166 nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs].src, 167 nir_src_for_ssa(index)); 168 tex->num_srcs++; 169 } else { 170 *const_index += MIN2(deref_array->base_offset, array_size - 1); 171 } 172 } 173 } 174 175 static void 176 cleanup_tex_deref(nir_tex_instr *tex, nir_deref_var *deref) 177 { 178 if (deref->deref.child == NULL) 179 return; 180 181 nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child); 182 183 if (deref_array->deref_array_type != nir_deref_array_type_indirect) 184 return; 185 186 nir_instr_rewrite_src(&tex->instr, &deref_array->indirect, NIR_SRC_INIT); 187 } 188 189 static void 190 lower_tex(nir_tex_instr *tex, struct apply_pipeline_layout_state *state) 191 { 192 /* No one should have come by and lowered it already */ 193 assert(tex->texture); 194 195 state->builder.cursor = nir_before_instr(&tex->instr); 196 197 unsigned set = tex->texture->var->data.descriptor_set; 198 unsigned binding = tex->texture->var->data.binding; 199 unsigned array_size = 200 state->layout->set[set].layout->binding[binding].array_size; 201 tex->texture_index = state->set[set].surface_offsets[binding]; 202 lower_tex_deref(tex, tex->texture, &tex->texture_index, array_size, 203 nir_tex_src_texture_offset, state); 204 205 if (tex->sampler) { 206 unsigned set = tex->sampler->var->data.descriptor_set; 207 unsigned binding = tex->sampler->var->data.binding; 208 unsigned array_size = 209 state->layout->set[set].layout->binding[binding].array_size; 210 tex->sampler_index = state->set[set].sampler_offsets[binding]; 211 lower_tex_deref(tex, tex->sampler, &tex->sampler_index, array_size, 212 nir_tex_src_sampler_offset, state); 213 } 214 215 /* The backend only ever uses this to mark used surfaces. We don't care 216 * about that little optimization so it just needs to be non-zero. 217 */ 218 tex->texture_array_size = 1; 219 220 cleanup_tex_deref(tex, tex->texture); 221 if (tex->sampler) 222 cleanup_tex_deref(tex, tex->sampler); 223 tex->texture = NULL; 224 tex->sampler = NULL; 225 } 226 227 static void 228 apply_pipeline_layout_block(nir_block *block, 229 struct apply_pipeline_layout_state *state) 230 { 231 nir_foreach_instr_safe(instr, block) { 232 switch (instr->type) { 233 case nir_instr_type_intrinsic: { 234 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); 235 if (intrin->intrinsic == nir_intrinsic_vulkan_resource_index) { 236 lower_res_index_intrinsic(intrin, state); 237 } 238 break; 239 } 240 case nir_instr_type_tex: 241 lower_tex(nir_instr_as_tex(instr), state); 242 break; 243 default: 244 continue; 245 } 246 } 247 } 248 249 static void 250 setup_vec4_uniform_value(const union gl_constant_value **params, 251 const union gl_constant_value *values, 252 unsigned n) 253 { 254 static const gl_constant_value zero = { 0 }; 255 256 for (unsigned i = 0; i < n; ++i) 257 params[i] = &values[i]; 258 259 for (unsigned i = n; i < 4; ++i) 260 params[i] = &zero; 261 } 262 263 void 264 anv_nir_apply_pipeline_layout(struct anv_pipeline *pipeline, 265 nir_shader *shader, 266 struct brw_stage_prog_data *prog_data, 267 struct anv_pipeline_bind_map *map) 268 { 269 struct anv_pipeline_layout *layout = pipeline->layout; 270 271 struct apply_pipeline_layout_state state = { 272 .shader = shader, 273 .layout = layout, 274 .add_bounds_checks = pipeline->device->robust_buffer_access, 275 }; 276 277 void *mem_ctx = ralloc_context(NULL); 278 279 for (unsigned s = 0; s < layout->num_sets; s++) { 280 const unsigned count = layout->set[s].layout->binding_count; 281 const unsigned words = BITSET_WORDS(count); 282 state.set[s].used = rzalloc_array(mem_ctx, BITSET_WORD, words); 283 state.set[s].surface_offsets = rzalloc_array(mem_ctx, uint8_t, count); 284 state.set[s].sampler_offsets = rzalloc_array(mem_ctx, uint8_t, count); 285 state.set[s].image_offsets = rzalloc_array(mem_ctx, uint8_t, count); 286 } 287 288 nir_foreach_function(function, shader) { 289 if (!function->impl) 290 continue; 291 292 nir_foreach_block(block, function->impl) 293 get_used_bindings_block(block, &state); 294 } 295 296 for (uint32_t set = 0; set < layout->num_sets; set++) { 297 struct anv_descriptor_set_layout *set_layout = layout->set[set].layout; 298 299 BITSET_WORD b, _tmp; 300 BITSET_FOREACH_SET(b, _tmp, state.set[set].used, 301 set_layout->binding_count) { 302 if (set_layout->binding[b].stage[shader->stage].surface_index >= 0) 303 map->surface_count += set_layout->binding[b].array_size; 304 if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0) 305 map->sampler_count += set_layout->binding[b].array_size; 306 if (set_layout->binding[b].stage[shader->stage].image_index >= 0) 307 map->image_count += set_layout->binding[b].array_size; 308 } 309 } 310 311 unsigned surface = 0; 312 unsigned sampler = 0; 313 unsigned image = 0; 314 for (uint32_t set = 0; set < layout->num_sets; set++) { 315 struct anv_descriptor_set_layout *set_layout = layout->set[set].layout; 316 317 BITSET_WORD b, _tmp; 318 BITSET_FOREACH_SET(b, _tmp, state.set[set].used, 319 set_layout->binding_count) { 320 unsigned array_size = set_layout->binding[b].array_size; 321 322 if (set_layout->binding[b].stage[shader->stage].surface_index >= 0) { 323 state.set[set].surface_offsets[b] = surface; 324 for (unsigned i = 0; i < array_size; i++) { 325 map->surface_to_descriptor[surface + i].set = set; 326 map->surface_to_descriptor[surface + i].binding = b; 327 map->surface_to_descriptor[surface + i].index = i; 328 } 329 surface += array_size; 330 } 331 332 if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0) { 333 state.set[set].sampler_offsets[b] = sampler; 334 for (unsigned i = 0; i < array_size; i++) { 335 map->sampler_to_descriptor[sampler + i].set = set; 336 map->sampler_to_descriptor[sampler + i].binding = b; 337 map->sampler_to_descriptor[sampler + i].index = i; 338 } 339 sampler += array_size; 340 } 341 342 if (set_layout->binding[b].stage[shader->stage].image_index >= 0) { 343 state.set[set].image_offsets[b] = image; 344 image += array_size; 345 } 346 } 347 } 348 349 nir_foreach_variable(var, &shader->uniforms) { 350 if (!glsl_type_is_image(var->interface_type)) 351 continue; 352 353 enum glsl_sampler_dim dim = glsl_get_sampler_dim(var->interface_type); 354 if (dim != GLSL_SAMPLER_DIM_SUBPASS && 355 dim != GLSL_SAMPLER_DIM_SUBPASS_MS) 356 continue; 357 358 const uint32_t set = var->data.descriptor_set; 359 const uint32_t binding = var->data.binding; 360 const uint32_t array_size = 361 layout->set[set].layout->binding[binding].array_size; 362 363 if (!BITSET_TEST(state.set[set].used, binding)) 364 continue; 365 366 struct anv_pipeline_binding *pipe_binding = 367 &map->surface_to_descriptor[state.set[set].surface_offsets[binding]]; 368 for (unsigned i = 0; i < array_size; i++) { 369 assert(pipe_binding[i].set == set); 370 assert(pipe_binding[i].binding == binding); 371 assert(pipe_binding[i].index == i); 372 pipe_binding[i].input_attachment_index = var->data.index + i; 373 } 374 } 375 376 nir_foreach_function(function, shader) { 377 if (!function->impl) 378 continue; 379 380 nir_builder_init(&state.builder, function->impl); 381 nir_foreach_block(block, function->impl) 382 apply_pipeline_layout_block(block, &state); 383 nir_metadata_preserve(function->impl, nir_metadata_block_index | 384 nir_metadata_dominance); 385 } 386 387 if (map->image_count > 0) { 388 assert(map->image_count <= MAX_IMAGES); 389 nir_foreach_variable(var, &shader->uniforms) { 390 if (glsl_type_is_image(var->type) || 391 (glsl_type_is_array(var->type) && 392 glsl_type_is_image(glsl_get_array_element(var->type)))) { 393 /* Images are represented as uniform push constants and the actual 394 * information required for reading/writing to/from the image is 395 * storred in the uniform. 396 */ 397 unsigned set = var->data.descriptor_set; 398 unsigned binding = var->data.binding; 399 unsigned image_index = state.set[set].image_offsets[binding]; 400 401 var->data.driver_location = shader->num_uniforms + 402 image_index * BRW_IMAGE_PARAM_SIZE * 4; 403 } 404 } 405 406 struct anv_push_constants *null_data = NULL; 407 const gl_constant_value **param = 408 prog_data->param + (shader->num_uniforms / 4); 409 const struct brw_image_param *image_param = null_data->images; 410 for (uint32_t i = 0; i < map->image_count; i++) { 411 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SURFACE_IDX_OFFSET, 412 (const union gl_constant_value *)&image_param->surface_idx, 1); 413 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_OFFSET_OFFSET, 414 (const union gl_constant_value *)image_param->offset, 2); 415 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SIZE_OFFSET, 416 (const union gl_constant_value *)image_param->size, 3); 417 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_STRIDE_OFFSET, 418 (const union gl_constant_value *)image_param->stride, 4); 419 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_TILING_OFFSET, 420 (const union gl_constant_value *)image_param->tiling, 3); 421 setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SWIZZLING_OFFSET, 422 (const union gl_constant_value *)image_param->swizzling, 2); 423 424 param += BRW_IMAGE_PARAM_SIZE; 425 image_param ++; 426 } 427 428 shader->num_uniforms += map->image_count * BRW_IMAGE_PARAM_SIZE * 4; 429 } 430 431 ralloc_free(mem_ctx); 432 } 433
