1 /* 2 * Copyright 2016 Red Hat. 3 * Copyright 2016 Bas Nieuwenhuizen 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 */ 24 #include "radv_meta.h" 25 #include "nir/nir_builder.h" 26 27 /* 28 * GFX queue: Compute shader implementation of image->buffer copy 29 * Compute queue: implementation also of buffer->image, image->image, and image clear. 30 */ 31 32 /* GFX9 needs to use a 3D sampler to access 3D resources, so the shader has the options 33 * for that. 34 */ 35 static nir_shader * 36 build_nir_itob_compute_shader(struct radv_device *dev, bool is_3d) 37 { 38 nir_builder b; 39 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D; 40 const struct glsl_type *sampler_type = glsl_sampler_type(dim, 41 false, 42 false, 43 GLSL_TYPE_FLOAT); 44 const struct glsl_type *img_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, 45 false, 46 false, 47 GLSL_TYPE_FLOAT); 48 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL); 49 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itob_cs_3d" : "meta_itob_cs"); 50 b.shader->info.cs.local_size[0] = 16; 51 b.shader->info.cs.local_size[1] = 16; 52 b.shader->info.cs.local_size[2] = 1; 53 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform, 54 sampler_type, "s_tex"); 55 input_img->data.descriptor_set = 0; 56 input_img->data.binding = 0; 57 58 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform, 59 img_type, "out_img"); 60 output_img->data.descriptor_set = 0; 61 output_img->data.binding = 1; 62 63 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0); 64 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0); 65 nir_ssa_def *block_size = nir_imm_ivec4(&b, 66 b.shader->info.cs.local_size[0], 67 b.shader->info.cs.local_size[1], 68 b.shader->info.cs.local_size[2], 0); 69 70 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id); 71 72 73 74 nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 75 nir_intrinsic_set_base(offset, 0); 76 nir_intrinsic_set_range(offset, 16); 77 offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0)); 78 offset->num_components = is_3d ? 3 : 2; 79 nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset"); 80 nir_builder_instr_insert(&b, &offset->instr); 81 82 nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 83 nir_intrinsic_set_base(stride, 0); 84 nir_intrinsic_set_range(stride, 16); 85 stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12)); 86 stride->num_components = 1; 87 nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride"); 88 nir_builder_instr_insert(&b, &stride->instr); 89 90 nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa); 91 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2); 92 tex->sampler_dim = dim; 93 tex->op = nir_texop_txf; 94 tex->src[0].src_type = nir_tex_src_coord; 95 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, img_coord, is_3d ? 0x7 : 0x3)); 96 tex->src[1].src_type = nir_tex_src_lod; 97 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0)); 98 tex->dest_type = nir_type_float; 99 tex->is_array = false; 100 tex->coord_components = is_3d ? 3 : 2; 101 tex->texture = nir_deref_var_create(tex, input_img); 102 tex->sampler = NULL; 103 104 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex"); 105 nir_builder_instr_insert(&b, &tex->instr); 106 107 nir_ssa_def *pos_x = nir_channel(&b, global_id, 0); 108 nir_ssa_def *pos_y = nir_channel(&b, global_id, 1); 109 110 nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa); 111 tmp = nir_iadd(&b, tmp, pos_x); 112 113 nir_ssa_def *coord = nir_vec4(&b, tmp, tmp, tmp, tmp); 114 115 nir_ssa_def *outval = &tex->dest.ssa; 116 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store); 117 store->src[0] = nir_src_for_ssa(coord); 118 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32)); 119 store->src[2] = nir_src_for_ssa(outval); 120 store->variables[0] = nir_deref_var_create(store, output_img); 121 122 nir_builder_instr_insert(&b, &store->instr); 123 return b.shader; 124 } 125 126 /* Image to buffer - don't write use image accessors */ 127 static VkResult 128 radv_device_init_meta_itob_state(struct radv_device *device) 129 { 130 VkResult result; 131 struct radv_shader_module cs = { .nir = NULL }; 132 struct radv_shader_module cs_3d = { .nir = NULL }; 133 134 cs.nir = build_nir_itob_compute_shader(device, false); 135 if (device->physical_device->rad_info.chip_class >= GFX9) 136 cs_3d.nir = build_nir_itob_compute_shader(device, true); 137 138 /* 139 * two descriptors one for the image being sampled 140 * one for the buffer being written. 141 */ 142 VkDescriptorSetLayoutCreateInfo ds_create_info = { 143 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 144 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, 145 .bindingCount = 2, 146 .pBindings = (VkDescriptorSetLayoutBinding[]) { 147 { 148 .binding = 0, 149 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 150 .descriptorCount = 1, 151 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 152 .pImmutableSamplers = NULL 153 }, 154 { 155 .binding = 1, 156 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 157 .descriptorCount = 1, 158 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 159 .pImmutableSamplers = NULL 160 }, 161 } 162 }; 163 164 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device), 165 &ds_create_info, 166 &device->meta_state.alloc, 167 &device->meta_state.itob.img_ds_layout); 168 if (result != VK_SUCCESS) 169 goto fail; 170 171 172 VkPipelineLayoutCreateInfo pl_create_info = { 173 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 174 .setLayoutCount = 1, 175 .pSetLayouts = &device->meta_state.itob.img_ds_layout, 176 .pushConstantRangeCount = 1, 177 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16}, 178 }; 179 180 result = radv_CreatePipelineLayout(radv_device_to_handle(device), 181 &pl_create_info, 182 &device->meta_state.alloc, 183 &device->meta_state.itob.img_p_layout); 184 if (result != VK_SUCCESS) 185 goto fail; 186 187 /* compute shader */ 188 189 VkPipelineShaderStageCreateInfo pipeline_shader_stage = { 190 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 191 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 192 .module = radv_shader_module_to_handle(&cs), 193 .pName = "main", 194 .pSpecializationInfo = NULL, 195 }; 196 197 VkComputePipelineCreateInfo vk_pipeline_info = { 198 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 199 .stage = pipeline_shader_stage, 200 .flags = 0, 201 .layout = device->meta_state.itob.img_p_layout, 202 }; 203 204 result = radv_CreateComputePipelines(radv_device_to_handle(device), 205 radv_pipeline_cache_to_handle(&device->meta_state.cache), 206 1, &vk_pipeline_info, NULL, 207 &device->meta_state.itob.pipeline); 208 if (result != VK_SUCCESS) 209 goto fail; 210 211 if (device->physical_device->rad_info.chip_class >= GFX9) { 212 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = { 213 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 214 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 215 .module = radv_shader_module_to_handle(&cs_3d), 216 .pName = "main", 217 .pSpecializationInfo = NULL, 218 }; 219 220 VkComputePipelineCreateInfo vk_pipeline_info_3d = { 221 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 222 .stage = pipeline_shader_stage_3d, 223 .flags = 0, 224 .layout = device->meta_state.itob.img_p_layout, 225 }; 226 227 result = radv_CreateComputePipelines(radv_device_to_handle(device), 228 radv_pipeline_cache_to_handle(&device->meta_state.cache), 229 1, &vk_pipeline_info_3d, NULL, 230 &device->meta_state.itob.pipeline_3d); 231 if (result != VK_SUCCESS) 232 goto fail; 233 ralloc_free(cs_3d.nir); 234 } 235 ralloc_free(cs.nir); 236 237 return VK_SUCCESS; 238 fail: 239 ralloc_free(cs.nir); 240 ralloc_free(cs_3d.nir); 241 return result; 242 } 243 244 static void 245 radv_device_finish_meta_itob_state(struct radv_device *device) 246 { 247 struct radv_meta_state *state = &device->meta_state; 248 249 radv_DestroyPipelineLayout(radv_device_to_handle(device), 250 state->itob.img_p_layout, &state->alloc); 251 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device), 252 state->itob.img_ds_layout, 253 &state->alloc); 254 radv_DestroyPipeline(radv_device_to_handle(device), 255 state->itob.pipeline, &state->alloc); 256 if (device->physical_device->rad_info.chip_class >= GFX9) 257 radv_DestroyPipeline(radv_device_to_handle(device), 258 state->itob.pipeline_3d, &state->alloc); 259 } 260 261 static nir_shader * 262 build_nir_btoi_compute_shader(struct radv_device *dev, bool is_3d) 263 { 264 nir_builder b; 265 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D; 266 const struct glsl_type *buf_type = glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, 267 false, 268 false, 269 GLSL_TYPE_FLOAT); 270 const struct glsl_type *img_type = glsl_sampler_type(dim, 271 false, 272 false, 273 GLSL_TYPE_FLOAT); 274 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL); 275 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_btoi_cs_3d" : "meta_btoi_cs"); 276 b.shader->info.cs.local_size[0] = 16; 277 b.shader->info.cs.local_size[1] = 16; 278 b.shader->info.cs.local_size[2] = 1; 279 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform, 280 buf_type, "s_tex"); 281 input_img->data.descriptor_set = 0; 282 input_img->data.binding = 0; 283 284 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform, 285 img_type, "out_img"); 286 output_img->data.descriptor_set = 0; 287 output_img->data.binding = 1; 288 289 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0); 290 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0); 291 nir_ssa_def *block_size = nir_imm_ivec4(&b, 292 b.shader->info.cs.local_size[0], 293 b.shader->info.cs.local_size[1], 294 b.shader->info.cs.local_size[2], 0); 295 296 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id); 297 298 nir_intrinsic_instr *offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 299 nir_intrinsic_set_base(offset, 0); 300 nir_intrinsic_set_range(offset, 16); 301 offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0)); 302 offset->num_components = is_3d ? 3 : 2; 303 nir_ssa_dest_init(&offset->instr, &offset->dest, is_3d ? 3 : 2, 32, "offset"); 304 nir_builder_instr_insert(&b, &offset->instr); 305 306 nir_intrinsic_instr *stride = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 307 nir_intrinsic_set_base(stride, 0); 308 nir_intrinsic_set_range(stride, 16); 309 stride->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12)); 310 stride->num_components = 1; 311 nir_ssa_dest_init(&stride->instr, &stride->dest, 1, 32, "stride"); 312 nir_builder_instr_insert(&b, &stride->instr); 313 314 nir_ssa_def *pos_x = nir_channel(&b, global_id, 0); 315 nir_ssa_def *pos_y = nir_channel(&b, global_id, 1); 316 317 nir_ssa_def *tmp = nir_imul(&b, pos_y, &stride->dest.ssa); 318 tmp = nir_iadd(&b, tmp, pos_x); 319 320 nir_ssa_def *buf_coord = nir_vec4(&b, tmp, tmp, tmp, tmp); 321 322 nir_ssa_def *img_coord = nir_iadd(&b, global_id, &offset->dest.ssa); 323 324 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2); 325 tex->sampler_dim = GLSL_SAMPLER_DIM_BUF; 326 tex->op = nir_texop_txf; 327 tex->src[0].src_type = nir_tex_src_coord; 328 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, buf_coord, 1)); 329 tex->src[1].src_type = nir_tex_src_lod; 330 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0)); 331 tex->dest_type = nir_type_float; 332 tex->is_array = false; 333 tex->coord_components = 1; 334 tex->texture = nir_deref_var_create(tex, input_img); 335 tex->sampler = NULL; 336 337 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex"); 338 nir_builder_instr_insert(&b, &tex->instr); 339 340 nir_ssa_def *outval = &tex->dest.ssa; 341 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store); 342 store->src[0] = nir_src_for_ssa(img_coord); 343 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32)); 344 store->src[2] = nir_src_for_ssa(outval); 345 store->variables[0] = nir_deref_var_create(store, output_img); 346 347 nir_builder_instr_insert(&b, &store->instr); 348 return b.shader; 349 } 350 351 /* Buffer to image - don't write use image accessors */ 352 static VkResult 353 radv_device_init_meta_btoi_state(struct radv_device *device) 354 { 355 VkResult result; 356 struct radv_shader_module cs = { .nir = NULL }; 357 struct radv_shader_module cs_3d = { .nir = NULL }; 358 cs.nir = build_nir_btoi_compute_shader(device, false); 359 if (device->physical_device->rad_info.chip_class >= GFX9) 360 cs_3d.nir = build_nir_btoi_compute_shader(device, true); 361 /* 362 * two descriptors one for the image being sampled 363 * one for the buffer being written. 364 */ 365 VkDescriptorSetLayoutCreateInfo ds_create_info = { 366 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 367 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, 368 .bindingCount = 2, 369 .pBindings = (VkDescriptorSetLayoutBinding[]) { 370 { 371 .binding = 0, 372 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 373 .descriptorCount = 1, 374 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 375 .pImmutableSamplers = NULL 376 }, 377 { 378 .binding = 1, 379 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 380 .descriptorCount = 1, 381 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 382 .pImmutableSamplers = NULL 383 }, 384 } 385 }; 386 387 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device), 388 &ds_create_info, 389 &device->meta_state.alloc, 390 &device->meta_state.btoi.img_ds_layout); 391 if (result != VK_SUCCESS) 392 goto fail; 393 394 395 VkPipelineLayoutCreateInfo pl_create_info = { 396 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 397 .setLayoutCount = 1, 398 .pSetLayouts = &device->meta_state.btoi.img_ds_layout, 399 .pushConstantRangeCount = 1, 400 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 16}, 401 }; 402 403 result = radv_CreatePipelineLayout(radv_device_to_handle(device), 404 &pl_create_info, 405 &device->meta_state.alloc, 406 &device->meta_state.btoi.img_p_layout); 407 if (result != VK_SUCCESS) 408 goto fail; 409 410 /* compute shader */ 411 412 VkPipelineShaderStageCreateInfo pipeline_shader_stage = { 413 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 414 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 415 .module = radv_shader_module_to_handle(&cs), 416 .pName = "main", 417 .pSpecializationInfo = NULL, 418 }; 419 420 VkComputePipelineCreateInfo vk_pipeline_info = { 421 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 422 .stage = pipeline_shader_stage, 423 .flags = 0, 424 .layout = device->meta_state.btoi.img_p_layout, 425 }; 426 427 result = radv_CreateComputePipelines(radv_device_to_handle(device), 428 radv_pipeline_cache_to_handle(&device->meta_state.cache), 429 1, &vk_pipeline_info, NULL, 430 &device->meta_state.btoi.pipeline); 431 if (result != VK_SUCCESS) 432 goto fail; 433 434 if (device->physical_device->rad_info.chip_class >= GFX9) { 435 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = { 436 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 437 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 438 .module = radv_shader_module_to_handle(&cs_3d), 439 .pName = "main", 440 .pSpecializationInfo = NULL, 441 }; 442 443 VkComputePipelineCreateInfo vk_pipeline_info_3d = { 444 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 445 .stage = pipeline_shader_stage_3d, 446 .flags = 0, 447 .layout = device->meta_state.btoi.img_p_layout, 448 }; 449 450 result = radv_CreateComputePipelines(radv_device_to_handle(device), 451 radv_pipeline_cache_to_handle(&device->meta_state.cache), 452 1, &vk_pipeline_info_3d, NULL, 453 &device->meta_state.btoi.pipeline_3d); 454 ralloc_free(cs_3d.nir); 455 } 456 ralloc_free(cs.nir); 457 458 return VK_SUCCESS; 459 fail: 460 ralloc_free(cs_3d.nir); 461 ralloc_free(cs.nir); 462 return result; 463 } 464 465 static void 466 radv_device_finish_meta_btoi_state(struct radv_device *device) 467 { 468 struct radv_meta_state *state = &device->meta_state; 469 470 radv_DestroyPipelineLayout(radv_device_to_handle(device), 471 state->btoi.img_p_layout, &state->alloc); 472 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device), 473 state->btoi.img_ds_layout, 474 &state->alloc); 475 radv_DestroyPipeline(radv_device_to_handle(device), 476 state->btoi.pipeline, &state->alloc); 477 radv_DestroyPipeline(radv_device_to_handle(device), 478 state->btoi.pipeline_3d, &state->alloc); 479 } 480 481 static nir_shader * 482 build_nir_itoi_compute_shader(struct radv_device *dev, bool is_3d) 483 { 484 nir_builder b; 485 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D; 486 const struct glsl_type *buf_type = glsl_sampler_type(dim, 487 false, 488 false, 489 GLSL_TYPE_FLOAT); 490 const struct glsl_type *img_type = glsl_sampler_type(dim, 491 false, 492 false, 493 GLSL_TYPE_FLOAT); 494 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL); 495 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_itoi_cs_3d" : "meta_itoi_cs"); 496 b.shader->info.cs.local_size[0] = 16; 497 b.shader->info.cs.local_size[1] = 16; 498 b.shader->info.cs.local_size[2] = 1; 499 nir_variable *input_img = nir_variable_create(b.shader, nir_var_uniform, 500 buf_type, "s_tex"); 501 input_img->data.descriptor_set = 0; 502 input_img->data.binding = 0; 503 504 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform, 505 img_type, "out_img"); 506 output_img->data.descriptor_set = 0; 507 output_img->data.binding = 1; 508 509 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0); 510 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0); 511 nir_ssa_def *block_size = nir_imm_ivec4(&b, 512 b.shader->info.cs.local_size[0], 513 b.shader->info.cs.local_size[1], 514 b.shader->info.cs.local_size[2], 0); 515 516 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id); 517 518 nir_intrinsic_instr *src_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 519 nir_intrinsic_set_base(src_offset, 0); 520 nir_intrinsic_set_range(src_offset, 24); 521 src_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0)); 522 src_offset->num_components = is_3d ? 3 : 2; 523 nir_ssa_dest_init(&src_offset->instr, &src_offset->dest, is_3d ? 3 : 2, 32, "src_offset"); 524 nir_builder_instr_insert(&b, &src_offset->instr); 525 526 nir_intrinsic_instr *dst_offset = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 527 nir_intrinsic_set_base(dst_offset, 0); 528 nir_intrinsic_set_range(dst_offset, 24); 529 dst_offset->src[0] = nir_src_for_ssa(nir_imm_int(&b, 12)); 530 dst_offset->num_components = is_3d ? 3 : 2; 531 nir_ssa_dest_init(&dst_offset->instr, &dst_offset->dest, is_3d ? 3 : 2, 32, "dst_offset"); 532 nir_builder_instr_insert(&b, &dst_offset->instr); 533 534 nir_ssa_def *src_coord = nir_iadd(&b, global_id, &src_offset->dest.ssa); 535 536 nir_ssa_def *dst_coord = nir_iadd(&b, global_id, &dst_offset->dest.ssa); 537 538 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 2); 539 tex->sampler_dim = dim; 540 tex->op = nir_texop_txf; 541 tex->src[0].src_type = nir_tex_src_coord; 542 tex->src[0].src = nir_src_for_ssa(nir_channels(&b, src_coord, is_3d ? 0x7 : 0x3)); 543 tex->src[1].src_type = nir_tex_src_lod; 544 tex->src[1].src = nir_src_for_ssa(nir_imm_int(&b, 0)); 545 tex->dest_type = nir_type_float; 546 tex->is_array = false; 547 tex->coord_components = is_3d ? 3 : 2; 548 tex->texture = nir_deref_var_create(tex, input_img); 549 tex->sampler = NULL; 550 551 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex"); 552 nir_builder_instr_insert(&b, &tex->instr); 553 554 nir_ssa_def *outval = &tex->dest.ssa; 555 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store); 556 store->src[0] = nir_src_for_ssa(dst_coord); 557 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32)); 558 store->src[2] = nir_src_for_ssa(outval); 559 store->variables[0] = nir_deref_var_create(store, output_img); 560 561 nir_builder_instr_insert(&b, &store->instr); 562 return b.shader; 563 } 564 565 /* image to image - don't write use image accessors */ 566 static VkResult 567 radv_device_init_meta_itoi_state(struct radv_device *device) 568 { 569 VkResult result; 570 struct radv_shader_module cs = { .nir = NULL }; 571 struct radv_shader_module cs_3d = { .nir = NULL }; 572 cs.nir = build_nir_itoi_compute_shader(device, false); 573 if (device->physical_device->rad_info.chip_class >= GFX9) 574 cs_3d.nir = build_nir_itoi_compute_shader(device, true); 575 /* 576 * two descriptors one for the image being sampled 577 * one for the buffer being written. 578 */ 579 VkDescriptorSetLayoutCreateInfo ds_create_info = { 580 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 581 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, 582 .bindingCount = 2, 583 .pBindings = (VkDescriptorSetLayoutBinding[]) { 584 { 585 .binding = 0, 586 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 587 .descriptorCount = 1, 588 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 589 .pImmutableSamplers = NULL 590 }, 591 { 592 .binding = 1, 593 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 594 .descriptorCount = 1, 595 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 596 .pImmutableSamplers = NULL 597 }, 598 } 599 }; 600 601 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device), 602 &ds_create_info, 603 &device->meta_state.alloc, 604 &device->meta_state.itoi.img_ds_layout); 605 if (result != VK_SUCCESS) 606 goto fail; 607 608 609 VkPipelineLayoutCreateInfo pl_create_info = { 610 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 611 .setLayoutCount = 1, 612 .pSetLayouts = &device->meta_state.itoi.img_ds_layout, 613 .pushConstantRangeCount = 1, 614 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 24}, 615 }; 616 617 result = radv_CreatePipelineLayout(radv_device_to_handle(device), 618 &pl_create_info, 619 &device->meta_state.alloc, 620 &device->meta_state.itoi.img_p_layout); 621 if (result != VK_SUCCESS) 622 goto fail; 623 624 /* compute shader */ 625 626 VkPipelineShaderStageCreateInfo pipeline_shader_stage = { 627 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 628 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 629 .module = radv_shader_module_to_handle(&cs), 630 .pName = "main", 631 .pSpecializationInfo = NULL, 632 }; 633 634 VkComputePipelineCreateInfo vk_pipeline_info = { 635 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 636 .stage = pipeline_shader_stage, 637 .flags = 0, 638 .layout = device->meta_state.itoi.img_p_layout, 639 }; 640 641 result = radv_CreateComputePipelines(radv_device_to_handle(device), 642 radv_pipeline_cache_to_handle(&device->meta_state.cache), 643 1, &vk_pipeline_info, NULL, 644 &device->meta_state.itoi.pipeline); 645 if (result != VK_SUCCESS) 646 goto fail; 647 648 if (device->physical_device->rad_info.chip_class >= GFX9) { 649 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = { 650 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 651 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 652 .module = radv_shader_module_to_handle(&cs_3d), 653 .pName = "main", 654 .pSpecializationInfo = NULL, 655 }; 656 657 VkComputePipelineCreateInfo vk_pipeline_info_3d = { 658 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 659 .stage = pipeline_shader_stage_3d, 660 .flags = 0, 661 .layout = device->meta_state.itoi.img_p_layout, 662 }; 663 664 result = radv_CreateComputePipelines(radv_device_to_handle(device), 665 radv_pipeline_cache_to_handle(&device->meta_state.cache), 666 1, &vk_pipeline_info_3d, NULL, 667 &device->meta_state.itoi.pipeline_3d); 668 669 ralloc_free(cs_3d.nir); 670 } 671 ralloc_free(cs.nir); 672 673 return VK_SUCCESS; 674 fail: 675 ralloc_free(cs.nir); 676 ralloc_free(cs_3d.nir); 677 return result; 678 } 679 680 static void 681 radv_device_finish_meta_itoi_state(struct radv_device *device) 682 { 683 struct radv_meta_state *state = &device->meta_state; 684 685 radv_DestroyPipelineLayout(radv_device_to_handle(device), 686 state->itoi.img_p_layout, &state->alloc); 687 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device), 688 state->itoi.img_ds_layout, 689 &state->alloc); 690 radv_DestroyPipeline(radv_device_to_handle(device), 691 state->itoi.pipeline, &state->alloc); 692 if (device->physical_device->rad_info.chip_class >= GFX9) 693 radv_DestroyPipeline(radv_device_to_handle(device), 694 state->itoi.pipeline_3d, &state->alloc); 695 } 696 697 static nir_shader * 698 build_nir_cleari_compute_shader(struct radv_device *dev, bool is_3d) 699 { 700 nir_builder b; 701 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D : GLSL_SAMPLER_DIM_2D; 702 const struct glsl_type *img_type = glsl_sampler_type(dim, 703 false, 704 false, 705 GLSL_TYPE_FLOAT); 706 nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_COMPUTE, NULL); 707 b.shader->info.name = ralloc_strdup(b.shader, is_3d ? "meta_cleari_cs_3d" : "meta_cleari_cs"); 708 b.shader->info.cs.local_size[0] = 16; 709 b.shader->info.cs.local_size[1] = 16; 710 b.shader->info.cs.local_size[2] = 1; 711 712 nir_variable *output_img = nir_variable_create(b.shader, nir_var_uniform, 713 img_type, "out_img"); 714 output_img->data.descriptor_set = 0; 715 output_img->data.binding = 0; 716 717 nir_ssa_def *invoc_id = nir_load_system_value(&b, nir_intrinsic_load_local_invocation_id, 0); 718 nir_ssa_def *wg_id = nir_load_system_value(&b, nir_intrinsic_load_work_group_id, 0); 719 nir_ssa_def *block_size = nir_imm_ivec4(&b, 720 b.shader->info.cs.local_size[0], 721 b.shader->info.cs.local_size[1], 722 b.shader->info.cs.local_size[2], 0); 723 724 nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id); 725 726 nir_intrinsic_instr *clear_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 727 nir_intrinsic_set_base(clear_val, 0); 728 nir_intrinsic_set_range(clear_val, 20); 729 clear_val->src[0] = nir_src_for_ssa(nir_imm_int(&b, 0)); 730 clear_val->num_components = 4; 731 nir_ssa_dest_init(&clear_val->instr, &clear_val->dest, 4, 32, "clear_value"); 732 nir_builder_instr_insert(&b, &clear_val->instr); 733 734 nir_intrinsic_instr *layer = nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_push_constant); 735 nir_intrinsic_set_base(layer, 0); 736 nir_intrinsic_set_range(layer, 20); 737 layer->src[0] = nir_src_for_ssa(nir_imm_int(&b, 16)); 738 layer->num_components = 1; 739 nir_ssa_dest_init(&layer->instr, &layer->dest, 1, 32, "layer"); 740 nir_builder_instr_insert(&b, &layer->instr); 741 742 nir_ssa_def *global_z = nir_iadd(&b, nir_channel(&b, global_id, 2), &layer->dest.ssa); 743 744 nir_ssa_def *comps[4]; 745 comps[0] = nir_channel(&b, global_id, 0); 746 comps[1] = nir_channel(&b, global_id, 1); 747 comps[2] = global_z; 748 comps[3] = nir_imm_int(&b, 0); 749 global_id = nir_vec(&b, comps, 4); 750 751 nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_store); 752 store->src[0] = nir_src_for_ssa(global_id); 753 store->src[1] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32)); 754 store->src[2] = nir_src_for_ssa(&clear_val->dest.ssa); 755 store->variables[0] = nir_deref_var_create(store, output_img); 756 757 nir_builder_instr_insert(&b, &store->instr); 758 return b.shader; 759 } 760 761 static VkResult 762 radv_device_init_meta_cleari_state(struct radv_device *device) 763 { 764 VkResult result; 765 struct radv_shader_module cs = { .nir = NULL }; 766 struct radv_shader_module cs_3d = { .nir = NULL }; 767 cs.nir = build_nir_cleari_compute_shader(device, false); 768 if (device->physical_device->rad_info.chip_class >= GFX9) 769 cs_3d.nir = build_nir_cleari_compute_shader(device, true); 770 771 /* 772 * two descriptors one for the image being sampled 773 * one for the buffer being written. 774 */ 775 VkDescriptorSetLayoutCreateInfo ds_create_info = { 776 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 777 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, 778 .bindingCount = 1, 779 .pBindings = (VkDescriptorSetLayoutBinding[]) { 780 { 781 .binding = 0, 782 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 783 .descriptorCount = 1, 784 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, 785 .pImmutableSamplers = NULL 786 }, 787 } 788 }; 789 790 result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device), 791 &ds_create_info, 792 &device->meta_state.alloc, 793 &device->meta_state.cleari.img_ds_layout); 794 if (result != VK_SUCCESS) 795 goto fail; 796 797 798 VkPipelineLayoutCreateInfo pl_create_info = { 799 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 800 .setLayoutCount = 1, 801 .pSetLayouts = &device->meta_state.cleari.img_ds_layout, 802 .pushConstantRangeCount = 1, 803 .pPushConstantRanges = &(VkPushConstantRange){VK_SHADER_STAGE_COMPUTE_BIT, 0, 20}, 804 }; 805 806 result = radv_CreatePipelineLayout(radv_device_to_handle(device), 807 &pl_create_info, 808 &device->meta_state.alloc, 809 &device->meta_state.cleari.img_p_layout); 810 if (result != VK_SUCCESS) 811 goto fail; 812 813 /* compute shader */ 814 815 VkPipelineShaderStageCreateInfo pipeline_shader_stage = { 816 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 817 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 818 .module = radv_shader_module_to_handle(&cs), 819 .pName = "main", 820 .pSpecializationInfo = NULL, 821 }; 822 823 VkComputePipelineCreateInfo vk_pipeline_info = { 824 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 825 .stage = pipeline_shader_stage, 826 .flags = 0, 827 .layout = device->meta_state.cleari.img_p_layout, 828 }; 829 830 result = radv_CreateComputePipelines(radv_device_to_handle(device), 831 radv_pipeline_cache_to_handle(&device->meta_state.cache), 832 1, &vk_pipeline_info, NULL, 833 &device->meta_state.cleari.pipeline); 834 if (result != VK_SUCCESS) 835 goto fail; 836 837 838 if (device->physical_device->rad_info.chip_class >= GFX9) { 839 /* compute shader */ 840 VkPipelineShaderStageCreateInfo pipeline_shader_stage_3d = { 841 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 842 .stage = VK_SHADER_STAGE_COMPUTE_BIT, 843 .module = radv_shader_module_to_handle(&cs_3d), 844 .pName = "main", 845 .pSpecializationInfo = NULL, 846 }; 847 848 VkComputePipelineCreateInfo vk_pipeline_info_3d = { 849 .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, 850 .stage = pipeline_shader_stage_3d, 851 .flags = 0, 852 .layout = device->meta_state.cleari.img_p_layout, 853 }; 854 855 result = radv_CreateComputePipelines(radv_device_to_handle(device), 856 radv_pipeline_cache_to_handle(&device->meta_state.cache), 857 1, &vk_pipeline_info_3d, NULL, 858 &device->meta_state.cleari.pipeline_3d); 859 if (result != VK_SUCCESS) 860 goto fail; 861 862 ralloc_free(cs_3d.nir); 863 } 864 ralloc_free(cs.nir); 865 return VK_SUCCESS; 866 fail: 867 ralloc_free(cs.nir); 868 ralloc_free(cs_3d.nir); 869 return result; 870 } 871 872 static void 873 radv_device_finish_meta_cleari_state(struct radv_device *device) 874 { 875 struct radv_meta_state *state = &device->meta_state; 876 877 radv_DestroyPipelineLayout(radv_device_to_handle(device), 878 state->cleari.img_p_layout, &state->alloc); 879 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device), 880 state->cleari.img_ds_layout, 881 &state->alloc); 882 radv_DestroyPipeline(radv_device_to_handle(device), 883 state->cleari.pipeline, &state->alloc); 884 radv_DestroyPipeline(radv_device_to_handle(device), 885 state->cleari.pipeline_3d, &state->alloc); 886 } 887 888 void 889 radv_device_finish_meta_bufimage_state(struct radv_device *device) 890 { 891 radv_device_finish_meta_itob_state(device); 892 radv_device_finish_meta_btoi_state(device); 893 radv_device_finish_meta_itoi_state(device); 894 radv_device_finish_meta_cleari_state(device); 895 } 896 897 VkResult 898 radv_device_init_meta_bufimage_state(struct radv_device *device) 899 { 900 VkResult result; 901 902 result = radv_device_init_meta_itob_state(device); 903 if (result != VK_SUCCESS) 904 goto fail_itob; 905 906 result = radv_device_init_meta_btoi_state(device); 907 if (result != VK_SUCCESS) 908 goto fail_btoi; 909 910 result = radv_device_init_meta_itoi_state(device); 911 if (result != VK_SUCCESS) 912 goto fail_itoi; 913 914 result = radv_device_init_meta_cleari_state(device); 915 if (result != VK_SUCCESS) 916 goto fail_cleari; 917 918 return VK_SUCCESS; 919 fail_cleari: 920 radv_device_finish_meta_cleari_state(device); 921 fail_itoi: 922 radv_device_finish_meta_itoi_state(device); 923 fail_btoi: 924 radv_device_finish_meta_btoi_state(device); 925 fail_itob: 926 radv_device_finish_meta_itob_state(device); 927 return result; 928 } 929 930 static void 931 create_iview(struct radv_cmd_buffer *cmd_buffer, 932 struct radv_meta_blit2d_surf *surf, 933 struct radv_image_view *iview) 934 { 935 VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 ? VK_IMAGE_VIEW_TYPE_2D : 936 radv_meta_get_view_type(surf->image); 937 radv_image_view_init(iview, cmd_buffer->device, 938 &(VkImageViewCreateInfo) { 939 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, 940 .image = radv_image_to_handle(surf->image), 941 .viewType = view_type, 942 .format = surf->format, 943 .subresourceRange = { 944 .aspectMask = surf->aspect_mask, 945 .baseMipLevel = surf->level, 946 .levelCount = 1, 947 .baseArrayLayer = surf->layer, 948 .layerCount = 1 949 }, 950 }); 951 } 952 953 static void 954 create_bview(struct radv_cmd_buffer *cmd_buffer, 955 struct radv_buffer *buffer, 956 unsigned offset, 957 VkFormat format, 958 struct radv_buffer_view *bview) 959 { 960 radv_buffer_view_init(bview, cmd_buffer->device, 961 &(VkBufferViewCreateInfo) { 962 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, 963 .flags = 0, 964 .buffer = radv_buffer_to_handle(buffer), 965 .format = format, 966 .offset = offset, 967 .range = VK_WHOLE_SIZE, 968 }); 969 970 } 971 972 static void 973 itob_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, 974 struct radv_image_view *src, 975 struct radv_buffer_view *dst) 976 { 977 struct radv_device *device = cmd_buffer->device; 978 979 radv_meta_push_descriptor_set(cmd_buffer, 980 VK_PIPELINE_BIND_POINT_COMPUTE, 981 device->meta_state.itob.img_p_layout, 982 0, /* set */ 983 2, /* descriptorWriteCount */ 984 (VkWriteDescriptorSet[]) { 985 { 986 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 987 .dstBinding = 0, 988 .dstArrayElement = 0, 989 .descriptorCount = 1, 990 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 991 .pImageInfo = (VkDescriptorImageInfo[]) { 992 { 993 .sampler = VK_NULL_HANDLE, 994 .imageView = radv_image_view_to_handle(src), 995 .imageLayout = VK_IMAGE_LAYOUT_GENERAL, 996 }, 997 } 998 }, 999 { 1000 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 1001 .dstBinding = 1, 1002 .dstArrayElement = 0, 1003 .descriptorCount = 1, 1004 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 1005 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(dst) }, 1006 } 1007 }); 1008 } 1009 1010 void 1011 radv_meta_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, 1012 struct radv_meta_blit2d_surf *src, 1013 struct radv_meta_blit2d_buffer *dst, 1014 unsigned num_rects, 1015 struct radv_meta_blit2d_rect *rects) 1016 { 1017 VkPipeline pipeline = cmd_buffer->device->meta_state.itob.pipeline; 1018 struct radv_device *device = cmd_buffer->device; 1019 struct radv_image_view src_view; 1020 struct radv_buffer_view dst_view; 1021 1022 create_iview(cmd_buffer, src, &src_view); 1023 create_bview(cmd_buffer, dst->buffer, dst->offset, dst->format, &dst_view); 1024 itob_bind_descriptors(cmd_buffer, &src_view, &dst_view); 1025 1026 if (device->physical_device->rad_info.chip_class >= GFX9 && 1027 src->image->type == VK_IMAGE_TYPE_3D) 1028 pipeline = cmd_buffer->device->meta_state.itob.pipeline_3d; 1029 1030 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), 1031 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); 1032 1033 for (unsigned r = 0; r < num_rects; ++r) { 1034 unsigned push_constants[4] = { 1035 rects[r].src_x, 1036 rects[r].src_y, 1037 src->layer, 1038 dst->pitch 1039 }; 1040 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 1041 device->meta_state.itob.img_p_layout, 1042 VK_SHADER_STAGE_COMPUTE_BIT, 0, 16, 1043 push_constants); 1044 1045 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1); 1046 } 1047 } 1048 1049 static void 1050 btoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, 1051 struct radv_buffer_view *src, 1052 struct radv_image_view *dst) 1053 { 1054 struct radv_device *device = cmd_buffer->device; 1055 1056 radv_meta_push_descriptor_set(cmd_buffer, 1057 VK_PIPELINE_BIND_POINT_COMPUTE, 1058 device->meta_state.btoi.img_p_layout, 1059 0, /* set */ 1060 2, /* descriptorWriteCount */ 1061 (VkWriteDescriptorSet[]) { 1062 { 1063 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 1064 .dstBinding = 0, 1065 .dstArrayElement = 0, 1066 .descriptorCount = 1, 1067 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER, 1068 .pTexelBufferView = (VkBufferView[]) { radv_buffer_view_to_handle(src) }, 1069 }, 1070 { 1071 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 1072 .dstBinding = 1, 1073 .dstArrayElement = 0, 1074 .descriptorCount = 1, 1075 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1076 .pImageInfo = (VkDescriptorImageInfo[]) { 1077 { 1078 .sampler = VK_NULL_HANDLE, 1079 .imageView = radv_image_view_to_handle(dst), 1080 .imageLayout = VK_IMAGE_LAYOUT_GENERAL, 1081 }, 1082 } 1083 } 1084 }); 1085 } 1086 1087 void 1088 radv_meta_buffer_to_image_cs(struct radv_cmd_buffer *cmd_buffer, 1089 struct radv_meta_blit2d_buffer *src, 1090 struct radv_meta_blit2d_surf *dst, 1091 unsigned num_rects, 1092 struct radv_meta_blit2d_rect *rects) 1093 { 1094 VkPipeline pipeline = cmd_buffer->device->meta_state.btoi.pipeline; 1095 struct radv_device *device = cmd_buffer->device; 1096 struct radv_buffer_view src_view; 1097 struct radv_image_view dst_view; 1098 1099 create_bview(cmd_buffer, src->buffer, src->offset, src->format, &src_view); 1100 create_iview(cmd_buffer, dst, &dst_view); 1101 btoi_bind_descriptors(cmd_buffer, &src_view, &dst_view); 1102 1103 if (device->physical_device->rad_info.chip_class >= GFX9 && 1104 dst->image->type == VK_IMAGE_TYPE_3D) 1105 pipeline = cmd_buffer->device->meta_state.btoi.pipeline_3d; 1106 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), 1107 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); 1108 1109 for (unsigned r = 0; r < num_rects; ++r) { 1110 unsigned push_constants[4] = { 1111 rects[r].dst_x, 1112 rects[r].dst_y, 1113 dst->layer, 1114 src->pitch, 1115 }; 1116 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 1117 device->meta_state.btoi.img_p_layout, 1118 VK_SHADER_STAGE_COMPUTE_BIT, 0, 16, 1119 push_constants); 1120 1121 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1); 1122 } 1123 } 1124 1125 static void 1126 itoi_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, 1127 struct radv_image_view *src, 1128 struct radv_image_view *dst) 1129 { 1130 struct radv_device *device = cmd_buffer->device; 1131 1132 radv_meta_push_descriptor_set(cmd_buffer, 1133 VK_PIPELINE_BIND_POINT_COMPUTE, 1134 device->meta_state.itoi.img_p_layout, 1135 0, /* set */ 1136 2, /* descriptorWriteCount */ 1137 (VkWriteDescriptorSet[]) { 1138 { 1139 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 1140 .dstBinding = 0, 1141 .dstArrayElement = 0, 1142 .descriptorCount = 1, 1143 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1144 .pImageInfo = (VkDescriptorImageInfo[]) { 1145 { 1146 .sampler = VK_NULL_HANDLE, 1147 .imageView = radv_image_view_to_handle(src), 1148 .imageLayout = VK_IMAGE_LAYOUT_GENERAL, 1149 }, 1150 } 1151 }, 1152 { 1153 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 1154 .dstBinding = 1, 1155 .dstArrayElement = 0, 1156 .descriptorCount = 1, 1157 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1158 .pImageInfo = (VkDescriptorImageInfo[]) { 1159 { 1160 .sampler = VK_NULL_HANDLE, 1161 .imageView = radv_image_view_to_handle(dst), 1162 .imageLayout = VK_IMAGE_LAYOUT_GENERAL, 1163 }, 1164 } 1165 } 1166 }); 1167 } 1168 1169 void 1170 radv_meta_image_to_image_cs(struct radv_cmd_buffer *cmd_buffer, 1171 struct radv_meta_blit2d_surf *src, 1172 struct radv_meta_blit2d_surf *dst, 1173 unsigned num_rects, 1174 struct radv_meta_blit2d_rect *rects) 1175 { 1176 VkPipeline pipeline = cmd_buffer->device->meta_state.itoi.pipeline; 1177 struct radv_device *device = cmd_buffer->device; 1178 struct radv_image_view src_view, dst_view; 1179 1180 create_iview(cmd_buffer, src, &src_view); 1181 create_iview(cmd_buffer, dst, &dst_view); 1182 1183 itoi_bind_descriptors(cmd_buffer, &src_view, &dst_view); 1184 1185 if (device->physical_device->rad_info.chip_class >= GFX9 && 1186 src->image->type == VK_IMAGE_TYPE_3D) 1187 pipeline = cmd_buffer->device->meta_state.itoi.pipeline_3d; 1188 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), 1189 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); 1190 1191 for (unsigned r = 0; r < num_rects; ++r) { 1192 unsigned push_constants[6] = { 1193 rects[r].src_x, 1194 rects[r].src_y, 1195 src->layer, 1196 rects[r].dst_x, 1197 rects[r].dst_y, 1198 dst->layer, 1199 }; 1200 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 1201 device->meta_state.itoi.img_p_layout, 1202 VK_SHADER_STAGE_COMPUTE_BIT, 0, 24, 1203 push_constants); 1204 1205 radv_unaligned_dispatch(cmd_buffer, rects[r].width, rects[r].height, 1); 1206 } 1207 } 1208 1209 static void 1210 cleari_bind_descriptors(struct radv_cmd_buffer *cmd_buffer, 1211 struct radv_image_view *dst_iview) 1212 { 1213 struct radv_device *device = cmd_buffer->device; 1214 1215 radv_meta_push_descriptor_set(cmd_buffer, 1216 VK_PIPELINE_BIND_POINT_COMPUTE, 1217 device->meta_state.cleari.img_p_layout, 1218 0, /* set */ 1219 1, /* descriptorWriteCount */ 1220 (VkWriteDescriptorSet[]) { 1221 { 1222 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 1223 .dstBinding = 0, 1224 .dstArrayElement = 0, 1225 .descriptorCount = 1, 1226 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1227 .pImageInfo = (VkDescriptorImageInfo[]) { 1228 { 1229 .sampler = VK_NULL_HANDLE, 1230 .imageView = radv_image_view_to_handle(dst_iview), 1231 .imageLayout = VK_IMAGE_LAYOUT_GENERAL, 1232 }, 1233 } 1234 }, 1235 }); 1236 } 1237 1238 void 1239 radv_meta_clear_image_cs(struct radv_cmd_buffer *cmd_buffer, 1240 struct radv_meta_blit2d_surf *dst, 1241 const VkClearColorValue *clear_color) 1242 { 1243 VkPipeline pipeline = cmd_buffer->device->meta_state.cleari.pipeline; 1244 struct radv_device *device = cmd_buffer->device; 1245 struct radv_image_view dst_iview; 1246 1247 create_iview(cmd_buffer, dst, &dst_iview); 1248 cleari_bind_descriptors(cmd_buffer, &dst_iview); 1249 1250 if (device->physical_device->rad_info.chip_class >= GFX9 && 1251 dst->image->type == VK_IMAGE_TYPE_3D) 1252 pipeline = cmd_buffer->device->meta_state.cleari.pipeline_3d; 1253 1254 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), 1255 VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); 1256 1257 unsigned push_constants[5] = { 1258 clear_color->uint32[0], 1259 clear_color->uint32[1], 1260 clear_color->uint32[2], 1261 clear_color->uint32[3], 1262 dst->layer, 1263 }; 1264 1265 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 1266 device->meta_state.cleari.img_p_layout, 1267 VK_SHADER_STAGE_COMPUTE_BIT, 0, 20, 1268 push_constants); 1269 1270 radv_unaligned_dispatch(cmd_buffer, dst->image->info.width, dst->image->info.height, 1); 1271 } 1272
