1 /* 2 * Copyright 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include "radv_meta.h" 25 #include "vk_format.h" 26 27 static VkExtent3D 28 meta_image_block_size(const struct radv_image *image) 29 { 30 const struct vk_format_description *desc = vk_format_description(image->vk_format); 31 return (VkExtent3D) { desc->block.width, desc->block.height, 1 }; 32 } 33 34 /* Returns the user-provided VkBufferImageCopy::imageExtent in units of 35 * elements rather than texels. One element equals one texel or one block 36 * if Image is uncompressed or compressed, respectively. 37 */ 38 static struct VkExtent3D 39 meta_region_extent_el(const struct radv_image *image, 40 const struct VkExtent3D *extent) 41 { 42 const VkExtent3D block = meta_image_block_size(image); 43 return radv_sanitize_image_extent(image->type, (VkExtent3D) { 44 .width = DIV_ROUND_UP(extent->width , block.width), 45 .height = DIV_ROUND_UP(extent->height, block.height), 46 .depth = DIV_ROUND_UP(extent->depth , block.depth), 47 }); 48 } 49 50 /* Returns the user-provided VkBufferImageCopy::imageOffset in units of 51 * elements rather than texels. One element equals one texel or one block 52 * if Image is uncompressed or compressed, respectively. 53 */ 54 static struct VkOffset3D 55 meta_region_offset_el(const struct radv_image *image, 56 const struct VkOffset3D *offset) 57 { 58 const VkExtent3D block = meta_image_block_size(image); 59 return radv_sanitize_image_offset(image->type, (VkOffset3D) { 60 .x = offset->x / block.width, 61 .y = offset->y / block.height, 62 .z = offset->z / block.depth, 63 }); 64 } 65 66 static VkFormat 67 vk_format_for_size(int bs) 68 { 69 switch (bs) { 70 case 1: return VK_FORMAT_R8_UINT; 71 case 2: return VK_FORMAT_R8G8_UINT; 72 case 4: return VK_FORMAT_R8G8B8A8_UINT; 73 case 8: return VK_FORMAT_R16G16B16A16_UINT; 74 case 16: return VK_FORMAT_R32G32B32A32_UINT; 75 default: 76 unreachable("Invalid format block size"); 77 } 78 } 79 80 static struct radv_meta_blit2d_surf 81 blit_surf_for_image_level_layer(struct radv_image *image, 82 const VkImageSubresourceLayers *subres) 83 { 84 VkFormat format = image->vk_format; 85 if (subres->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) 86 format = vk_format_depth_only(format); 87 else if (subres->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) 88 format = vk_format_stencil_only(format); 89 90 if (!image->surface.dcc_size) 91 format = vk_format_for_size(vk_format_get_blocksize(format)); 92 93 return (struct radv_meta_blit2d_surf) { 94 .format = format, 95 .bs = vk_format_get_blocksize(format), 96 .level = subres->mipLevel, 97 .layer = subres->baseArrayLayer, 98 .image = image, 99 .aspect_mask = subres->aspectMask, 100 }; 101 } 102 103 union meta_saved_state { 104 struct radv_meta_saved_state gfx; 105 struct radv_meta_saved_compute_state compute; 106 }; 107 108 static void 109 meta_copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer, 110 struct radv_buffer* buffer, 111 struct radv_image* image, 112 uint32_t regionCount, 113 const VkBufferImageCopy* pRegions) 114 { 115 bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE; 116 union meta_saved_state saved_state; 117 118 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to 119 * VK_SAMPLE_COUNT_1_BIT." 120 */ 121 assert(image->samples == 1); 122 123 if (cs) 124 radv_meta_begin_bufimage(cmd_buffer, &saved_state.compute); 125 else 126 radv_meta_save_graphics_reset_vport_scissor(&saved_state.gfx, cmd_buffer); 127 128 for (unsigned r = 0; r < regionCount; r++) { 129 130 /** 131 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images 132 * extent is the size in texels of the source image to copy in width, 133 * height and depth. 1D images use only x and width. 2D images use x, y, 134 * width and height. 3D images use x, y, z, width, height and depth. 135 * 136 * 137 * Also, convert the offsets and extent from units of texels to units of 138 * blocks - which is the highest resolution accessible in this command. 139 */ 140 const VkOffset3D img_offset_el = 141 meta_region_offset_el(image, &pRegions[r].imageOffset); 142 const VkExtent3D bufferExtent = { 143 .width = pRegions[r].bufferRowLength ? 144 pRegions[r].bufferRowLength : pRegions[r].imageExtent.width, 145 .height = pRegions[r].bufferImageHeight ? 146 pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height, 147 }; 148 const VkExtent3D buf_extent_el = 149 meta_region_extent_el(image, &bufferExtent); 150 151 /* Start creating blit rect */ 152 const VkExtent3D img_extent_el = 153 meta_region_extent_el(image, &pRegions[r].imageExtent); 154 struct radv_meta_blit2d_rect rect = { 155 .width = img_extent_el.width, 156 .height = img_extent_el.height, 157 }; 158 159 /* Create blit surfaces */ 160 struct radv_meta_blit2d_surf img_bsurf = 161 blit_surf_for_image_level_layer(image, 162 &pRegions[r].imageSubresource); 163 164 struct radv_meta_blit2d_buffer buf_bsurf = { 165 .bs = img_bsurf.bs, 166 .format = img_bsurf.format, 167 .buffer = buffer, 168 .offset = pRegions[r].bufferOffset, 169 .pitch = buf_extent_el.width, 170 }; 171 172 /* Loop through each 3D or array slice */ 173 unsigned num_slices_3d = img_extent_el.depth; 174 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount; 175 unsigned slice_3d = 0; 176 unsigned slice_array = 0; 177 while (slice_3d < num_slices_3d && slice_array < num_slices_array) { 178 179 rect.dst_x = img_offset_el.x; 180 rect.dst_y = img_offset_el.y; 181 182 183 /* Perform Blit */ 184 if (cs) 185 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, 1, &rect); 186 else 187 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, 1, &rect); 188 189 /* Once we've done the blit, all of the actual information about 190 * the image is embedded in the command buffer so we can just 191 * increment the offset directly in the image effectively 192 * re-binding it to different backing memory. 193 */ 194 buf_bsurf.offset += buf_extent_el.width * 195 buf_extent_el.height * buf_bsurf.bs; 196 img_bsurf.layer++; 197 if (image->type == VK_IMAGE_TYPE_3D) 198 slice_3d++; 199 else 200 slice_array++; 201 } 202 } 203 if (cs) 204 radv_meta_end_bufimage(cmd_buffer, &saved_state.compute); 205 else 206 radv_meta_restore(&saved_state.gfx, cmd_buffer); 207 } 208 209 void radv_CmdCopyBufferToImage( 210 VkCommandBuffer commandBuffer, 211 VkBuffer srcBuffer, 212 VkImage destImage, 213 VkImageLayout destImageLayout, 214 uint32_t regionCount, 215 const VkBufferImageCopy* pRegions) 216 { 217 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); 218 RADV_FROM_HANDLE(radv_image, dest_image, destImage); 219 RADV_FROM_HANDLE(radv_buffer, src_buffer, srcBuffer); 220 221 meta_copy_buffer_to_image(cmd_buffer, src_buffer, dest_image, 222 regionCount, pRegions); 223 } 224 225 static void 226 meta_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, 227 struct radv_buffer* buffer, 228 struct radv_image* image, 229 uint32_t regionCount, 230 const VkBufferImageCopy* pRegions) 231 { 232 struct radv_meta_saved_compute_state saved_state; 233 234 radv_meta_begin_bufimage(cmd_buffer, &saved_state); 235 for (unsigned r = 0; r < regionCount; r++) { 236 237 /** 238 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images 239 * extent is the size in texels of the source image to copy in width, 240 * height and depth. 1D images use only x and width. 2D images use x, y, 241 * width and height. 3D images use x, y, z, width, height and depth. 242 * 243 * 244 * Also, convert the offsets and extent from units of texels to units of 245 * blocks - which is the highest resolution accessible in this command. 246 */ 247 const VkOffset3D img_offset_el = 248 meta_region_offset_el(image, &pRegions[r].imageOffset); 249 const VkExtent3D bufferExtent = { 250 .width = pRegions[r].bufferRowLength ? 251 pRegions[r].bufferRowLength : pRegions[r].imageExtent.width, 252 .height = pRegions[r].bufferImageHeight ? 253 pRegions[r].bufferImageHeight : pRegions[r].imageExtent.height, 254 }; 255 const VkExtent3D buf_extent_el = 256 meta_region_extent_el(image, &bufferExtent); 257 258 /* Start creating blit rect */ 259 const VkExtent3D img_extent_el = 260 meta_region_extent_el(image, &pRegions[r].imageExtent); 261 struct radv_meta_blit2d_rect rect = { 262 .width = img_extent_el.width, 263 .height = img_extent_el.height, 264 }; 265 266 /* Create blit surfaces */ 267 struct radv_meta_blit2d_surf img_info = 268 blit_surf_for_image_level_layer(image, 269 &pRegions[r].imageSubresource); 270 271 struct radv_meta_blit2d_buffer buf_info = { 272 .bs = img_info.bs, 273 .format = img_info.format, 274 .buffer = buffer, 275 .offset = pRegions[r].bufferOffset, 276 .pitch = buf_extent_el.width, 277 }; 278 279 /* Loop through each 3D or array slice */ 280 unsigned num_slices_3d = img_extent_el.depth; 281 unsigned num_slices_array = pRegions[r].imageSubresource.layerCount; 282 unsigned slice_3d = 0; 283 unsigned slice_array = 0; 284 while (slice_3d < num_slices_3d && slice_array < num_slices_array) { 285 286 rect.src_x = img_offset_el.x; 287 rect.src_y = img_offset_el.y; 288 289 290 /* Perform Blit */ 291 radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, 1, &rect); 292 293 buf_info.offset += buf_extent_el.width * 294 buf_extent_el.height * buf_info.bs; 295 img_info.layer++; 296 if (image->type == VK_IMAGE_TYPE_3D) 297 slice_3d++; 298 else 299 slice_array++; 300 } 301 } 302 radv_meta_end_bufimage(cmd_buffer, &saved_state); 303 } 304 305 void radv_CmdCopyImageToBuffer( 306 VkCommandBuffer commandBuffer, 307 VkImage srcImage, 308 VkImageLayout srcImageLayout, 309 VkBuffer destBuffer, 310 uint32_t regionCount, 311 const VkBufferImageCopy* pRegions) 312 { 313 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); 314 RADV_FROM_HANDLE(radv_image, src_image, srcImage); 315 RADV_FROM_HANDLE(radv_buffer, dst_buffer, destBuffer); 316 317 meta_copy_image_to_buffer(cmd_buffer, dst_buffer, src_image, 318 regionCount, pRegions); 319 } 320 321 static void 322 meta_copy_image(struct radv_cmd_buffer *cmd_buffer, 323 struct radv_image *src_image, 324 struct radv_image *dest_image, 325 uint32_t regionCount, 326 const VkImageCopy *pRegions) 327 { 328 bool cs = cmd_buffer->queue_family_index == RADV_QUEUE_COMPUTE; 329 union meta_saved_state saved_state; 330 331 /* From the Vulkan 1.0 spec: 332 * 333 * vkCmdCopyImage can be used to copy image data between multisample 334 * images, but both images must have the same number of samples. 335 */ 336 assert(src_image->samples == dest_image->samples); 337 if (cs) 338 radv_meta_begin_itoi(cmd_buffer, &saved_state.compute); 339 else 340 radv_meta_save_graphics_reset_vport_scissor(&saved_state.gfx, cmd_buffer); 341 342 for (unsigned r = 0; r < regionCount; r++) { 343 assert(pRegions[r].srcSubresource.aspectMask == 344 pRegions[r].dstSubresource.aspectMask); 345 346 /* Create blit surfaces */ 347 struct radv_meta_blit2d_surf b_src = 348 blit_surf_for_image_level_layer(src_image, 349 &pRegions[r].srcSubresource); 350 351 struct radv_meta_blit2d_surf b_dst = 352 blit_surf_for_image_level_layer(dest_image, 353 &pRegions[r].dstSubresource); 354 355 /* for DCC */ 356 b_src.format = b_dst.format; 357 358 /** 359 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images 360 * imageExtent is the size in texels of the image to copy in width, height 361 * and depth. 1D images use only x and width. 2D images use x, y, width 362 * and height. 3D images use x, y, z, width, height and depth. 363 * 364 * Also, convert the offsets and extent from units of texels to units of 365 * blocks - which is the highest resolution accessible in this command. 366 */ 367 const VkOffset3D dst_offset_el = 368 meta_region_offset_el(dest_image, &pRegions[r].dstOffset); 369 const VkOffset3D src_offset_el = 370 meta_region_offset_el(src_image, &pRegions[r].srcOffset); 371 const VkExtent3D img_extent_el = 372 meta_region_extent_el(src_image, &pRegions[r].extent); 373 374 /* Start creating blit rect */ 375 struct radv_meta_blit2d_rect rect = { 376 .width = img_extent_el.width, 377 .height = img_extent_el.height, 378 }; 379 380 /* Loop through each 3D or array slice */ 381 unsigned num_slices_3d = img_extent_el.depth; 382 unsigned num_slices_array = pRegions[r].dstSubresource.layerCount; 383 unsigned slice_3d = 0; 384 unsigned slice_array = 0; 385 while (slice_3d < num_slices_3d && slice_array < num_slices_array) { 386 387 /* Finish creating blit rect */ 388 rect.dst_x = dst_offset_el.x; 389 rect.dst_y = dst_offset_el.y; 390 rect.src_x = src_offset_el.x; 391 rect.src_y = src_offset_el.y; 392 393 /* Perform Blit */ 394 if (cs) 395 radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, 1, &rect); 396 else 397 radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, 1, &rect); 398 399 b_src.layer++; 400 b_dst.layer++; 401 if (dest_image->type == VK_IMAGE_TYPE_3D) 402 slice_3d++; 403 else 404 slice_array++; 405 } 406 } 407 408 if (cs) 409 radv_meta_end_itoi(cmd_buffer, &saved_state.compute); 410 else 411 radv_meta_restore(&saved_state.gfx, cmd_buffer); 412 } 413 414 void radv_CmdCopyImage( 415 VkCommandBuffer commandBuffer, 416 VkImage srcImage, 417 VkImageLayout srcImageLayout, 418 VkImage destImage, 419 VkImageLayout destImageLayout, 420 uint32_t regionCount, 421 const VkImageCopy* pRegions) 422 { 423 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); 424 RADV_FROM_HANDLE(radv_image, src_image, srcImage); 425 RADV_FROM_HANDLE(radv_image, dest_image, destImage); 426 427 meta_copy_image(cmd_buffer, src_image, dest_image, 428 regionCount, pRegions); 429 } 430
