1 /* 2 * Copyright 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #ifndef ANV_PRIVATE_H 25 #define ANV_PRIVATE_H 26 27 #include <stdlib.h> 28 #include <stdio.h> 29 #include <stdbool.h> 30 #include <pthread.h> 31 #include <assert.h> 32 #include <stdint.h> 33 #include <i915_drm.h> 34 35 #ifdef HAVE_VALGRIND 36 #include <valgrind.h> 37 #include <memcheck.h> 38 #define VG(x) x 39 #define __gen_validate_value(x) VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x)) 40 #else 41 #define VG(x) 42 #endif 43 44 #include "common/gen_device_info.h" 45 #include "blorp/blorp.h" 46 #include "brw_compiler.h" 47 #include "util/macros.h" 48 #include "util/list.h" 49 #include "util/u_vector.h" 50 #include "util/vk_alloc.h" 51 52 /* Pre-declarations needed for WSI entrypoints */ 53 struct wl_surface; 54 struct wl_display; 55 typedef struct xcb_connection_t xcb_connection_t; 56 typedef uint32_t xcb_visualid_t; 57 typedef uint32_t xcb_window_t; 58 59 struct gen_l3_config; 60 61 #include <vulkan/vulkan.h> 62 #include <vulkan/vulkan_intel.h> 63 #include <vulkan/vk_icd.h> 64 65 #include "anv_entrypoints.h" 66 #include "brw_context.h" 67 #include "isl/isl.h" 68 69 #include "wsi_common.h" 70 71 #ifdef __cplusplus 72 extern "C" { 73 #endif 74 75 /* Allowing different clear colors requires us to perform a depth resolve at 76 * the end of certain render passes. This is because while slow clears store 77 * the clear color in the HiZ buffer, fast clears (without a resolve) don't. 78 * See the PRMs for examples describing when additional resolves would be 79 * necessary. To enable fast clears without requiring extra resolves, we set 80 * the clear value to a globally-defined one. We could allow different values 81 * if the user doesn't expect coherent data during or after a render passes 82 * (VK_ATTACHMENT_STORE_OP_DONT_CARE), but such users (aside from the CTS) 83 * don't seem to exist yet. In almost all Vulkan applications tested thus far, 84 * 1.0f seems to be the only value used. The only application that doesn't set 85 * this value does so through the usage of an seemingly uninitialized clear 86 * value. 87 */ 88 #define ANV_HZ_FC_VAL 1.0f 89 90 #define MAX_VBS 32 91 #define MAX_SETS 8 92 #define MAX_RTS 8 93 #define MAX_VIEWPORTS 16 94 #define MAX_SCISSORS 16 95 #define MAX_PUSH_CONSTANTS_SIZE 128 96 #define MAX_DYNAMIC_BUFFERS 16 97 #define MAX_IMAGES 8 98 99 #define anv_printflike(a, b) __attribute__((__format__(__printf__, a, b))) 100 101 static inline uint32_t 102 align_down_npot_u32(uint32_t v, uint32_t a) 103 { 104 return v - (v % a); 105 } 106 107 static inline uint32_t 108 align_u32(uint32_t v, uint32_t a) 109 { 110 assert(a != 0 && a == (a & -a)); 111 return (v + a - 1) & ~(a - 1); 112 } 113 114 static inline uint64_t 115 align_u64(uint64_t v, uint64_t a) 116 { 117 assert(a != 0 && a == (a & -a)); 118 return (v + a - 1) & ~(a - 1); 119 } 120 121 static inline int32_t 122 align_i32(int32_t v, int32_t a) 123 { 124 assert(a != 0 && a == (a & -a)); 125 return (v + a - 1) & ~(a - 1); 126 } 127 128 /** Alignment must be a power of 2. */ 129 static inline bool 130 anv_is_aligned(uintmax_t n, uintmax_t a) 131 { 132 assert(a == (a & -a)); 133 return (n & (a - 1)) == 0; 134 } 135 136 static inline uint32_t 137 anv_minify(uint32_t n, uint32_t levels) 138 { 139 if (unlikely(n == 0)) 140 return 0; 141 else 142 return MAX2(n >> levels, 1); 143 } 144 145 static inline float 146 anv_clamp_f(float f, float min, float max) 147 { 148 assert(min < max); 149 150 if (f > max) 151 return max; 152 else if (f < min) 153 return min; 154 else 155 return f; 156 } 157 158 static inline bool 159 anv_clear_mask(uint32_t *inout_mask, uint32_t clear_mask) 160 { 161 if (*inout_mask & clear_mask) { 162 *inout_mask &= ~clear_mask; 163 return true; 164 } else { 165 return false; 166 } 167 } 168 169 static inline union isl_color_value 170 vk_to_isl_color(VkClearColorValue color) 171 { 172 return (union isl_color_value) { 173 .u32 = { 174 color.uint32[0], 175 color.uint32[1], 176 color.uint32[2], 177 color.uint32[3], 178 }, 179 }; 180 } 181 182 #define for_each_bit(b, dword) \ 183 for (uint32_t __dword = (dword); \ 184 (b) = __builtin_ffs(__dword) - 1, __dword; \ 185 __dword &= ~(1 << (b))) 186 187 #define typed_memcpy(dest, src, count) ({ \ 188 STATIC_ASSERT(sizeof(*src) == sizeof(*dest)); \ 189 memcpy((dest), (src), (count) * sizeof(*(src))); \ 190 }) 191 192 /* Whenever we generate an error, pass it through this function. Useful for 193 * debugging, where we can break on it. Only call at error site, not when 194 * propagating errors. Might be useful to plug in a stack trace here. 195 */ 196 197 VkResult __vk_errorf(VkResult error, const char *file, int line, const char *format, ...); 198 199 #ifdef DEBUG 200 #define vk_error(error) __vk_errorf(error, __FILE__, __LINE__, NULL); 201 #define vk_errorf(error, format, ...) __vk_errorf(error, __FILE__, __LINE__, format, ## __VA_ARGS__); 202 #else 203 #define vk_error(error) error 204 #define vk_errorf(error, format, ...) error 205 #endif 206 207 void __anv_finishme(const char *file, int line, const char *format, ...) 208 anv_printflike(3, 4); 209 void anv_loge(const char *format, ...) anv_printflike(1, 2); 210 void anv_loge_v(const char *format, va_list va); 211 212 /** 213 * Print a FINISHME message, including its source location. 214 */ 215 #define anv_finishme(format, ...) \ 216 do { \ 217 static bool reported = false; \ 218 if (!reported) { \ 219 __anv_finishme(__FILE__, __LINE__, format, ##__VA_ARGS__); \ 220 reported = true; \ 221 } \ 222 } while (0) 223 224 /* A non-fatal assert. Useful for debugging. */ 225 #ifdef DEBUG 226 #define anv_assert(x) ({ \ 227 if (unlikely(!(x))) \ 228 fprintf(stderr, "%s:%d ASSERT: %s\n", __FILE__, __LINE__, #x); \ 229 }) 230 #else 231 #define anv_assert(x) 232 #endif 233 234 /** 235 * If a block of code is annotated with anv_validate, then the block runs only 236 * in debug builds. 237 */ 238 #ifdef DEBUG 239 #define anv_validate if (1) 240 #else 241 #define anv_validate if (0) 242 #endif 243 244 #define stub_return(v) \ 245 do { \ 246 anv_finishme("stub %s", __func__); \ 247 return (v); \ 248 } while (0) 249 250 #define stub() \ 251 do { \ 252 anv_finishme("stub %s", __func__); \ 253 return; \ 254 } while (0) 255 256 /** 257 * A dynamically growable, circular buffer. Elements are added at head and 258 * removed from tail. head and tail are free-running uint32_t indices and we 259 * only compute the modulo with size when accessing the array. This way, 260 * number of bytes in the queue is always head - tail, even in case of 261 * wraparound. 262 */ 263 264 struct anv_bo { 265 uint32_t gem_handle; 266 267 /* Index into the current validation list. This is used by the 268 * validation list building alrogithm to track which buffers are already 269 * in the validation list so that we can ensure uniqueness. 270 */ 271 uint32_t index; 272 273 /* Last known offset. This value is provided by the kernel when we 274 * execbuf and is used as the presumed offset for the next bunch of 275 * relocations. 276 */ 277 uint64_t offset; 278 279 uint64_t size; 280 void *map; 281 282 /* We need to set the WRITE flag on winsys bos so GEM will know we're 283 * writing to them and synchronize uses on other rings (eg if the display 284 * server uses the blitter ring). 285 */ 286 bool is_winsys_bo; 287 }; 288 289 static inline void 290 anv_bo_init(struct anv_bo *bo, uint32_t gem_handle, uint64_t size) 291 { 292 bo->gem_handle = gem_handle; 293 bo->index = 0; 294 bo->offset = -1; 295 bo->size = size; 296 bo->map = NULL; 297 bo->is_winsys_bo = false; 298 } 299 300 /* Represents a lock-free linked list of "free" things. This is used by 301 * both the block pool and the state pools. Unfortunately, in order to 302 * solve the ABA problem, we can't use a single uint32_t head. 303 */ 304 union anv_free_list { 305 struct { 306 int32_t offset; 307 308 /* A simple count that is incremented every time the head changes. */ 309 uint32_t count; 310 }; 311 uint64_t u64; 312 }; 313 314 #define ANV_FREE_LIST_EMPTY ((union anv_free_list) { { 1, 0 } }) 315 316 struct anv_block_state { 317 union { 318 struct { 319 uint32_t next; 320 uint32_t end; 321 }; 322 uint64_t u64; 323 }; 324 }; 325 326 struct anv_block_pool { 327 struct anv_device *device; 328 329 struct anv_bo bo; 330 331 /* The offset from the start of the bo to the "center" of the block 332 * pool. Pointers to allocated blocks are given by 333 * bo.map + center_bo_offset + offsets. 334 */ 335 uint32_t center_bo_offset; 336 337 /* Current memory map of the block pool. This pointer may or may not 338 * point to the actual beginning of the block pool memory. If 339 * anv_block_pool_alloc_back has ever been called, then this pointer 340 * will point to the "center" position of the buffer and all offsets 341 * (negative or positive) given out by the block pool alloc functions 342 * will be valid relative to this pointer. 343 * 344 * In particular, map == bo.map + center_offset 345 */ 346 void *map; 347 int fd; 348 349 /** 350 * Array of mmaps and gem handles owned by the block pool, reclaimed when 351 * the block pool is destroyed. 352 */ 353 struct u_vector mmap_cleanups; 354 355 uint32_t block_size; 356 357 union anv_free_list free_list; 358 struct anv_block_state state; 359 360 union anv_free_list back_free_list; 361 struct anv_block_state back_state; 362 }; 363 364 /* Block pools are backed by a fixed-size 2GB memfd */ 365 #define BLOCK_POOL_MEMFD_SIZE (1ull << 32) 366 367 /* The center of the block pool is also the middle of the memfd. This may 368 * change in the future if we decide differently for some reason. 369 */ 370 #define BLOCK_POOL_MEMFD_CENTER (BLOCK_POOL_MEMFD_SIZE / 2) 371 372 static inline uint32_t 373 anv_block_pool_size(struct anv_block_pool *pool) 374 { 375 return pool->state.end + pool->back_state.end; 376 } 377 378 struct anv_state { 379 int32_t offset; 380 uint32_t alloc_size; 381 void *map; 382 }; 383 384 struct anv_fixed_size_state_pool { 385 size_t state_size; 386 union anv_free_list free_list; 387 struct anv_block_state block; 388 }; 389 390 #define ANV_MIN_STATE_SIZE_LOG2 6 391 #define ANV_MAX_STATE_SIZE_LOG2 20 392 393 #define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1) 394 395 struct anv_state_pool { 396 struct anv_block_pool *block_pool; 397 struct anv_fixed_size_state_pool buckets[ANV_STATE_BUCKETS]; 398 }; 399 400 struct anv_state_stream_block; 401 402 struct anv_state_stream { 403 struct anv_block_pool *block_pool; 404 405 /* The current working block */ 406 struct anv_state_stream_block *block; 407 408 /* Offset at which the current block starts */ 409 uint32_t start; 410 /* Offset at which to allocate the next state */ 411 uint32_t next; 412 /* Offset at which the current block ends */ 413 uint32_t end; 414 }; 415 416 #define CACHELINE_SIZE 64 417 #define CACHELINE_MASK 63 418 419 static inline void 420 anv_clflush_range(void *start, size_t size) 421 { 422 void *p = (void *) (((uintptr_t) start) & ~CACHELINE_MASK); 423 void *end = start + size; 424 425 __builtin_ia32_mfence(); 426 while (p < end) { 427 __builtin_ia32_clflush(p); 428 p += CACHELINE_SIZE; 429 } 430 } 431 432 static inline void 433 anv_invalidate_range(void *start, size_t size) 434 { 435 void *p = (void *) (((uintptr_t) start) & ~CACHELINE_MASK); 436 void *end = start + size; 437 438 while (p < end) { 439 __builtin_ia32_clflush(p); 440 p += CACHELINE_SIZE; 441 } 442 __builtin_ia32_mfence(); 443 } 444 445 static void inline 446 anv_state_clflush(struct anv_state state) 447 { 448 anv_clflush_range(state.map, state.alloc_size); 449 } 450 451 VkResult anv_block_pool_init(struct anv_block_pool *pool, 452 struct anv_device *device, uint32_t block_size); 453 void anv_block_pool_finish(struct anv_block_pool *pool); 454 int32_t anv_block_pool_alloc(struct anv_block_pool *pool); 455 int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool); 456 void anv_block_pool_free(struct anv_block_pool *pool, int32_t offset); 457 void anv_state_pool_init(struct anv_state_pool *pool, 458 struct anv_block_pool *block_pool); 459 void anv_state_pool_finish(struct anv_state_pool *pool); 460 struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool, 461 size_t state_size, size_t alignment); 462 void anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state); 463 void anv_state_stream_init(struct anv_state_stream *stream, 464 struct anv_block_pool *block_pool); 465 void anv_state_stream_finish(struct anv_state_stream *stream); 466 struct anv_state anv_state_stream_alloc(struct anv_state_stream *stream, 467 uint32_t size, uint32_t alignment); 468 469 /** 470 * Implements a pool of re-usable BOs. The interface is identical to that 471 * of block_pool except that each block is its own BO. 472 */ 473 struct anv_bo_pool { 474 struct anv_device *device; 475 476 void *free_list[16]; 477 }; 478 479 void anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device); 480 void anv_bo_pool_finish(struct anv_bo_pool *pool); 481 VkResult anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo, 482 uint32_t size); 483 void anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo); 484 485 struct anv_scratch_bo { 486 bool exists; 487 struct anv_bo bo; 488 }; 489 490 struct anv_scratch_pool { 491 /* Indexed by Per-Thread Scratch Space number (the hardware value) and stage */ 492 struct anv_scratch_bo bos[16][MESA_SHADER_STAGES]; 493 }; 494 495 void anv_scratch_pool_init(struct anv_device *device, 496 struct anv_scratch_pool *pool); 497 void anv_scratch_pool_finish(struct anv_device *device, 498 struct anv_scratch_pool *pool); 499 struct anv_bo *anv_scratch_pool_alloc(struct anv_device *device, 500 struct anv_scratch_pool *pool, 501 gl_shader_stage stage, 502 unsigned per_thread_scratch); 503 504 extern struct anv_dispatch_table dtable; 505 506 struct anv_physical_device { 507 VK_LOADER_DATA _loader_data; 508 509 struct anv_instance * instance; 510 uint32_t chipset_id; 511 char path[20]; 512 const char * name; 513 struct gen_device_info info; 514 uint64_t aperture_size; 515 struct brw_compiler * compiler; 516 struct isl_device isl_dev; 517 int cmd_parser_version; 518 519 uint32_t eu_total; 520 uint32_t subslice_total; 521 522 uint8_t uuid[VK_UUID_SIZE]; 523 524 struct wsi_device wsi_device; 525 }; 526 527 struct anv_instance { 528 VK_LOADER_DATA _loader_data; 529 530 VkAllocationCallbacks alloc; 531 532 uint32_t apiVersion; 533 int physicalDeviceCount; 534 struct anv_physical_device physicalDevice; 535 }; 536 537 VkResult anv_init_wsi(struct anv_physical_device *physical_device); 538 void anv_finish_wsi(struct anv_physical_device *physical_device); 539 540 struct anv_queue { 541 VK_LOADER_DATA _loader_data; 542 543 struct anv_device * device; 544 545 struct anv_state_pool * pool; 546 }; 547 548 struct anv_pipeline_cache { 549 struct anv_device * device; 550 pthread_mutex_t mutex; 551 552 struct hash_table * cache; 553 }; 554 555 struct anv_pipeline_bind_map; 556 557 void anv_pipeline_cache_init(struct anv_pipeline_cache *cache, 558 struct anv_device *device, 559 bool cache_enabled); 560 void anv_pipeline_cache_finish(struct anv_pipeline_cache *cache); 561 562 struct anv_shader_bin * 563 anv_pipeline_cache_search(struct anv_pipeline_cache *cache, 564 const void *key, uint32_t key_size); 565 struct anv_shader_bin * 566 anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache, 567 const void *key_data, uint32_t key_size, 568 const void *kernel_data, uint32_t kernel_size, 569 const struct brw_stage_prog_data *prog_data, 570 uint32_t prog_data_size, 571 const struct anv_pipeline_bind_map *bind_map); 572 573 struct anv_device { 574 VK_LOADER_DATA _loader_data; 575 576 VkAllocationCallbacks alloc; 577 578 struct anv_instance * instance; 579 uint32_t chipset_id; 580 struct gen_device_info info; 581 struct isl_device isl_dev; 582 int context_id; 583 int fd; 584 bool can_chain_batches; 585 bool robust_buffer_access; 586 587 struct anv_bo_pool batch_bo_pool; 588 589 struct anv_block_pool dynamic_state_block_pool; 590 struct anv_state_pool dynamic_state_pool; 591 592 struct anv_block_pool instruction_block_pool; 593 struct anv_state_pool instruction_state_pool; 594 595 struct anv_block_pool surface_state_block_pool; 596 struct anv_state_pool surface_state_pool; 597 598 struct anv_bo workaround_bo; 599 600 struct anv_pipeline_cache blorp_shader_cache; 601 struct blorp_context blorp; 602 603 struct anv_state border_colors; 604 605 struct anv_queue queue; 606 607 struct anv_scratch_pool scratch_pool; 608 609 uint32_t default_mocs; 610 611 pthread_mutex_t mutex; 612 pthread_cond_t queue_submit; 613 }; 614 615 void anv_device_init_blorp(struct anv_device *device); 616 void anv_device_finish_blorp(struct anv_device *device); 617 618 VkResult anv_device_execbuf(struct anv_device *device, 619 struct drm_i915_gem_execbuffer2 *execbuf, 620 struct anv_bo **execbuf_bos); 621 622 void* anv_gem_mmap(struct anv_device *device, 623 uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags); 624 void anv_gem_munmap(void *p, uint64_t size); 625 uint32_t anv_gem_create(struct anv_device *device, size_t size); 626 void anv_gem_close(struct anv_device *device, uint32_t gem_handle); 627 uint32_t anv_gem_userptr(struct anv_device *device, void *mem, size_t size); 628 int anv_gem_wait(struct anv_device *device, uint32_t gem_handle, int64_t *timeout_ns); 629 int anv_gem_execbuffer(struct anv_device *device, 630 struct drm_i915_gem_execbuffer2 *execbuf); 631 int anv_gem_set_tiling(struct anv_device *device, uint32_t gem_handle, 632 uint32_t stride, uint32_t tiling); 633 int anv_gem_create_context(struct anv_device *device); 634 int anv_gem_destroy_context(struct anv_device *device, int context); 635 int anv_gem_get_param(int fd, uint32_t param); 636 bool anv_gem_get_bit6_swizzle(int fd, uint32_t tiling); 637 int anv_gem_get_aperture(int fd, uint64_t *size); 638 int anv_gem_handle_to_fd(struct anv_device *device, uint32_t gem_handle); 639 uint32_t anv_gem_fd_to_handle(struct anv_device *device, int fd); 640 int anv_gem_set_caching(struct anv_device *device, uint32_t gem_handle, uint32_t caching); 641 int anv_gem_set_domain(struct anv_device *device, uint32_t gem_handle, 642 uint32_t read_domains, uint32_t write_domain); 643 644 VkResult anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size); 645 646 struct anv_reloc_list { 647 size_t num_relocs; 648 size_t array_length; 649 struct drm_i915_gem_relocation_entry * relocs; 650 struct anv_bo ** reloc_bos; 651 }; 652 653 VkResult anv_reloc_list_init(struct anv_reloc_list *list, 654 const VkAllocationCallbacks *alloc); 655 void anv_reloc_list_finish(struct anv_reloc_list *list, 656 const VkAllocationCallbacks *alloc); 657 658 uint64_t anv_reloc_list_add(struct anv_reloc_list *list, 659 const VkAllocationCallbacks *alloc, 660 uint32_t offset, struct anv_bo *target_bo, 661 uint32_t delta); 662 663 struct anv_batch_bo { 664 /* Link in the anv_cmd_buffer.owned_batch_bos list */ 665 struct list_head link; 666 667 struct anv_bo bo; 668 669 /* Bytes actually consumed in this batch BO */ 670 size_t length; 671 672 struct anv_reloc_list relocs; 673 }; 674 675 struct anv_batch { 676 const VkAllocationCallbacks * alloc; 677 678 void * start; 679 void * end; 680 void * next; 681 682 struct anv_reloc_list * relocs; 683 684 /* This callback is called (with the associated user data) in the event 685 * that the batch runs out of space. 686 */ 687 VkResult (*extend_cb)(struct anv_batch *, void *); 688 void * user_data; 689 }; 690 691 void *anv_batch_emit_dwords(struct anv_batch *batch, int num_dwords); 692 void anv_batch_emit_batch(struct anv_batch *batch, struct anv_batch *other); 693 uint64_t anv_batch_emit_reloc(struct anv_batch *batch, 694 void *location, struct anv_bo *bo, uint32_t offset); 695 VkResult anv_device_submit_simple_batch(struct anv_device *device, 696 struct anv_batch *batch); 697 698 struct anv_address { 699 struct anv_bo *bo; 700 uint32_t offset; 701 }; 702 703 static inline uint64_t 704 _anv_combine_address(struct anv_batch *batch, void *location, 705 const struct anv_address address, uint32_t delta) 706 { 707 if (address.bo == NULL) { 708 return address.offset + delta; 709 } else { 710 assert(batch->start <= location && location < batch->end); 711 712 return anv_batch_emit_reloc(batch, location, address.bo, address.offset + delta); 713 } 714 } 715 716 #define __gen_address_type struct anv_address 717 #define __gen_user_data struct anv_batch 718 #define __gen_combine_address _anv_combine_address 719 720 /* Wrapper macros needed to work around preprocessor argument issues. In 721 * particular, arguments don't get pre-evaluated if they are concatenated. 722 * This means that, if you pass GENX(3DSTATE_PS) into the emit macro, the 723 * GENX macro won't get evaluated if the emit macro contains "cmd ## foo". 724 * We can work around this easily enough with these helpers. 725 */ 726 #define __anv_cmd_length(cmd) cmd ## _length 727 #define __anv_cmd_length_bias(cmd) cmd ## _length_bias 728 #define __anv_cmd_header(cmd) cmd ## _header 729 #define __anv_cmd_pack(cmd) cmd ## _pack 730 #define __anv_reg_num(reg) reg ## _num 731 732 #define anv_pack_struct(dst, struc, ...) do { \ 733 struct struc __template = { \ 734 __VA_ARGS__ \ 735 }; \ 736 __anv_cmd_pack(struc)(NULL, dst, &__template); \ 737 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dst, __anv_cmd_length(struc) * 4)); \ 738 } while (0) 739 740 #define anv_batch_emitn(batch, n, cmd, ...) ({ \ 741 void *__dst = anv_batch_emit_dwords(batch, n); \ 742 struct cmd __template = { \ 743 __anv_cmd_header(cmd), \ 744 .DWordLength = n - __anv_cmd_length_bias(cmd), \ 745 __VA_ARGS__ \ 746 }; \ 747 __anv_cmd_pack(cmd)(batch, __dst, &__template); \ 748 __dst; \ 749 }) 750 751 #define anv_batch_emit_merge(batch, dwords0, dwords1) \ 752 do { \ 753 uint32_t *dw; \ 754 \ 755 STATIC_ASSERT(ARRAY_SIZE(dwords0) == ARRAY_SIZE(dwords1)); \ 756 dw = anv_batch_emit_dwords((batch), ARRAY_SIZE(dwords0)); \ 757 for (uint32_t i = 0; i < ARRAY_SIZE(dwords0); i++) \ 758 dw[i] = (dwords0)[i] | (dwords1)[i]; \ 759 VG(VALGRIND_CHECK_MEM_IS_DEFINED(dw, ARRAY_SIZE(dwords0) * 4));\ 760 } while (0) 761 762 #define anv_batch_emit(batch, cmd, name) \ 763 for (struct cmd name = { __anv_cmd_header(cmd) }, \ 764 *_dst = anv_batch_emit_dwords(batch, __anv_cmd_length(cmd)); \ 765 __builtin_expect(_dst != NULL, 1); \ 766 ({ __anv_cmd_pack(cmd)(batch, _dst, &name); \ 767 VG(VALGRIND_CHECK_MEM_IS_DEFINED(_dst, __anv_cmd_length(cmd) * 4)); \ 768 _dst = NULL; \ 769 })) 770 771 #define anv_state_pool_emit(pool, cmd, align, ...) ({ \ 772 const uint32_t __size = __anv_cmd_length(cmd) * 4; \ 773 struct anv_state __state = \ 774 anv_state_pool_alloc((pool), __size, align); \ 775 struct cmd __template = { \ 776 __VA_ARGS__ \ 777 }; \ 778 __anv_cmd_pack(cmd)(NULL, __state.map, &__template); \ 779 VG(VALGRIND_CHECK_MEM_IS_DEFINED(__state.map, __anv_cmd_length(cmd) * 4)); \ 780 if (!(pool)->block_pool->device->info.has_llc) \ 781 anv_state_clflush(__state); \ 782 __state; \ 783 }) 784 785 #define GEN7_MOCS (struct GEN7_MEMORY_OBJECT_CONTROL_STATE) { \ 786 .GraphicsDataTypeGFDT = 0, \ 787 .LLCCacheabilityControlLLCCC = 0, \ 788 .L3CacheabilityControlL3CC = 1, \ 789 } 790 791 #define GEN75_MOCS (struct GEN75_MEMORY_OBJECT_CONTROL_STATE) { \ 792 .LLCeLLCCacheabilityControlLLCCC = 0, \ 793 .L3CacheabilityControlL3CC = 1, \ 794 } 795 796 #define GEN8_MOCS (struct GEN8_MEMORY_OBJECT_CONTROL_STATE) { \ 797 .MemoryTypeLLCeLLCCacheabilityControl = WB, \ 798 .TargetCache = L3DefertoPATforLLCeLLCselection, \ 799 .AgeforQUADLRU = 0 \ 800 } 801 802 /* Skylake: MOCS is now an index into an array of 62 different caching 803 * configurations programmed by the kernel. 804 */ 805 806 #define GEN9_MOCS (struct GEN9_MEMORY_OBJECT_CONTROL_STATE) { \ 807 /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \ 808 .IndextoMOCSTables = 2 \ 809 } 810 811 #define GEN9_MOCS_PTE { \ 812 /* TC=LLC/eLLC, LeCC=WB, LRUM=3, L3CC=WB */ \ 813 .IndextoMOCSTables = 1 \ 814 } 815 816 struct anv_device_memory { 817 struct anv_bo bo; 818 uint32_t type_index; 819 VkDeviceSize map_size; 820 void * map; 821 }; 822 823 /** 824 * Header for Vertex URB Entry (VUE) 825 */ 826 struct anv_vue_header { 827 uint32_t Reserved; 828 uint32_t RTAIndex; /* RenderTargetArrayIndex */ 829 uint32_t ViewportIndex; 830 float PointWidth; 831 }; 832 833 struct anv_descriptor_set_binding_layout { 834 #ifndef NDEBUG 835 /* The type of the descriptors in this binding */ 836 VkDescriptorType type; 837 #endif 838 839 /* Number of array elements in this binding */ 840 uint16_t array_size; 841 842 /* Index into the flattend descriptor set */ 843 uint16_t descriptor_index; 844 845 /* Index into the dynamic state array for a dynamic buffer */ 846 int16_t dynamic_offset_index; 847 848 /* Index into the descriptor set buffer views */ 849 int16_t buffer_index; 850 851 struct { 852 /* Index into the binding table for the associated surface */ 853 int16_t surface_index; 854 855 /* Index into the sampler table for the associated sampler */ 856 int16_t sampler_index; 857 858 /* Index into the image table for the associated image */ 859 int16_t image_index; 860 } stage[MESA_SHADER_STAGES]; 861 862 /* Immutable samplers (or NULL if no immutable samplers) */ 863 struct anv_sampler **immutable_samplers; 864 }; 865 866 struct anv_descriptor_set_layout { 867 /* Number of bindings in this descriptor set */ 868 uint16_t binding_count; 869 870 /* Total size of the descriptor set with room for all array entries */ 871 uint16_t size; 872 873 /* Shader stages affected by this descriptor set */ 874 uint16_t shader_stages; 875 876 /* Number of buffers in this descriptor set */ 877 uint16_t buffer_count; 878 879 /* Number of dynamic offsets used by this descriptor set */ 880 uint16_t dynamic_offset_count; 881 882 /* Bindings in this descriptor set */ 883 struct anv_descriptor_set_binding_layout binding[0]; 884 }; 885 886 struct anv_descriptor { 887 VkDescriptorType type; 888 889 union { 890 struct { 891 struct anv_image_view *image_view; 892 struct anv_sampler *sampler; 893 }; 894 895 struct anv_buffer_view *buffer_view; 896 }; 897 }; 898 899 struct anv_descriptor_set { 900 const struct anv_descriptor_set_layout *layout; 901 uint32_t size; 902 uint32_t buffer_count; 903 struct anv_buffer_view *buffer_views; 904 struct anv_descriptor descriptors[0]; 905 }; 906 907 struct anv_descriptor_pool { 908 uint32_t size; 909 uint32_t next; 910 uint32_t free_list; 911 912 struct anv_state_stream surface_state_stream; 913 void *surface_state_free_list; 914 915 char data[0]; 916 }; 917 918 VkResult 919 anv_descriptor_set_create(struct anv_device *device, 920 struct anv_descriptor_pool *pool, 921 const struct anv_descriptor_set_layout *layout, 922 struct anv_descriptor_set **out_set); 923 924 void 925 anv_descriptor_set_destroy(struct anv_device *device, 926 struct anv_descriptor_pool *pool, 927 struct anv_descriptor_set *set); 928 929 #define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX 930 931 struct anv_pipeline_binding { 932 /* The descriptor set this surface corresponds to. The special value of 933 * ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS indicates that the offset refers 934 * to a color attachment and not a regular descriptor. 935 */ 936 uint8_t set; 937 938 /* Binding in the descriptor set */ 939 uint8_t binding; 940 941 /* Index in the binding */ 942 uint8_t index; 943 944 /* Input attachment index (relative to the subpass) */ 945 uint8_t input_attachment_index; 946 }; 947 948 struct anv_pipeline_layout { 949 struct { 950 struct anv_descriptor_set_layout *layout; 951 uint32_t dynamic_offset_start; 952 } set[MAX_SETS]; 953 954 uint32_t num_sets; 955 956 struct { 957 bool has_dynamic_offsets; 958 } stage[MESA_SHADER_STAGES]; 959 960 unsigned char sha1[20]; 961 }; 962 963 struct anv_buffer { 964 struct anv_device * device; 965 VkDeviceSize size; 966 967 VkBufferUsageFlags usage; 968 969 /* Set when bound */ 970 struct anv_bo * bo; 971 VkDeviceSize offset; 972 }; 973 974 enum anv_cmd_dirty_bits { 975 ANV_CMD_DIRTY_DYNAMIC_VIEWPORT = 1 << 0, /* VK_DYNAMIC_STATE_VIEWPORT */ 976 ANV_CMD_DIRTY_DYNAMIC_SCISSOR = 1 << 1, /* VK_DYNAMIC_STATE_SCISSOR */ 977 ANV_CMD_DIRTY_DYNAMIC_LINE_WIDTH = 1 << 2, /* VK_DYNAMIC_STATE_LINE_WIDTH */ 978 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS = 1 << 3, /* VK_DYNAMIC_STATE_DEPTH_BIAS */ 979 ANV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS = 1 << 4, /* VK_DYNAMIC_STATE_BLEND_CONSTANTS */ 980 ANV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS = 1 << 5, /* VK_DYNAMIC_STATE_DEPTH_BOUNDS */ 981 ANV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6, /* VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK */ 982 ANV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7, /* VK_DYNAMIC_STATE_STENCIL_WRITE_MASK */ 983 ANV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE = 1 << 8, /* VK_DYNAMIC_STATE_STENCIL_REFERENCE */ 984 ANV_CMD_DIRTY_DYNAMIC_ALL = (1 << 9) - 1, 985 ANV_CMD_DIRTY_PIPELINE = 1 << 9, 986 ANV_CMD_DIRTY_INDEX_BUFFER = 1 << 10, 987 ANV_CMD_DIRTY_RENDER_TARGETS = 1 << 11, 988 }; 989 typedef uint32_t anv_cmd_dirty_mask_t; 990 991 enum anv_pipe_bits { 992 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT = (1 << 0), 993 ANV_PIPE_STALL_AT_SCOREBOARD_BIT = (1 << 1), 994 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT = (1 << 2), 995 ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT = (1 << 3), 996 ANV_PIPE_VF_CACHE_INVALIDATE_BIT = (1 << 4), 997 ANV_PIPE_DATA_CACHE_FLUSH_BIT = (1 << 5), 998 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT = (1 << 10), 999 ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT = (1 << 11), 1000 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT = (1 << 12), 1001 ANV_PIPE_DEPTH_STALL_BIT = (1 << 13), 1002 ANV_PIPE_CS_STALL_BIT = (1 << 20), 1003 1004 /* This bit does not exist directly in PIPE_CONTROL. Instead it means that 1005 * a flush has happened but not a CS stall. The next time we do any sort 1006 * of invalidation we need to insert a CS stall at that time. Otherwise, 1007 * we would have to CS stall on every flush which could be bad. 1008 */ 1009 ANV_PIPE_NEEDS_CS_STALL_BIT = (1 << 21), 1010 }; 1011 1012 #define ANV_PIPE_FLUSH_BITS ( \ 1013 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | \ 1014 ANV_PIPE_DATA_CACHE_FLUSH_BIT | \ 1015 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT) 1016 1017 #define ANV_PIPE_STALL_BITS ( \ 1018 ANV_PIPE_STALL_AT_SCOREBOARD_BIT | \ 1019 ANV_PIPE_DEPTH_STALL_BIT | \ 1020 ANV_PIPE_CS_STALL_BIT) 1021 1022 #define ANV_PIPE_INVALIDATE_BITS ( \ 1023 ANV_PIPE_STATE_CACHE_INVALIDATE_BIT | \ 1024 ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT | \ 1025 ANV_PIPE_VF_CACHE_INVALIDATE_BIT | \ 1026 ANV_PIPE_DATA_CACHE_FLUSH_BIT | \ 1027 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \ 1028 ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT) 1029 1030 struct anv_vertex_binding { 1031 struct anv_buffer * buffer; 1032 VkDeviceSize offset; 1033 }; 1034 1035 struct anv_push_constants { 1036 /* Current allocated size of this push constants data structure. 1037 * Because a decent chunk of it may not be used (images on SKL, for 1038 * instance), we won't actually allocate the entire structure up-front. 1039 */ 1040 uint32_t size; 1041 1042 /* Push constant data provided by the client through vkPushConstants */ 1043 uint8_t client_data[MAX_PUSH_CONSTANTS_SIZE]; 1044 1045 /* Our hardware only provides zero-based vertex and instance id so, in 1046 * order to satisfy the vulkan requirements, we may have to push one or 1047 * both of these into the shader. 1048 */ 1049 uint32_t base_vertex; 1050 uint32_t base_instance; 1051 1052 /* Offsets and ranges for dynamically bound buffers */ 1053 struct { 1054 uint32_t offset; 1055 uint32_t range; 1056 } dynamic[MAX_DYNAMIC_BUFFERS]; 1057 1058 /* Image data for image_load_store on pre-SKL */ 1059 struct brw_image_param images[MAX_IMAGES]; 1060 }; 1061 1062 struct anv_dynamic_state { 1063 struct { 1064 uint32_t count; 1065 VkViewport viewports[MAX_VIEWPORTS]; 1066 } viewport; 1067 1068 struct { 1069 uint32_t count; 1070 VkRect2D scissors[MAX_SCISSORS]; 1071 } scissor; 1072 1073 float line_width; 1074 1075 struct { 1076 float bias; 1077 float clamp; 1078 float slope; 1079 } depth_bias; 1080 1081 float blend_constants[4]; 1082 1083 struct { 1084 float min; 1085 float max; 1086 } depth_bounds; 1087 1088 struct { 1089 uint32_t front; 1090 uint32_t back; 1091 } stencil_compare_mask; 1092 1093 struct { 1094 uint32_t front; 1095 uint32_t back; 1096 } stencil_write_mask; 1097 1098 struct { 1099 uint32_t front; 1100 uint32_t back; 1101 } stencil_reference; 1102 }; 1103 1104 extern const struct anv_dynamic_state default_dynamic_state; 1105 1106 void anv_dynamic_state_copy(struct anv_dynamic_state *dest, 1107 const struct anv_dynamic_state *src, 1108 uint32_t copy_mask); 1109 1110 /** 1111 * Attachment state when recording a renderpass instance. 1112 * 1113 * The clear value is valid only if there exists a pending clear. 1114 */ 1115 struct anv_attachment_state { 1116 enum isl_aux_usage aux_usage; 1117 enum isl_aux_usage input_aux_usage; 1118 struct anv_state color_rt_state; 1119 struct anv_state input_att_state; 1120 1121 VkImageLayout current_layout; 1122 VkImageAspectFlags pending_clear_aspects; 1123 bool fast_clear; 1124 VkClearValue clear_value; 1125 bool clear_color_is_zero_one; 1126 }; 1127 1128 /** State required while building cmd buffer */ 1129 struct anv_cmd_state { 1130 /* PIPELINE_SELECT.PipelineSelection */ 1131 uint32_t current_pipeline; 1132 const struct gen_l3_config * current_l3_config; 1133 uint32_t vb_dirty; 1134 anv_cmd_dirty_mask_t dirty; 1135 anv_cmd_dirty_mask_t compute_dirty; 1136 enum anv_pipe_bits pending_pipe_bits; 1137 uint32_t num_workgroups_offset; 1138 struct anv_bo *num_workgroups_bo; 1139 VkShaderStageFlags descriptors_dirty; 1140 VkShaderStageFlags push_constants_dirty; 1141 uint32_t scratch_size; 1142 struct anv_pipeline * pipeline; 1143 struct anv_pipeline * compute_pipeline; 1144 struct anv_framebuffer * framebuffer; 1145 struct anv_render_pass * pass; 1146 struct anv_subpass * subpass; 1147 VkRect2D render_area; 1148 uint32_t restart_index; 1149 struct anv_vertex_binding vertex_bindings[MAX_VBS]; 1150 struct anv_descriptor_set * descriptors[MAX_SETS]; 1151 VkShaderStageFlags push_constant_stages; 1152 struct anv_push_constants * push_constants[MESA_SHADER_STAGES]; 1153 struct anv_state binding_tables[MESA_SHADER_STAGES]; 1154 struct anv_state samplers[MESA_SHADER_STAGES]; 1155 struct anv_dynamic_state dynamic; 1156 bool need_query_wa; 1157 1158 /** 1159 * Array length is anv_cmd_state::pass::attachment_count. Array content is 1160 * valid only when recording a render pass instance. 1161 */ 1162 struct anv_attachment_state * attachments; 1163 1164 /** 1165 * Surface states for color render targets. These are stored in a single 1166 * flat array. For depth-stencil attachments, the surface state is simply 1167 * left blank. 1168 */ 1169 struct anv_state render_pass_states; 1170 1171 /** 1172 * A null surface state of the right size to match the framebuffer. This 1173 * is one of the states in render_pass_states. 1174 */ 1175 struct anv_state null_surface_state; 1176 1177 struct { 1178 struct anv_buffer * index_buffer; 1179 uint32_t index_type; /**< 3DSTATE_INDEX_BUFFER.IndexFormat */ 1180 uint32_t index_offset; 1181 } gen7; 1182 }; 1183 1184 struct anv_cmd_pool { 1185 VkAllocationCallbacks alloc; 1186 struct list_head cmd_buffers; 1187 }; 1188 1189 #define ANV_CMD_BUFFER_BATCH_SIZE 8192 1190 1191 enum anv_cmd_buffer_exec_mode { 1192 ANV_CMD_BUFFER_EXEC_MODE_PRIMARY, 1193 ANV_CMD_BUFFER_EXEC_MODE_EMIT, 1194 ANV_CMD_BUFFER_EXEC_MODE_GROW_AND_EMIT, 1195 ANV_CMD_BUFFER_EXEC_MODE_CHAIN, 1196 ANV_CMD_BUFFER_EXEC_MODE_COPY_AND_CHAIN, 1197 }; 1198 1199 struct anv_cmd_buffer { 1200 VK_LOADER_DATA _loader_data; 1201 1202 struct anv_device * device; 1203 1204 struct anv_cmd_pool * pool; 1205 struct list_head pool_link; 1206 1207 struct anv_batch batch; 1208 1209 /* Fields required for the actual chain of anv_batch_bo's. 1210 * 1211 * These fields are initialized by anv_cmd_buffer_init_batch_bo_chain(). 1212 */ 1213 struct list_head batch_bos; 1214 enum anv_cmd_buffer_exec_mode exec_mode; 1215 1216 /* A vector of anv_batch_bo pointers for every batch or surface buffer 1217 * referenced by this command buffer 1218 * 1219 * initialized by anv_cmd_buffer_init_batch_bo_chain() 1220 */ 1221 struct u_vector seen_bbos; 1222 1223 /* A vector of int32_t's for every block of binding tables. 1224 * 1225 * initialized by anv_cmd_buffer_init_batch_bo_chain() 1226 */ 1227 struct u_vector bt_blocks; 1228 uint32_t bt_next; 1229 1230 struct anv_reloc_list surface_relocs; 1231 /** Last seen surface state block pool center bo offset */ 1232 uint32_t last_ss_pool_center; 1233 1234 /* Serial for tracking buffer completion */ 1235 uint32_t serial; 1236 1237 /* Stream objects for storing temporary data */ 1238 struct anv_state_stream surface_state_stream; 1239 struct anv_state_stream dynamic_state_stream; 1240 1241 VkCommandBufferUsageFlags usage_flags; 1242 VkCommandBufferLevel level; 1243 1244 struct anv_cmd_state state; 1245 }; 1246 1247 VkResult anv_cmd_buffer_init_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer); 1248 void anv_cmd_buffer_fini_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer); 1249 void anv_cmd_buffer_reset_batch_bo_chain(struct anv_cmd_buffer *cmd_buffer); 1250 void anv_cmd_buffer_end_batch_buffer(struct anv_cmd_buffer *cmd_buffer); 1251 void anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary, 1252 struct anv_cmd_buffer *secondary); 1253 void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer); 1254 VkResult anv_cmd_buffer_execbuf(struct anv_device *device, 1255 struct anv_cmd_buffer *cmd_buffer); 1256 1257 VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer); 1258 1259 VkResult 1260 anv_cmd_buffer_ensure_push_constants_size(struct anv_cmd_buffer *cmd_buffer, 1261 gl_shader_stage stage, uint32_t size); 1262 #define anv_cmd_buffer_ensure_push_constant_field(cmd_buffer, stage, field) \ 1263 anv_cmd_buffer_ensure_push_constants_size(cmd_buffer, stage, \ 1264 (offsetof(struct anv_push_constants, field) + \ 1265 sizeof(cmd_buffer->state.push_constants[0]->field))) 1266 1267 struct anv_state anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer, 1268 const void *data, uint32_t size, uint32_t alignment); 1269 struct anv_state anv_cmd_buffer_merge_dynamic(struct anv_cmd_buffer *cmd_buffer, 1270 uint32_t *a, uint32_t *b, 1271 uint32_t dwords, uint32_t alignment); 1272 1273 struct anv_address 1274 anv_cmd_buffer_surface_base_address(struct anv_cmd_buffer *cmd_buffer); 1275 struct anv_state 1276 anv_cmd_buffer_alloc_binding_table(struct anv_cmd_buffer *cmd_buffer, 1277 uint32_t entries, uint32_t *state_offset); 1278 struct anv_state 1279 anv_cmd_buffer_alloc_surface_state(struct anv_cmd_buffer *cmd_buffer); 1280 struct anv_state 1281 anv_cmd_buffer_alloc_dynamic_state(struct anv_cmd_buffer *cmd_buffer, 1282 uint32_t size, uint32_t alignment); 1283 1284 VkResult 1285 anv_cmd_buffer_new_binding_table_block(struct anv_cmd_buffer *cmd_buffer); 1286 1287 void gen8_cmd_buffer_emit_viewport(struct anv_cmd_buffer *cmd_buffer); 1288 void gen8_cmd_buffer_emit_depth_viewport(struct anv_cmd_buffer *cmd_buffer, 1289 bool depth_clamp_enable); 1290 void gen7_cmd_buffer_emit_scissor(struct anv_cmd_buffer *cmd_buffer); 1291 1292 void anv_cmd_buffer_setup_attachments(struct anv_cmd_buffer *cmd_buffer, 1293 struct anv_render_pass *pass, 1294 struct anv_framebuffer *framebuffer, 1295 const VkClearValue *clear_values); 1296 1297 void anv_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer *cmd_buffer); 1298 1299 struct anv_state 1300 anv_cmd_buffer_push_constants(struct anv_cmd_buffer *cmd_buffer, 1301 gl_shader_stage stage); 1302 struct anv_state 1303 anv_cmd_buffer_cs_push_constants(struct anv_cmd_buffer *cmd_buffer); 1304 1305 void anv_cmd_buffer_clear_subpass(struct anv_cmd_buffer *cmd_buffer); 1306 void anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer *cmd_buffer); 1307 1308 const struct anv_image_view * 1309 anv_cmd_buffer_get_depth_stencil_view(const struct anv_cmd_buffer *cmd_buffer); 1310 1311 struct anv_state 1312 anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer, 1313 uint32_t num_entries, 1314 uint32_t *state_offset); 1315 1316 void anv_cmd_buffer_dump(struct anv_cmd_buffer *cmd_buffer); 1317 1318 enum anv_fence_state { 1319 /** Indicates that this is a new (or newly reset fence) */ 1320 ANV_FENCE_STATE_RESET, 1321 1322 /** Indicates that this fence has been submitted to the GPU but is still 1323 * (as far as we know) in use by the GPU. 1324 */ 1325 ANV_FENCE_STATE_SUBMITTED, 1326 1327 ANV_FENCE_STATE_SIGNALED, 1328 }; 1329 1330 struct anv_fence { 1331 struct anv_bo bo; 1332 struct drm_i915_gem_execbuffer2 execbuf; 1333 struct drm_i915_gem_exec_object2 exec2_objects[1]; 1334 enum anv_fence_state state; 1335 }; 1336 1337 struct anv_event { 1338 uint64_t semaphore; 1339 struct anv_state state; 1340 }; 1341 1342 struct anv_shader_module { 1343 unsigned char sha1[20]; 1344 uint32_t size; 1345 char data[0]; 1346 }; 1347 1348 void anv_hash_shader(unsigned char *hash, const void *key, size_t key_size, 1349 struct anv_shader_module *module, 1350 const char *entrypoint, 1351 const struct anv_pipeline_layout *pipeline_layout, 1352 const VkSpecializationInfo *spec_info); 1353 1354 static inline gl_shader_stage 1355 vk_to_mesa_shader_stage(VkShaderStageFlagBits vk_stage) 1356 { 1357 assert(__builtin_popcount(vk_stage) == 1); 1358 return ffs(vk_stage) - 1; 1359 } 1360 1361 static inline VkShaderStageFlagBits 1362 mesa_to_vk_shader_stage(gl_shader_stage mesa_stage) 1363 { 1364 return (1 << mesa_stage); 1365 } 1366 1367 #define ANV_STAGE_MASK ((1 << MESA_SHADER_STAGES) - 1) 1368 1369 #define anv_foreach_stage(stage, stage_bits) \ 1370 for (gl_shader_stage stage, \ 1371 __tmp = (gl_shader_stage)((stage_bits) & ANV_STAGE_MASK); \ 1372 stage = __builtin_ffs(__tmp) - 1, __tmp; \ 1373 __tmp &= ~(1 << (stage))) 1374 1375 struct anv_pipeline_bind_map { 1376 uint32_t surface_count; 1377 uint32_t sampler_count; 1378 uint32_t image_count; 1379 1380 struct anv_pipeline_binding * surface_to_descriptor; 1381 struct anv_pipeline_binding * sampler_to_descriptor; 1382 }; 1383 1384 struct anv_shader_bin_key { 1385 uint32_t size; 1386 uint8_t data[0]; 1387 }; 1388 1389 struct anv_shader_bin { 1390 uint32_t ref_cnt; 1391 1392 const struct anv_shader_bin_key *key; 1393 1394 struct anv_state kernel; 1395 uint32_t kernel_size; 1396 1397 const struct brw_stage_prog_data *prog_data; 1398 uint32_t prog_data_size; 1399 1400 struct anv_pipeline_bind_map bind_map; 1401 1402 /* Prog data follows, then params, then the key, all aligned to 8-bytes */ 1403 }; 1404 1405 struct anv_shader_bin * 1406 anv_shader_bin_create(struct anv_device *device, 1407 const void *key, uint32_t key_size, 1408 const void *kernel, uint32_t kernel_size, 1409 const struct brw_stage_prog_data *prog_data, 1410 uint32_t prog_data_size, const void *prog_data_param, 1411 const struct anv_pipeline_bind_map *bind_map); 1412 1413 void 1414 anv_shader_bin_destroy(struct anv_device *device, struct anv_shader_bin *shader); 1415 1416 static inline void 1417 anv_shader_bin_ref(struct anv_shader_bin *shader) 1418 { 1419 assert(shader->ref_cnt >= 1); 1420 __sync_fetch_and_add(&shader->ref_cnt, 1); 1421 } 1422 1423 static inline void 1424 anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader) 1425 { 1426 assert(shader->ref_cnt >= 1); 1427 if (__sync_fetch_and_add(&shader->ref_cnt, -1) == 1) 1428 anv_shader_bin_destroy(device, shader); 1429 } 1430 1431 struct anv_pipeline { 1432 struct anv_device * device; 1433 struct anv_batch batch; 1434 uint32_t batch_data[512]; 1435 struct anv_reloc_list batch_relocs; 1436 uint32_t dynamic_state_mask; 1437 struct anv_dynamic_state dynamic_state; 1438 1439 struct anv_pipeline_layout * layout; 1440 1441 bool needs_data_cache; 1442 1443 struct anv_shader_bin * shaders[MESA_SHADER_STAGES]; 1444 1445 struct { 1446 const struct gen_l3_config * l3_config; 1447 uint32_t total_size; 1448 } urb; 1449 1450 VkShaderStageFlags active_stages; 1451 struct anv_state blend_state; 1452 1453 uint32_t vb_used; 1454 uint32_t binding_stride[MAX_VBS]; 1455 bool instancing_enable[MAX_VBS]; 1456 bool primitive_restart; 1457 uint32_t topology; 1458 1459 uint32_t cs_right_mask; 1460 1461 bool depth_clamp_enable; 1462 1463 struct { 1464 uint32_t sf[7]; 1465 uint32_t depth_stencil_state[3]; 1466 } gen7; 1467 1468 struct { 1469 uint32_t sf[4]; 1470 uint32_t raster[5]; 1471 uint32_t wm_depth_stencil[3]; 1472 } gen8; 1473 1474 struct { 1475 uint32_t wm_depth_stencil[4]; 1476 } gen9; 1477 1478 uint32_t interface_descriptor_data[8]; 1479 }; 1480 1481 static inline bool 1482 anv_pipeline_has_stage(const struct anv_pipeline *pipeline, 1483 gl_shader_stage stage) 1484 { 1485 return (pipeline->active_stages & mesa_to_vk_shader_stage(stage)) != 0; 1486 } 1487 1488 #define ANV_DECL_GET_PROG_DATA_FUNC(prefix, stage) \ 1489 static inline const struct brw_##prefix##_prog_data * \ 1490 get_##prefix##_prog_data(const struct anv_pipeline *pipeline) \ 1491 { \ 1492 if (anv_pipeline_has_stage(pipeline, stage)) { \ 1493 return (const struct brw_##prefix##_prog_data *) \ 1494 pipeline->shaders[stage]->prog_data; \ 1495 } else { \ 1496 return NULL; \ 1497 } \ 1498 } 1499 1500 ANV_DECL_GET_PROG_DATA_FUNC(vs, MESA_SHADER_VERTEX) 1501 ANV_DECL_GET_PROG_DATA_FUNC(tcs, MESA_SHADER_TESS_CTRL) 1502 ANV_DECL_GET_PROG_DATA_FUNC(tes, MESA_SHADER_TESS_EVAL) 1503 ANV_DECL_GET_PROG_DATA_FUNC(gs, MESA_SHADER_GEOMETRY) 1504 ANV_DECL_GET_PROG_DATA_FUNC(wm, MESA_SHADER_FRAGMENT) 1505 ANV_DECL_GET_PROG_DATA_FUNC(cs, MESA_SHADER_COMPUTE) 1506 1507 static inline const struct brw_vue_prog_data * 1508 anv_pipeline_get_last_vue_prog_data(const struct anv_pipeline *pipeline) 1509 { 1510 if (anv_pipeline_has_stage(pipeline, MESA_SHADER_GEOMETRY)) 1511 return &get_gs_prog_data(pipeline)->base; 1512 else if (anv_pipeline_has_stage(pipeline, MESA_SHADER_TESS_EVAL)) 1513 return &get_tes_prog_data(pipeline)->base; 1514 else 1515 return &get_vs_prog_data(pipeline)->base; 1516 } 1517 1518 VkResult 1519 anv_pipeline_init(struct anv_pipeline *pipeline, struct anv_device *device, 1520 struct anv_pipeline_cache *cache, 1521 const VkGraphicsPipelineCreateInfo *pCreateInfo, 1522 const VkAllocationCallbacks *alloc); 1523 1524 VkResult 1525 anv_pipeline_compile_cs(struct anv_pipeline *pipeline, 1526 struct anv_pipeline_cache *cache, 1527 const VkComputePipelineCreateInfo *info, 1528 struct anv_shader_module *module, 1529 const char *entrypoint, 1530 const VkSpecializationInfo *spec_info); 1531 1532 struct anv_format { 1533 enum isl_format isl_format:16; 1534 struct isl_swizzle swizzle; 1535 }; 1536 1537 struct anv_format 1538 anv_get_format(const struct gen_device_info *devinfo, VkFormat format, 1539 VkImageAspectFlags aspect, VkImageTiling tiling); 1540 1541 static inline enum isl_format 1542 anv_get_isl_format(const struct gen_device_info *devinfo, VkFormat vk_format, 1543 VkImageAspectFlags aspect, VkImageTiling tiling) 1544 { 1545 return anv_get_format(devinfo, vk_format, aspect, tiling).isl_format; 1546 } 1547 1548 void 1549 anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm); 1550 1551 /** 1552 * Subsurface of an anv_image. 1553 */ 1554 struct anv_surface { 1555 /** Valid only if isl_surf::size > 0. */ 1556 struct isl_surf isl; 1557 1558 /** 1559 * Offset from VkImage's base address, as bound by vkBindImageMemory(). 1560 */ 1561 uint32_t offset; 1562 }; 1563 1564 struct anv_image { 1565 VkImageType type; 1566 /* The original VkFormat provided by the client. This may not match any 1567 * of the actual surface formats. 1568 */ 1569 VkFormat vk_format; 1570 VkImageAspectFlags aspects; 1571 VkExtent3D extent; 1572 uint32_t levels; 1573 uint32_t array_size; 1574 uint32_t samples; /**< VkImageCreateInfo::samples */ 1575 VkImageUsageFlags usage; /**< Superset of VkImageCreateInfo::usage. */ 1576 VkImageTiling tiling; /** VkImageCreateInfo::tiling */ 1577 1578 VkDeviceSize size; 1579 uint32_t alignment; 1580 1581 /* Set when bound */ 1582 struct anv_bo *bo; 1583 VkDeviceSize offset; 1584 1585 /** 1586 * Image subsurfaces 1587 * 1588 * For each foo, anv_image::foo_surface is valid if and only if 1589 * anv_image::aspects has a foo aspect. 1590 * 1591 * The hardware requires that the depth buffer and stencil buffer be 1592 * separate surfaces. From Vulkan's perspective, though, depth and stencil 1593 * reside in the same VkImage. To satisfy both the hardware and Vulkan, we 1594 * allocate the depth and stencil buffers as separate surfaces in the same 1595 * bo. 1596 */ 1597 union { 1598 struct anv_surface color_surface; 1599 1600 struct { 1601 struct anv_surface depth_surface; 1602 struct anv_surface stencil_surface; 1603 }; 1604 }; 1605 1606 /** 1607 * For color images, this is the aux usage for this image when not used as a 1608 * color attachment. 1609 * 1610 * For depth/stencil images, this is set to ISL_AUX_USAGE_HIZ if the image 1611 * has a HiZ buffer. 1612 */ 1613 enum isl_aux_usage aux_usage; 1614 1615 struct anv_surface aux_surface; 1616 }; 1617 1618 /* Returns true if a HiZ-enabled depth buffer can be sampled from. */ 1619 static inline bool 1620 anv_can_sample_with_hiz(uint8_t gen, uint32_t samples) 1621 { 1622 return gen >= 8 && samples == 1; 1623 } 1624 1625 void 1626 anv_gen8_hiz_op_resolve(struct anv_cmd_buffer *cmd_buffer, 1627 const struct anv_image *image, 1628 enum blorp_hiz_op op); 1629 1630 /* This is defined as a macro so that it works for both 1631 * VkImageSubresourceRange and VkImageSubresourceLayers 1632 */ 1633 #define anv_get_layerCount(_image, _range) \ 1634 ((_range)->layerCount == VK_REMAINING_ARRAY_LAYERS ? \ 1635 (_image)->array_size - (_range)->baseArrayLayer : (_range)->layerCount) 1636 1637 static inline uint32_t 1638 anv_get_levelCount(const struct anv_image *image, 1639 const VkImageSubresourceRange *range) 1640 { 1641 return range->levelCount == VK_REMAINING_MIP_LEVELS ? 1642 image->levels - range->baseMipLevel : range->levelCount; 1643 } 1644 1645 1646 struct anv_image_view { 1647 const struct anv_image *image; /**< VkImageViewCreateInfo::image */ 1648 struct anv_bo *bo; 1649 uint32_t offset; /**< Offset into bo. */ 1650 1651 struct isl_view isl; 1652 1653 VkImageAspectFlags aspect_mask; 1654 VkFormat vk_format; 1655 VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */ 1656 1657 /** RENDER_SURFACE_STATE when using image as a sampler surface. */ 1658 struct anv_state sampler_surface_state; 1659 1660 /** RENDER_SURFACE_STATE when using image as a storage image. */ 1661 struct anv_state storage_surface_state; 1662 1663 struct brw_image_param storage_image_param; 1664 }; 1665 1666 struct anv_image_create_info { 1667 const VkImageCreateInfo *vk_info; 1668 1669 /** An opt-in bitmask which filters an ISL-mapping of the Vulkan tiling. */ 1670 isl_tiling_flags_t isl_tiling_flags; 1671 1672 uint32_t stride; 1673 }; 1674 1675 VkResult anv_image_create(VkDevice _device, 1676 const struct anv_image_create_info *info, 1677 const VkAllocationCallbacks* alloc, 1678 VkImage *pImage); 1679 1680 const struct anv_surface * 1681 anv_image_get_surface_for_aspect_mask(const struct anv_image *image, 1682 VkImageAspectFlags aspect_mask); 1683 1684 struct anv_buffer_view { 1685 enum isl_format format; /**< VkBufferViewCreateInfo::format */ 1686 struct anv_bo *bo; 1687 uint32_t offset; /**< Offset into bo. */ 1688 uint64_t range; /**< VkBufferViewCreateInfo::range */ 1689 1690 struct anv_state surface_state; 1691 struct anv_state storage_surface_state; 1692 1693 struct brw_image_param storage_image_param; 1694 }; 1695 1696 enum isl_format 1697 anv_isl_format_for_descriptor_type(VkDescriptorType type); 1698 1699 static inline struct VkExtent3D 1700 anv_sanitize_image_extent(const VkImageType imageType, 1701 const struct VkExtent3D imageExtent) 1702 { 1703 switch (imageType) { 1704 case VK_IMAGE_TYPE_1D: 1705 return (VkExtent3D) { imageExtent.width, 1, 1 }; 1706 case VK_IMAGE_TYPE_2D: 1707 return (VkExtent3D) { imageExtent.width, imageExtent.height, 1 }; 1708 case VK_IMAGE_TYPE_3D: 1709 return imageExtent; 1710 default: 1711 unreachable("invalid image type"); 1712 } 1713 } 1714 1715 static inline struct VkOffset3D 1716 anv_sanitize_image_offset(const VkImageType imageType, 1717 const struct VkOffset3D imageOffset) 1718 { 1719 switch (imageType) { 1720 case VK_IMAGE_TYPE_1D: 1721 return (VkOffset3D) { imageOffset.x, 0, 0 }; 1722 case VK_IMAGE_TYPE_2D: 1723 return (VkOffset3D) { imageOffset.x, imageOffset.y, 0 }; 1724 case VK_IMAGE_TYPE_3D: 1725 return imageOffset; 1726 default: 1727 unreachable("invalid image type"); 1728 } 1729 } 1730 1731 1732 void anv_fill_buffer_surface_state(struct anv_device *device, 1733 struct anv_state state, 1734 enum isl_format format, 1735 uint32_t offset, uint32_t range, 1736 uint32_t stride); 1737 1738 void anv_image_view_fill_image_param(struct anv_device *device, 1739 struct anv_image_view *view, 1740 struct brw_image_param *param); 1741 void anv_buffer_view_fill_image_param(struct anv_device *device, 1742 struct anv_buffer_view *view, 1743 struct brw_image_param *param); 1744 1745 struct anv_sampler { 1746 uint32_t state[4]; 1747 }; 1748 1749 struct anv_framebuffer { 1750 uint32_t width; 1751 uint32_t height; 1752 uint32_t layers; 1753 1754 uint32_t attachment_count; 1755 struct anv_image_view * attachments[0]; 1756 }; 1757 1758 struct anv_subpass { 1759 uint32_t input_count; 1760 uint32_t * input_attachments; 1761 uint32_t color_count; 1762 uint32_t * color_attachments; 1763 uint32_t * resolve_attachments; 1764 1765 /* TODO: Consider storing the depth/stencil VkAttachmentReference 1766 * instead of its two structure members (below) individually. 1767 */ 1768 uint32_t depth_stencil_attachment; 1769 VkImageLayout depth_stencil_layout; 1770 1771 /** Subpass has a depth/stencil self-dependency */ 1772 bool has_ds_self_dep; 1773 1774 /** Subpass has at least one resolve attachment */ 1775 bool has_resolve; 1776 }; 1777 1778 enum anv_subpass_usage { 1779 ANV_SUBPASS_USAGE_DRAW = (1 << 0), 1780 ANV_SUBPASS_USAGE_INPUT = (1 << 1), 1781 ANV_SUBPASS_USAGE_RESOLVE_SRC = (1 << 2), 1782 ANV_SUBPASS_USAGE_RESOLVE_DST = (1 << 3), 1783 }; 1784 1785 struct anv_render_pass_attachment { 1786 /* TODO: Consider using VkAttachmentDescription instead of storing each of 1787 * its members individually. 1788 */ 1789 VkFormat format; 1790 uint32_t samples; 1791 VkImageUsageFlags usage; 1792 VkAttachmentLoadOp load_op; 1793 VkAttachmentStoreOp store_op; 1794 VkAttachmentLoadOp stencil_load_op; 1795 VkImageLayout initial_layout; 1796 VkImageLayout final_layout; 1797 1798 /* An array, indexed by subpass id, of how the attachment will be used. */ 1799 enum anv_subpass_usage * subpass_usage; 1800 1801 /* The subpass id in which the attachment will be used last. */ 1802 uint32_t last_subpass_idx; 1803 }; 1804 1805 struct anv_render_pass { 1806 uint32_t attachment_count; 1807 uint32_t subpass_count; 1808 uint32_t * subpass_attachments; 1809 enum anv_subpass_usage * subpass_usages; 1810 struct anv_render_pass_attachment * attachments; 1811 struct anv_subpass subpasses[0]; 1812 }; 1813 1814 struct anv_query_pool_slot { 1815 uint64_t begin; 1816 uint64_t end; 1817 uint64_t available; 1818 }; 1819 1820 struct anv_query_pool { 1821 VkQueryType type; 1822 uint32_t slots; 1823 struct anv_bo bo; 1824 }; 1825 1826 void *anv_lookup_entrypoint(const struct gen_device_info *devinfo, 1827 const char *name); 1828 1829 void anv_dump_image_to_ppm(struct anv_device *device, 1830 struct anv_image *image, unsigned miplevel, 1831 unsigned array_layer, VkImageAspectFlagBits aspect, 1832 const char *filename); 1833 1834 enum anv_dump_action { 1835 ANV_DUMP_FRAMEBUFFERS_BIT = 0x1, 1836 }; 1837 1838 void anv_dump_start(struct anv_device *device, enum anv_dump_action actions); 1839 void anv_dump_finish(void); 1840 1841 void anv_dump_add_framebuffer(struct anv_cmd_buffer *cmd_buffer, 1842 struct anv_framebuffer *fb); 1843 1844 #define ANV_DEFINE_HANDLE_CASTS(__anv_type, __VkType) \ 1845 \ 1846 static inline struct __anv_type * \ 1847 __anv_type ## _from_handle(__VkType _handle) \ 1848 { \ 1849 return (struct __anv_type *) _handle; \ 1850 } \ 1851 \ 1852 static inline __VkType \ 1853 __anv_type ## _to_handle(struct __anv_type *_obj) \ 1854 { \ 1855 return (__VkType) _obj; \ 1856 } 1857 1858 #define ANV_DEFINE_NONDISP_HANDLE_CASTS(__anv_type, __VkType) \ 1859 \ 1860 static inline struct __anv_type * \ 1861 __anv_type ## _from_handle(__VkType _handle) \ 1862 { \ 1863 return (struct __anv_type *)(uintptr_t) _handle; \ 1864 } \ 1865 \ 1866 static inline __VkType \ 1867 __anv_type ## _to_handle(struct __anv_type *_obj) \ 1868 { \ 1869 return (__VkType)(uintptr_t) _obj; \ 1870 } 1871 1872 #define ANV_FROM_HANDLE(__anv_type, __name, __handle) \ 1873 struct __anv_type *__name = __anv_type ## _from_handle(__handle) 1874 1875 ANV_DEFINE_HANDLE_CASTS(anv_cmd_buffer, VkCommandBuffer) 1876 ANV_DEFINE_HANDLE_CASTS(anv_device, VkDevice) 1877 ANV_DEFINE_HANDLE_CASTS(anv_instance, VkInstance) 1878 ANV_DEFINE_HANDLE_CASTS(anv_physical_device, VkPhysicalDevice) 1879 ANV_DEFINE_HANDLE_CASTS(anv_queue, VkQueue) 1880 1881 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_cmd_pool, VkCommandPool) 1882 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_buffer, VkBuffer) 1883 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_buffer_view, VkBufferView) 1884 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_pool, VkDescriptorPool) 1885 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_set, VkDescriptorSet) 1886 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_descriptor_set_layout, VkDescriptorSetLayout) 1887 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_device_memory, VkDeviceMemory) 1888 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_fence, VkFence) 1889 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_event, VkEvent) 1890 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_framebuffer, VkFramebuffer) 1891 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_image, VkImage) 1892 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_image_view, VkImageView); 1893 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_pipeline_cache, VkPipelineCache) 1894 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_pipeline, VkPipeline) 1895 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_pipeline_layout, VkPipelineLayout) 1896 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_query_pool, VkQueryPool) 1897 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_render_pass, VkRenderPass) 1898 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_sampler, VkSampler) 1899 ANV_DEFINE_NONDISP_HANDLE_CASTS(anv_shader_module, VkShaderModule) 1900 1901 /* Gen-specific function declarations */ 1902 #ifdef genX 1903 # include "anv_genX.h" 1904 #else 1905 # define genX(x) gen7_##x 1906 # include "anv_genX.h" 1907 # undef genX 1908 # define genX(x) gen75_##x 1909 # include "anv_genX.h" 1910 # undef genX 1911 # define genX(x) gen8_##x 1912 # include "anv_genX.h" 1913 # undef genX 1914 # define genX(x) gen9_##x 1915 # include "anv_genX.h" 1916 # undef genX 1917 #endif 1918 1919 #ifdef __cplusplus 1920 } 1921 #endif 1922 1923 #endif /* ANV_PRIVATE_H */ 1924
