1 /* 2 * Copyright 2014-2015 Broadcom 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #include <errno.h> 25 #include <err.h> 26 #include <sys/mman.h> 27 #include <fcntl.h> 28 #include <xf86drm.h> 29 #include <xf86drmMode.h> 30 31 #include "util/u_hash_table.h" 32 #include "util/u_memory.h" 33 #include "util/ralloc.h" 34 35 #include "vc4_context.h" 36 #include "vc4_screen.h" 37 38 #ifdef HAVE_VALGRIND 39 #include <valgrind.h> 40 #include <memcheck.h> 41 #define VG(x) x 42 #else 43 #define VG(x) 44 #endif 45 46 static bool dump_stats = false; 47 48 static void 49 vc4_bo_cache_free_all(struct vc4_bo_cache *cache); 50 51 static void 52 vc4_bo_dump_stats(struct vc4_screen *screen) 53 { 54 struct vc4_bo_cache *cache = &screen->bo_cache; 55 56 fprintf(stderr, " BOs allocated: %d\n", screen->bo_count); 57 fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024); 58 fprintf(stderr, " BOs cached: %d\n", cache->bo_count); 59 fprintf(stderr, " BOs cached size: %dkb\n", cache->bo_size / 1024); 60 61 if (!list_empty(&cache->time_list)) { 62 struct vc4_bo *first = LIST_ENTRY(struct vc4_bo, 63 cache->time_list.next, 64 time_list); 65 struct vc4_bo *last = LIST_ENTRY(struct vc4_bo, 66 cache->time_list.prev, 67 time_list); 68 69 fprintf(stderr, " oldest cache time: %ld\n", 70 (long)first->free_time); 71 fprintf(stderr, " newest cache time: %ld\n", 72 (long)last->free_time); 73 74 struct timespec time; 75 clock_gettime(CLOCK_MONOTONIC, &time); 76 fprintf(stderr, " now: %ld\n", 77 time.tv_sec); 78 } 79 } 80 81 static void 82 vc4_bo_remove_from_cache(struct vc4_bo_cache *cache, struct vc4_bo *bo) 83 { 84 list_del(&bo->time_list); 85 list_del(&bo->size_list); 86 cache->bo_count--; 87 cache->bo_size -= bo->size; 88 } 89 90 static struct vc4_bo * 91 vc4_bo_from_cache(struct vc4_screen *screen, uint32_t size, const char *name) 92 { 93 struct vc4_bo_cache *cache = &screen->bo_cache; 94 uint32_t page_index = size / 4096 - 1; 95 96 if (cache->size_list_size <= page_index) 97 return NULL; 98 99 struct vc4_bo *bo = NULL; 100 pipe_mutex_lock(cache->lock); 101 if (!list_empty(&cache->size_list[page_index])) { 102 bo = LIST_ENTRY(struct vc4_bo, cache->size_list[page_index].next, 103 size_list); 104 105 /* Check that the BO has gone idle. If not, then we want to 106 * allocate something new instead, since we assume that the 107 * user will proceed to CPU map it and fill it with stuff. 108 */ 109 if (!vc4_bo_wait(bo, 0, NULL)) { 110 pipe_mutex_unlock(cache->lock); 111 return NULL; 112 } 113 114 pipe_reference_init(&bo->reference, 1); 115 vc4_bo_remove_from_cache(cache, bo); 116 117 bo->name = name; 118 } 119 pipe_mutex_unlock(cache->lock); 120 return bo; 121 } 122 123 struct vc4_bo * 124 vc4_bo_alloc(struct vc4_screen *screen, uint32_t size, const char *name) 125 { 126 struct vc4_bo *bo; 127 int ret; 128 129 size = align(size, 4096); 130 131 bo = vc4_bo_from_cache(screen, size, name); 132 if (bo) { 133 if (dump_stats) { 134 fprintf(stderr, "Allocated %s %dkb from cache:\n", 135 name, size / 1024); 136 vc4_bo_dump_stats(screen); 137 } 138 return bo; 139 } 140 141 bo = CALLOC_STRUCT(vc4_bo); 142 if (!bo) 143 return NULL; 144 145 pipe_reference_init(&bo->reference, 1); 146 bo->screen = screen; 147 bo->size = size; 148 bo->name = name; 149 bo->private = true; 150 151 retry: 152 ; 153 154 bool cleared_and_retried = false; 155 struct drm_vc4_create_bo create = { 156 .size = size 157 }; 158 159 ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_BO, &create); 160 bo->handle = create.handle; 161 162 if (ret != 0) { 163 if (!list_empty(&screen->bo_cache.time_list) && 164 !cleared_and_retried) { 165 cleared_and_retried = true; 166 vc4_bo_cache_free_all(&screen->bo_cache); 167 goto retry; 168 } 169 170 free(bo); 171 return NULL; 172 } 173 174 screen->bo_count++; 175 screen->bo_size += bo->size; 176 if (dump_stats) { 177 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024); 178 vc4_bo_dump_stats(screen); 179 } 180 181 return bo; 182 } 183 184 void 185 vc4_bo_last_unreference(struct vc4_bo *bo) 186 { 187 struct vc4_screen *screen = bo->screen; 188 189 struct timespec time; 190 clock_gettime(CLOCK_MONOTONIC, &time); 191 pipe_mutex_lock(screen->bo_cache.lock); 192 vc4_bo_last_unreference_locked_timed(bo, time.tv_sec); 193 pipe_mutex_unlock(screen->bo_cache.lock); 194 } 195 196 static void 197 vc4_bo_free(struct vc4_bo *bo) 198 { 199 struct vc4_screen *screen = bo->screen; 200 201 if (bo->map) { 202 if (using_vc4_simulator && bo->name && 203 strcmp(bo->name, "winsys") == 0) { 204 free(bo->map); 205 } else { 206 munmap(bo->map, bo->size); 207 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0)); 208 } 209 } 210 211 struct drm_gem_close c; 212 memset(&c, 0, sizeof(c)); 213 c.handle = bo->handle; 214 int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c); 215 if (ret != 0) 216 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno)); 217 218 screen->bo_count--; 219 screen->bo_size -= bo->size; 220 221 if (dump_stats) { 222 fprintf(stderr, "Freed %s%s%dkb:\n", 223 bo->name ? bo->name : "", 224 bo->name ? " " : "", 225 bo->size / 1024); 226 vc4_bo_dump_stats(screen); 227 } 228 229 free(bo); 230 } 231 232 static void 233 free_stale_bos(struct vc4_screen *screen, time_t time) 234 { 235 struct vc4_bo_cache *cache = &screen->bo_cache; 236 bool freed_any = false; 237 238 list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list, 239 time_list) { 240 if (dump_stats && !freed_any) { 241 fprintf(stderr, "Freeing stale BOs:\n"); 242 vc4_bo_dump_stats(screen); 243 freed_any = true; 244 } 245 246 /* If it's more than a second old, free it. */ 247 if (time - bo->free_time > 2) { 248 vc4_bo_remove_from_cache(cache, bo); 249 vc4_bo_free(bo); 250 } else { 251 break; 252 } 253 } 254 255 if (dump_stats && freed_any) { 256 fprintf(stderr, "Freed stale BOs:\n"); 257 vc4_bo_dump_stats(screen); 258 } 259 } 260 261 static void 262 vc4_bo_cache_free_all(struct vc4_bo_cache *cache) 263 { 264 pipe_mutex_lock(cache->lock); 265 list_for_each_entry_safe(struct vc4_bo, bo, &cache->time_list, 266 time_list) { 267 vc4_bo_remove_from_cache(cache, bo); 268 vc4_bo_free(bo); 269 } 270 pipe_mutex_unlock(cache->lock); 271 } 272 273 void 274 vc4_bo_last_unreference_locked_timed(struct vc4_bo *bo, time_t time) 275 { 276 struct vc4_screen *screen = bo->screen; 277 struct vc4_bo_cache *cache = &screen->bo_cache; 278 uint32_t page_index = bo->size / 4096 - 1; 279 280 if (!bo->private) { 281 vc4_bo_free(bo); 282 return; 283 } 284 285 if (cache->size_list_size <= page_index) { 286 struct list_head *new_list = 287 ralloc_array(screen, struct list_head, page_index + 1); 288 289 /* Move old list contents over (since the array has moved, and 290 * therefore the pointers to the list heads have to change). 291 */ 292 for (int i = 0; i < cache->size_list_size; i++) 293 list_replace(&cache->size_list[i], &new_list[i]); 294 for (int i = cache->size_list_size; i < page_index + 1; i++) 295 list_inithead(&new_list[i]); 296 297 cache->size_list = new_list; 298 cache->size_list_size = page_index + 1; 299 } 300 301 bo->free_time = time; 302 list_addtail(&bo->size_list, &cache->size_list[page_index]); 303 list_addtail(&bo->time_list, &cache->time_list); 304 cache->bo_count++; 305 cache->bo_size += bo->size; 306 if (dump_stats) { 307 fprintf(stderr, "Freed %s %dkb to cache:\n", 308 bo->name, bo->size / 1024); 309 vc4_bo_dump_stats(screen); 310 } 311 bo->name = NULL; 312 313 free_stale_bos(screen, time); 314 } 315 316 static struct vc4_bo * 317 vc4_bo_open_handle(struct vc4_screen *screen, 318 uint32_t winsys_stride, 319 uint32_t handle, uint32_t size) 320 { 321 struct vc4_bo *bo; 322 323 assert(size); 324 325 pipe_mutex_lock(screen->bo_handles_mutex); 326 327 bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle); 328 if (bo) { 329 pipe_reference(NULL, &bo->reference); 330 goto done; 331 } 332 333 bo = CALLOC_STRUCT(vc4_bo); 334 pipe_reference_init(&bo->reference, 1); 335 bo->screen = screen; 336 bo->handle = handle; 337 bo->size = size; 338 bo->name = "winsys"; 339 bo->private = false; 340 341 #ifdef USE_VC4_SIMULATOR 342 vc4_simulator_open_from_handle(screen->fd, winsys_stride, 343 bo->handle, bo->size); 344 bo->map = malloc(bo->size); 345 #endif 346 347 util_hash_table_set(screen->bo_handles, (void *)(uintptr_t)handle, bo); 348 349 done: 350 pipe_mutex_unlock(screen->bo_handles_mutex); 351 return bo; 352 } 353 354 struct vc4_bo * 355 vc4_bo_open_name(struct vc4_screen *screen, uint32_t name, 356 uint32_t winsys_stride) 357 { 358 struct drm_gem_open o = { 359 .name = name 360 }; 361 int ret = vc4_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o); 362 if (ret) { 363 fprintf(stderr, "Failed to open bo %d: %s\n", 364 name, strerror(errno)); 365 return NULL; 366 } 367 368 return vc4_bo_open_handle(screen, winsys_stride, o.handle, o.size); 369 } 370 371 struct vc4_bo * 372 vc4_bo_open_dmabuf(struct vc4_screen *screen, int fd, uint32_t winsys_stride) 373 { 374 uint32_t handle; 375 int ret = drmPrimeFDToHandle(screen->fd, fd, &handle); 376 int size; 377 if (ret) { 378 fprintf(stderr, "Failed to get vc4 handle for dmabuf %d\n", fd); 379 return NULL; 380 } 381 382 /* Determine the size of the bo we were handed. */ 383 size = lseek(fd, 0, SEEK_END); 384 if (size == -1) { 385 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd); 386 return NULL; 387 } 388 389 return vc4_bo_open_handle(screen, winsys_stride, handle, size); 390 } 391 392 int 393 vc4_bo_get_dmabuf(struct vc4_bo *bo) 394 { 395 int fd; 396 int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle, 397 O_CLOEXEC, &fd); 398 if (ret != 0) { 399 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n", 400 bo->handle); 401 return -1; 402 } 403 404 pipe_mutex_lock(bo->screen->bo_handles_mutex); 405 bo->private = false; 406 util_hash_table_set(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo); 407 pipe_mutex_unlock(bo->screen->bo_handles_mutex); 408 409 return fd; 410 } 411 412 struct vc4_bo * 413 vc4_bo_alloc_shader(struct vc4_screen *screen, const void *data, uint32_t size) 414 { 415 struct vc4_bo *bo; 416 int ret; 417 418 bo = CALLOC_STRUCT(vc4_bo); 419 if (!bo) 420 return NULL; 421 422 pipe_reference_init(&bo->reference, 1); 423 bo->screen = screen; 424 bo->size = align(size, 4096); 425 bo->name = "code"; 426 bo->private = false; /* Make sure it doesn't go back to the cache. */ 427 428 struct drm_vc4_create_shader_bo create = { 429 .size = size, 430 .data = (uintptr_t)data, 431 }; 432 433 ret = vc4_ioctl(screen->fd, DRM_IOCTL_VC4_CREATE_SHADER_BO, 434 &create); 435 bo->handle = create.handle; 436 437 if (ret != 0) { 438 fprintf(stderr, "create shader ioctl failure\n"); 439 abort(); 440 } 441 442 screen->bo_count++; 443 screen->bo_size += bo->size; 444 if (dump_stats) { 445 fprintf(stderr, "Allocated shader %dkb:\n", bo->size / 1024); 446 vc4_bo_dump_stats(screen); 447 } 448 449 return bo; 450 } 451 452 bool 453 vc4_bo_flink(struct vc4_bo *bo, uint32_t *name) 454 { 455 struct drm_gem_flink flink = { 456 .handle = bo->handle, 457 }; 458 int ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink); 459 if (ret) { 460 fprintf(stderr, "Failed to flink bo %d: %s\n", 461 bo->handle, strerror(errno)); 462 free(bo); 463 return false; 464 } 465 466 bo->private = false; 467 *name = flink.name; 468 469 return true; 470 } 471 472 static int vc4_wait_seqno_ioctl(int fd, uint64_t seqno, uint64_t timeout_ns) 473 { 474 struct drm_vc4_wait_seqno wait = { 475 .seqno = seqno, 476 .timeout_ns = timeout_ns, 477 }; 478 int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_SEQNO, &wait); 479 if (ret == -1) 480 return -errno; 481 else 482 return 0; 483 484 } 485 486 bool 487 vc4_wait_seqno(struct vc4_screen *screen, uint64_t seqno, uint64_t timeout_ns, 488 const char *reason) 489 { 490 if (screen->finished_seqno >= seqno) 491 return true; 492 493 if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) { 494 if (vc4_wait_seqno_ioctl(screen->fd, seqno, 0) == -ETIME) { 495 fprintf(stderr, "Blocking on seqno %lld for %s\n", 496 (long long)seqno, reason); 497 } 498 } 499 500 int ret = vc4_wait_seqno_ioctl(screen->fd, seqno, timeout_ns); 501 if (ret) { 502 if (ret != -ETIME) { 503 fprintf(stderr, "wait failed: %d\n", ret); 504 abort(); 505 } 506 507 return false; 508 } 509 510 screen->finished_seqno = seqno; 511 return true; 512 } 513 514 static int vc4_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns) 515 { 516 struct drm_vc4_wait_bo wait = { 517 .handle = handle, 518 .timeout_ns = timeout_ns, 519 }; 520 int ret = vc4_ioctl(fd, DRM_IOCTL_VC4_WAIT_BO, &wait); 521 if (ret == -1) 522 return -errno; 523 else 524 return 0; 525 526 } 527 528 bool 529 vc4_bo_wait(struct vc4_bo *bo, uint64_t timeout_ns, const char *reason) 530 { 531 struct vc4_screen *screen = bo->screen; 532 533 if (unlikely(vc4_debug & VC4_DEBUG_PERF) && timeout_ns && reason) { 534 if (vc4_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) { 535 fprintf(stderr, "Blocking on %s BO for %s\n", 536 bo->name, reason); 537 } 538 } 539 540 int ret = vc4_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns); 541 if (ret) { 542 if (ret != -ETIME) { 543 fprintf(stderr, "wait failed: %d\n", ret); 544 abort(); 545 } 546 547 return false; 548 } 549 550 return true; 551 } 552 553 void * 554 vc4_bo_map_unsynchronized(struct vc4_bo *bo) 555 { 556 uint64_t offset; 557 int ret; 558 559 if (bo->map) 560 return bo->map; 561 562 struct drm_vc4_mmap_bo map; 563 memset(&map, 0, sizeof(map)); 564 map.handle = bo->handle; 565 ret = vc4_ioctl(bo->screen->fd, DRM_IOCTL_VC4_MMAP_BO, &map); 566 offset = map.offset; 567 if (ret != 0) { 568 fprintf(stderr, "map ioctl failure\n"); 569 abort(); 570 } 571 572 bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED, 573 bo->screen->fd, offset); 574 if (bo->map == MAP_FAILED) { 575 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n", 576 bo->handle, (long long)offset, bo->size); 577 abort(); 578 } 579 VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false)); 580 581 return bo->map; 582 } 583 584 void * 585 vc4_bo_map(struct vc4_bo *bo) 586 { 587 void *map = vc4_bo_map_unsynchronized(bo); 588 589 bool ok = vc4_bo_wait(bo, PIPE_TIMEOUT_INFINITE, "bo map"); 590 if (!ok) { 591 fprintf(stderr, "BO wait for map failed\n"); 592 abort(); 593 } 594 595 return map; 596 } 597 598 void 599 vc4_bufmgr_destroy(struct pipe_screen *pscreen) 600 { 601 struct vc4_screen *screen = vc4_screen(pscreen); 602 struct vc4_bo_cache *cache = &screen->bo_cache; 603 604 vc4_bo_cache_free_all(cache); 605 606 if (dump_stats) { 607 fprintf(stderr, "BO stats after screen destroy:\n"); 608 vc4_bo_dump_stats(screen); 609 } 610 } 611
