1 /************************************************************************** 2 * 3 * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 /* Originally a fake version of the buffer manager so that we can 29 * prototype the changes in a driver fairly quickly, has been fleshed 30 * out to a fully functional interim solution. 31 * 32 * Basically wraps the old style memory management in the new 33 * programming interface, but is more expressive and avoids many of 34 * the bugs in the old texture manager. 35 */ 36 37 #ifdef HAVE_CONFIG_H 38 #include "config.h" 39 #endif 40 41 #include <stdlib.h> 42 #include <string.h> 43 #include <assert.h> 44 #include <errno.h> 45 #include <strings.h> 46 #include <xf86drm.h> 47 #include <pthread.h> 48 #include "intel_bufmgr.h" 49 #include "intel_bufmgr_priv.h" 50 #include "drm.h" 51 #include "i915_drm.h" 52 #include "mm.h" 53 #include "libdrm_macros.h" 54 #include "libdrm_lists.h" 55 56 #define DBG(...) do { \ 57 if (bufmgr_fake->bufmgr.debug) \ 58 drmMsg(__VA_ARGS__); \ 59 } while (0) 60 61 /* Internal flags: 62 */ 63 #define BM_NO_BACKING_STORE 0x00000001 64 #define BM_NO_FENCE_SUBDATA 0x00000002 65 #define BM_PINNED 0x00000004 66 67 /* Wrapper around mm.c's mem_block, which understands that you must 68 * wait for fences to expire before memory can be freed. This is 69 * specific to our use of memcpy for uploads - an upload that was 70 * processed through the command queue wouldn't need to care about 71 * fences. 72 */ 73 #define MAX_RELOCS 4096 74 75 struct fake_buffer_reloc { 76 /** Buffer object that the relocation points at. */ 77 drm_intel_bo *target_buf; 78 /** Offset of the relocation entry within reloc_buf. */ 79 uint32_t offset; 80 /** 81 * Cached value of the offset when we last performed this relocation. 82 */ 83 uint32_t last_target_offset; 84 /** Value added to target_buf's offset to get the relocation entry. */ 85 uint32_t delta; 86 /** Cache domains the target buffer is read into. */ 87 uint32_t read_domains; 88 /** Cache domain the target buffer will have dirty cachelines in. */ 89 uint32_t write_domain; 90 }; 91 92 struct block { 93 struct block *next, *prev; 94 struct mem_block *mem; /* BM_MEM_AGP */ 95 96 /** 97 * Marks that the block is currently in the aperture and has yet to be 98 * fenced. 99 */ 100 unsigned on_hardware:1; 101 /** 102 * Marks that the block is currently fenced (being used by rendering) 103 * and can't be freed until @fence is passed. 104 */ 105 unsigned fenced:1; 106 107 /** Fence cookie for the block. */ 108 unsigned fence; /* Split to read_fence, write_fence */ 109 110 drm_intel_bo *bo; 111 void *virtual; 112 }; 113 114 typedef struct _bufmgr_fake { 115 drm_intel_bufmgr bufmgr; 116 117 pthread_mutex_t lock; 118 119 unsigned long low_offset; 120 unsigned long size; 121 void *virtual; 122 123 struct mem_block *heap; 124 125 unsigned buf_nr; /* for generating ids */ 126 127 /** 128 * List of blocks which are currently in the GART but haven't been 129 * fenced yet. 130 */ 131 struct block on_hardware; 132 /** 133 * List of blocks which are in the GART and have an active fence on 134 * them. 135 */ 136 struct block fenced; 137 /** 138 * List of blocks which have an expired fence and are ready to be 139 * evicted. 140 */ 141 struct block lru; 142 143 unsigned int last_fence; 144 145 unsigned fail:1; 146 unsigned need_fence:1; 147 int thrashing; 148 149 /** 150 * Driver callback to emit a fence, returning the cookie. 151 * 152 * This allows the driver to hook in a replacement for the DRM usage in 153 * bufmgr_fake. 154 * 155 * Currently, this also requires that a write flush be emitted before 156 * emitting the fence, but this should change. 157 */ 158 unsigned int (*fence_emit) (void *private); 159 /** Driver callback to wait for a fence cookie to have passed. */ 160 void (*fence_wait) (unsigned int fence, void *private); 161 void *fence_priv; 162 163 /** 164 * Driver callback to execute a buffer. 165 * 166 * This allows the driver to hook in a replacement for the DRM usage in 167 * bufmgr_fake. 168 */ 169 int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv); 170 void *exec_priv; 171 172 /** Driver-supplied argument to driver callbacks */ 173 void *driver_priv; 174 /** 175 * Pointer to kernel-updated sarea data for the last completed user irq 176 */ 177 volatile int *last_dispatch; 178 179 int fd; 180 181 int debug; 182 183 int performed_rendering; 184 } drm_intel_bufmgr_fake; 185 186 typedef struct _drm_intel_bo_fake { 187 drm_intel_bo bo; 188 189 unsigned id; /* debug only */ 190 const char *name; 191 192 unsigned dirty:1; 193 /** 194 * has the card written to this buffer - we make need to copy it back 195 */ 196 unsigned card_dirty:1; 197 unsigned int refcount; 198 /* Flags may consist of any of the DRM_BO flags, plus 199 * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the 200 * first two driver private flags. 201 */ 202 uint64_t flags; 203 /** Cache domains the target buffer is read into. */ 204 uint32_t read_domains; 205 /** Cache domain the target buffer will have dirty cachelines in. */ 206 uint32_t write_domain; 207 208 unsigned int alignment; 209 int is_static, validated; 210 unsigned int map_count; 211 212 /** relocation list */ 213 struct fake_buffer_reloc *relocs; 214 int nr_relocs; 215 /** 216 * Total size of the target_bos of this buffer. 217 * 218 * Used for estimation in check_aperture. 219 */ 220 unsigned int child_size; 221 222 struct block *block; 223 void *backing_store; 224 void (*invalidate_cb) (drm_intel_bo *bo, void *ptr); 225 void *invalidate_ptr; 226 } drm_intel_bo_fake; 227 228 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, 229 unsigned int fence_cookie); 230 231 #define MAXFENCE 0x7fffffff 232 233 static int 234 FENCE_LTE(unsigned a, unsigned b) 235 { 236 if (a == b) 237 return 1; 238 239 if (a < b && b - a < (1 << 24)) 240 return 1; 241 242 if (a > b && MAXFENCE - a + b < (1 << 24)) 243 return 1; 244 245 return 0; 246 } 247 248 void 249 drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr, 250 unsigned int (*emit) (void *priv), 251 void (*wait) (unsigned int fence, 252 void *priv), 253 void *priv) 254 { 255 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 256 257 bufmgr_fake->fence_emit = emit; 258 bufmgr_fake->fence_wait = wait; 259 bufmgr_fake->fence_priv = priv; 260 } 261 262 static unsigned int 263 _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake) 264 { 265 struct drm_i915_irq_emit ie; 266 int ret, seq = 1; 267 268 if (bufmgr_fake->fence_emit != NULL) { 269 seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv); 270 return seq; 271 } 272 273 ie.irq_seq = &seq; 274 ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT, 275 &ie, sizeof(ie)); 276 if (ret) { 277 drmMsg("%s: drm_i915_irq_emit: %d\n", __func__, ret); 278 abort(); 279 } 280 281 DBG("emit 0x%08x\n", seq); 282 return seq; 283 } 284 285 static void 286 _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq) 287 { 288 struct drm_i915_irq_wait iw; 289 int hw_seq, busy_count = 0; 290 int ret; 291 int kernel_lied; 292 293 if (bufmgr_fake->fence_wait != NULL) { 294 bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv); 295 clear_fenced(bufmgr_fake, seq); 296 return; 297 } 298 299 iw.irq_seq = seq; 300 301 DBG("wait 0x%08x\n", iw.irq_seq); 302 303 /* The kernel IRQ_WAIT implementation is all sorts of broken. 304 * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit 305 * unsigned range. 306 * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit 307 * signed range. 308 * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit 309 * signed range. 310 * 4) It returns -EBUSY in 3 seconds even if the hardware is still 311 * successfully chewing through buffers. 312 * 313 * Assume that in userland we treat sequence numbers as ints, which 314 * makes some of the comparisons convenient, since the sequence 315 * numbers are all positive signed integers. 316 * 317 * From this we get several cases we need to handle. Here's a timeline. 318 * 0x2 0x7 0x7ffffff8 0x7ffffffd 319 * | | | | 320 * ------------------------------------------------------------ 321 * 322 * A) Normal wait for hw to catch up 323 * hw_seq seq 324 * | | 325 * ------------------------------------------------------------ 326 * seq - hw_seq = 5. If we call IRQ_WAIT, it will wait for hw to 327 * catch up. 328 * 329 * B) Normal wait for a sequence number that's already passed. 330 * seq hw_seq 331 * | | 332 * ------------------------------------------------------------ 333 * seq - hw_seq = -5. If we call IRQ_WAIT, it returns 0 quickly. 334 * 335 * C) Hardware has already wrapped around ahead of us 336 * hw_seq seq 337 * | | 338 * ------------------------------------------------------------ 339 * seq - hw_seq = 0x80000000 - 5. If we called IRQ_WAIT, it would wait 340 * for hw_seq >= seq, which may never occur. Thus, we want to catch 341 * this in userland and return 0. 342 * 343 * D) We've wrapped around ahead of the hardware. 344 * seq hw_seq 345 * | | 346 * ------------------------------------------------------------ 347 * seq - hw_seq = -(0x80000000 - 5). If we called IRQ_WAIT, it would 348 * return 0 quickly because hw_seq >= seq, even though the hardware 349 * isn't caught up. Thus, we need to catch this early return in 350 * userland and bother the kernel until the hardware really does 351 * catch up. 352 * 353 * E) Hardware might wrap after we test in userland. 354 * hw_seq seq 355 * | | 356 * ------------------------------------------------------------ 357 * seq - hw_seq = 5. If we call IRQ_WAIT, it will likely see seq >= 358 * hw_seq and wait. However, suppose hw_seq wraps before we make it 359 * into the kernel. The kernel sees hw_seq >= seq and waits for 3 360 * seconds then returns -EBUSY. This is case C). We should catch 361 * this and then return successfully. 362 * 363 * F) Hardware might take a long time on a buffer. 364 * hw_seq seq 365 * | | 366 * ------------------------------------------------------------------- 367 * seq - hw_seq = 5. If we call IRQ_WAIT, if sequence 2 through 5 368 * take too long, it will return -EBUSY. Batchbuffers in the 369 * gltestperf demo were seen to take up to 7 seconds. We should 370 * catch early -EBUSY return and keep trying. 371 */ 372 373 do { 374 /* Keep a copy of last_dispatch so that if the wait -EBUSYs 375 * because the hardware didn't catch up in 3 seconds, we can 376 * see if it at least made progress and retry. 377 */ 378 hw_seq = *bufmgr_fake->last_dispatch; 379 380 /* Catch case C */ 381 if (seq - hw_seq > 0x40000000) 382 return; 383 384 ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT, 385 &iw, sizeof(iw)); 386 /* Catch case D */ 387 kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch < 388 -0x40000000); 389 390 /* Catch case E */ 391 if (ret == -EBUSY 392 && (seq - *bufmgr_fake->last_dispatch > 0x40000000)) 393 ret = 0; 394 395 /* Catch case F: Allow up to 15 seconds chewing on one buffer. */ 396 if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch)) 397 busy_count = 0; 398 else 399 busy_count++; 400 } while (kernel_lied || ret == -EAGAIN || ret == -EINTR || 401 (ret == -EBUSY && busy_count < 5)); 402 403 if (ret != 0) { 404 drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__, 405 __LINE__, strerror(-ret)); 406 abort(); 407 } 408 clear_fenced(bufmgr_fake, seq); 409 } 410 411 static int 412 _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence) 413 { 414 /* Slight problem with wrap-around: 415 */ 416 return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence); 417 } 418 419 /** 420 * Allocate a memory manager block for the buffer. 421 */ 422 static int 423 alloc_block(drm_intel_bo *bo) 424 { 425 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 426 drm_intel_bufmgr_fake *bufmgr_fake = 427 (drm_intel_bufmgr_fake *) bo->bufmgr; 428 struct block *block = (struct block *)calloc(sizeof *block, 1); 429 unsigned int align_log2 = ffs(bo_fake->alignment) - 1; 430 unsigned int sz; 431 432 if (!block) 433 return 1; 434 435 sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); 436 437 block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0); 438 if (!block->mem) { 439 free(block); 440 return 0; 441 } 442 443 DRMINITLISTHEAD(block); 444 445 /* Insert at head or at tail??? */ 446 DRMLISTADDTAIL(block, &bufmgr_fake->lru); 447 448 block->virtual = (uint8_t *) bufmgr_fake->virtual + 449 block->mem->ofs - bufmgr_fake->low_offset; 450 block->bo = bo; 451 452 bo_fake->block = block; 453 454 return 1; 455 } 456 457 /* Release the card storage associated with buf: 458 */ 459 static void 460 free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block, 461 int skip_dirty_copy) 462 { 463 drm_intel_bo_fake *bo_fake; 464 DBG("free block %p %08x %d %d\n", block, block->mem->ofs, 465 block->on_hardware, block->fenced); 466 467 if (!block) 468 return; 469 470 bo_fake = (drm_intel_bo_fake *) block->bo; 471 472 if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)) 473 skip_dirty_copy = 1; 474 475 if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) { 476 memcpy(bo_fake->backing_store, block->virtual, block->bo->size); 477 bo_fake->card_dirty = 0; 478 bo_fake->dirty = 1; 479 } 480 481 if (block->on_hardware) { 482 block->bo = NULL; 483 } else if (block->fenced) { 484 block->bo = NULL; 485 } else { 486 DBG(" - free immediately\n"); 487 DRMLISTDEL(block); 488 489 mmFreeMem(block->mem); 490 free(block); 491 } 492 } 493 494 static void 495 alloc_backing_store(drm_intel_bo *bo) 496 { 497 drm_intel_bufmgr_fake *bufmgr_fake = 498 (drm_intel_bufmgr_fake *) bo->bufmgr; 499 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 500 assert(!bo_fake->backing_store); 501 assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))); 502 503 bo_fake->backing_store = malloc(bo->size); 504 505 DBG("alloc_backing - buf %d %p %lu\n", bo_fake->id, 506 bo_fake->backing_store, bo->size); 507 assert(bo_fake->backing_store); 508 } 509 510 static void 511 free_backing_store(drm_intel_bo *bo) 512 { 513 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 514 515 if (bo_fake->backing_store) { 516 assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))); 517 free(bo_fake->backing_store); 518 bo_fake->backing_store = NULL; 519 } 520 } 521 522 static void 523 set_dirty(drm_intel_bo *bo) 524 { 525 drm_intel_bufmgr_fake *bufmgr_fake = 526 (drm_intel_bufmgr_fake *) bo->bufmgr; 527 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 528 529 if (bo_fake->flags & BM_NO_BACKING_STORE 530 && bo_fake->invalidate_cb != NULL) 531 bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr); 532 533 assert(!(bo_fake->flags & BM_PINNED)); 534 535 DBG("set_dirty - buf %d\n", bo_fake->id); 536 bo_fake->dirty = 1; 537 } 538 539 static int 540 evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence) 541 { 542 struct block *block, *tmp; 543 544 DBG("%s\n", __func__); 545 546 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { 547 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 548 549 if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) 550 continue; 551 552 if (block->fence && max_fence && !FENCE_LTE(block->fence, 553 max_fence)) 554 return 0; 555 556 set_dirty(&bo_fake->bo); 557 bo_fake->block = NULL; 558 559 free_block(bufmgr_fake, block, 0); 560 return 1; 561 } 562 563 return 0; 564 } 565 566 static int 567 evict_mru(drm_intel_bufmgr_fake *bufmgr_fake) 568 { 569 struct block *block, *tmp; 570 571 DBG("%s\n", __func__); 572 573 DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) { 574 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 575 576 if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) 577 continue; 578 579 set_dirty(&bo_fake->bo); 580 bo_fake->block = NULL; 581 582 free_block(bufmgr_fake, block, 0); 583 return 1; 584 } 585 586 return 0; 587 } 588 589 /** 590 * Removes all objects from the fenced list older than the given fence. 591 */ 592 static int 593 clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie) 594 { 595 struct block *block, *tmp; 596 int ret = 0; 597 598 bufmgr_fake->last_fence = fence_cookie; 599 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) { 600 assert(block->fenced); 601 602 if (_fence_test(bufmgr_fake, block->fence)) { 603 604 block->fenced = 0; 605 606 if (!block->bo) { 607 DBG("delayed free: offset %x sz %x\n", 608 block->mem->ofs, block->mem->size); 609 DRMLISTDEL(block); 610 mmFreeMem(block->mem); 611 free(block); 612 } else { 613 DBG("return to lru: offset %x sz %x\n", 614 block->mem->ofs, block->mem->size); 615 DRMLISTDEL(block); 616 DRMLISTADDTAIL(block, &bufmgr_fake->lru); 617 } 618 619 ret = 1; 620 } else { 621 /* Blocks are ordered by fence, so if one fails, all 622 * from here will fail also: 623 */ 624 DBG("fence not passed: offset %x sz %x %d %d \n", 625 block->mem->ofs, block->mem->size, block->fence, 626 bufmgr_fake->last_fence); 627 break; 628 } 629 } 630 631 DBG("%s: %d\n", __func__, ret); 632 return ret; 633 } 634 635 static void 636 fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence) 637 { 638 struct block *block, *tmp; 639 640 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) { 641 DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n", 642 block, block->mem->size, block->mem->ofs, block->bo, fence); 643 block->fence = fence; 644 645 block->on_hardware = 0; 646 block->fenced = 1; 647 648 /* Move to tail of pending list here 649 */ 650 DRMLISTDEL(block); 651 DRMLISTADDTAIL(block, &bufmgr_fake->fenced); 652 } 653 654 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); 655 } 656 657 static int 658 evict_and_alloc_block(drm_intel_bo *bo) 659 { 660 drm_intel_bufmgr_fake *bufmgr_fake = 661 (drm_intel_bufmgr_fake *) bo->bufmgr; 662 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 663 664 assert(bo_fake->block == NULL); 665 666 /* Search for already free memory: 667 */ 668 if (alloc_block(bo)) 669 return 1; 670 671 /* If we're not thrashing, allow lru eviction to dig deeper into 672 * recently used textures. We'll probably be thrashing soon: 673 */ 674 if (!bufmgr_fake->thrashing) { 675 while (evict_lru(bufmgr_fake, 0)) 676 if (alloc_block(bo)) 677 return 1; 678 } 679 680 /* Keep thrashing counter alive? 681 */ 682 if (bufmgr_fake->thrashing) 683 bufmgr_fake->thrashing = 20; 684 685 /* Wait on any already pending fences - here we are waiting for any 686 * freed memory that has been submitted to hardware and fenced to 687 * become available: 688 */ 689 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) { 690 uint32_t fence = bufmgr_fake->fenced.next->fence; 691 _fence_wait_internal(bufmgr_fake, fence); 692 693 if (alloc_block(bo)) 694 return 1; 695 } 696 697 if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) { 698 while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) { 699 uint32_t fence = bufmgr_fake->fenced.next->fence; 700 _fence_wait_internal(bufmgr_fake, fence); 701 } 702 703 if (!bufmgr_fake->thrashing) { 704 DBG("thrashing\n"); 705 } 706 bufmgr_fake->thrashing = 20; 707 708 if (alloc_block(bo)) 709 return 1; 710 } 711 712 while (evict_mru(bufmgr_fake)) 713 if (alloc_block(bo)) 714 return 1; 715 716 DBG("%s 0x%lx bytes failed\n", __func__, bo->size); 717 718 return 0; 719 } 720 721 /*********************************************************************** 722 * Public functions 723 */ 724 725 /** 726 * Wait for hardware idle by emitting a fence and waiting for it. 727 */ 728 static void 729 drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake) 730 { 731 unsigned int cookie; 732 733 cookie = _fence_emit_internal(bufmgr_fake); 734 _fence_wait_internal(bufmgr_fake, cookie); 735 } 736 737 /** 738 * Wait for rendering to a buffer to complete. 739 * 740 * It is assumed that the batchbuffer which performed the rendering included 741 * the necessary flushing. 742 */ 743 static void 744 drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo) 745 { 746 drm_intel_bufmgr_fake *bufmgr_fake = 747 (drm_intel_bufmgr_fake *) bo->bufmgr; 748 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 749 750 if (bo_fake->block == NULL || !bo_fake->block->fenced) 751 return; 752 753 _fence_wait_internal(bufmgr_fake, bo_fake->block->fence); 754 } 755 756 static void 757 drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo) 758 { 759 drm_intel_bufmgr_fake *bufmgr_fake = 760 (drm_intel_bufmgr_fake *) bo->bufmgr; 761 762 pthread_mutex_lock(&bufmgr_fake->lock); 763 drm_intel_fake_bo_wait_rendering_locked(bo); 764 pthread_mutex_unlock(&bufmgr_fake->lock); 765 } 766 767 /* Specifically ignore texture memory sharing. 768 * -- just evict everything 769 * -- and wait for idle 770 */ 771 void 772 drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr) 773 { 774 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 775 struct block *block, *tmp; 776 777 pthread_mutex_lock(&bufmgr_fake->lock); 778 779 bufmgr_fake->need_fence = 1; 780 bufmgr_fake->fail = 0; 781 782 /* Wait for hardware idle. We don't know where acceleration has been 783 * happening, so we'll need to wait anyway before letting anything get 784 * put on the card again. 785 */ 786 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 787 788 /* Check that we hadn't released the lock without having fenced the last 789 * set of buffers. 790 */ 791 assert(DRMLISTEMPTY(&bufmgr_fake->fenced)); 792 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); 793 794 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { 795 assert(_fence_test(bufmgr_fake, block->fence)); 796 set_dirty(block->bo); 797 } 798 799 pthread_mutex_unlock(&bufmgr_fake->lock); 800 } 801 802 static drm_intel_bo * 803 drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr, 804 const char *name, 805 unsigned long size, 806 unsigned int alignment) 807 { 808 drm_intel_bufmgr_fake *bufmgr_fake; 809 drm_intel_bo_fake *bo_fake; 810 811 bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 812 813 assert(size != 0); 814 815 bo_fake = calloc(1, sizeof(*bo_fake)); 816 if (!bo_fake) 817 return NULL; 818 819 bo_fake->bo.size = size; 820 bo_fake->bo.offset = -1; 821 bo_fake->bo.virtual = NULL; 822 bo_fake->bo.bufmgr = bufmgr; 823 bo_fake->refcount = 1; 824 825 /* Alignment must be a power of two */ 826 assert((alignment & (alignment - 1)) == 0); 827 if (alignment == 0) 828 alignment = 1; 829 bo_fake->alignment = alignment; 830 bo_fake->id = ++bufmgr_fake->buf_nr; 831 bo_fake->name = name; 832 bo_fake->flags = 0; 833 bo_fake->is_static = 0; 834 835 DBG("drm_bo_alloc: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name, 836 bo_fake->bo.size / 1024); 837 838 return &bo_fake->bo; 839 } 840 841 static drm_intel_bo * 842 drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr, 843 const char *name, 844 int x, int y, int cpp, 845 uint32_t *tiling_mode, 846 unsigned long *pitch, 847 unsigned long flags) 848 { 849 unsigned long stride, aligned_y; 850 851 /* No runtime tiling support for fake. */ 852 *tiling_mode = I915_TILING_NONE; 853 854 /* Align it for being a render target. Shouldn't need anything else. */ 855 stride = x * cpp; 856 stride = ROUND_UP_TO(stride, 64); 857 858 /* 965 subspan loading alignment */ 859 aligned_y = ALIGN(y, 2); 860 861 *pitch = stride; 862 863 return drm_intel_fake_bo_alloc(bufmgr, name, stride * aligned_y, 864 4096); 865 } 866 867 drm_intel_bo * 868 drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr, 869 const char *name, 870 unsigned long offset, 871 unsigned long size, void *virtual) 872 { 873 drm_intel_bufmgr_fake *bufmgr_fake; 874 drm_intel_bo_fake *bo_fake; 875 876 bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 877 878 assert(size != 0); 879 880 bo_fake = calloc(1, sizeof(*bo_fake)); 881 if (!bo_fake) 882 return NULL; 883 884 bo_fake->bo.size = size; 885 bo_fake->bo.offset = offset; 886 bo_fake->bo.virtual = virtual; 887 bo_fake->bo.bufmgr = bufmgr; 888 bo_fake->refcount = 1; 889 bo_fake->id = ++bufmgr_fake->buf_nr; 890 bo_fake->name = name; 891 bo_fake->flags = BM_PINNED; 892 bo_fake->is_static = 1; 893 894 DBG("drm_bo_alloc_static: (buf %d: %s, %lu kb)\n", bo_fake->id, 895 bo_fake->name, bo_fake->bo.size / 1024); 896 897 return &bo_fake->bo; 898 } 899 900 static void 901 drm_intel_fake_bo_reference(drm_intel_bo *bo) 902 { 903 drm_intel_bufmgr_fake *bufmgr_fake = 904 (drm_intel_bufmgr_fake *) bo->bufmgr; 905 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 906 907 pthread_mutex_lock(&bufmgr_fake->lock); 908 bo_fake->refcount++; 909 pthread_mutex_unlock(&bufmgr_fake->lock); 910 } 911 912 static void 913 drm_intel_fake_bo_reference_locked(drm_intel_bo *bo) 914 { 915 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 916 917 bo_fake->refcount++; 918 } 919 920 static void 921 drm_intel_fake_bo_unreference_locked(drm_intel_bo *bo) 922 { 923 drm_intel_bufmgr_fake *bufmgr_fake = 924 (drm_intel_bufmgr_fake *) bo->bufmgr; 925 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 926 int i; 927 928 if (--bo_fake->refcount == 0) { 929 assert(bo_fake->map_count == 0); 930 /* No remaining references, so free it */ 931 if (bo_fake->block) 932 free_block(bufmgr_fake, bo_fake->block, 1); 933 free_backing_store(bo); 934 935 for (i = 0; i < bo_fake->nr_relocs; i++) 936 drm_intel_fake_bo_unreference_locked(bo_fake->relocs[i]. 937 target_buf); 938 939 DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id, 940 bo_fake->name); 941 942 free(bo_fake->relocs); 943 free(bo); 944 } 945 } 946 947 static void 948 drm_intel_fake_bo_unreference(drm_intel_bo *bo) 949 { 950 drm_intel_bufmgr_fake *bufmgr_fake = 951 (drm_intel_bufmgr_fake *) bo->bufmgr; 952 953 pthread_mutex_lock(&bufmgr_fake->lock); 954 drm_intel_fake_bo_unreference_locked(bo); 955 pthread_mutex_unlock(&bufmgr_fake->lock); 956 } 957 958 /** 959 * Set the buffer as not requiring backing store, and instead get the callback 960 * invoked whenever it would be set dirty. 961 */ 962 void 963 drm_intel_bo_fake_disable_backing_store(drm_intel_bo *bo, 964 void (*invalidate_cb) (drm_intel_bo *bo, 965 void *ptr), 966 void *ptr) 967 { 968 drm_intel_bufmgr_fake *bufmgr_fake = 969 (drm_intel_bufmgr_fake *) bo->bufmgr; 970 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 971 972 pthread_mutex_lock(&bufmgr_fake->lock); 973 974 if (bo_fake->backing_store) 975 free_backing_store(bo); 976 977 bo_fake->flags |= BM_NO_BACKING_STORE; 978 979 DBG("disable_backing_store set buf %d dirty\n", bo_fake->id); 980 bo_fake->dirty = 1; 981 bo_fake->invalidate_cb = invalidate_cb; 982 bo_fake->invalidate_ptr = ptr; 983 984 /* Note that it is invalid right from the start. Also note 985 * invalidate_cb is called with the bufmgr locked, so cannot 986 * itself make bufmgr calls. 987 */ 988 if (invalidate_cb != NULL) 989 invalidate_cb(bo, ptr); 990 991 pthread_mutex_unlock(&bufmgr_fake->lock); 992 } 993 994 /** 995 * Map a buffer into bo->virtual, allocating either card memory space (If 996 * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary. 997 */ 998 static int 999 drm_intel_fake_bo_map_locked(drm_intel_bo *bo, int write_enable) 1000 { 1001 drm_intel_bufmgr_fake *bufmgr_fake = 1002 (drm_intel_bufmgr_fake *) bo->bufmgr; 1003 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1004 1005 /* Static buffers are always mapped. */ 1006 if (bo_fake->is_static) { 1007 if (bo_fake->card_dirty) { 1008 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 1009 bo_fake->card_dirty = 0; 1010 } 1011 return 0; 1012 } 1013 1014 /* Allow recursive mapping. Mesa may recursively map buffers with 1015 * nested display loops, and it is used internally in bufmgr_fake 1016 * for relocation. 1017 */ 1018 if (bo_fake->map_count++ != 0) 1019 return 0; 1020 1021 { 1022 DBG("drm_bo_map: (buf %d: %s, %lu kb)\n", bo_fake->id, 1023 bo_fake->name, bo_fake->bo.size / 1024); 1024 1025 if (bo->virtual != NULL) { 1026 drmMsg("%s: already mapped\n", __func__); 1027 abort(); 1028 } else if (bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)) { 1029 1030 if (!bo_fake->block && !evict_and_alloc_block(bo)) { 1031 DBG("%s: alloc failed\n", __func__); 1032 bufmgr_fake->fail = 1; 1033 return 1; 1034 } else { 1035 assert(bo_fake->block); 1036 bo_fake->dirty = 0; 1037 1038 if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) && 1039 bo_fake->block->fenced) { 1040 drm_intel_fake_bo_wait_rendering_locked 1041 (bo); 1042 } 1043 1044 bo->virtual = bo_fake->block->virtual; 1045 } 1046 } else { 1047 if (write_enable) 1048 set_dirty(bo); 1049 1050 if (bo_fake->backing_store == 0) 1051 alloc_backing_store(bo); 1052 1053 if ((bo_fake->card_dirty == 1) && bo_fake->block) { 1054 if (bo_fake->block->fenced) 1055 drm_intel_fake_bo_wait_rendering_locked 1056 (bo); 1057 1058 memcpy(bo_fake->backing_store, 1059 bo_fake->block->virtual, 1060 bo_fake->block->bo->size); 1061 bo_fake->card_dirty = 0; 1062 } 1063 1064 bo->virtual = bo_fake->backing_store; 1065 } 1066 } 1067 1068 return 0; 1069 } 1070 1071 static int 1072 drm_intel_fake_bo_map(drm_intel_bo *bo, int write_enable) 1073 { 1074 drm_intel_bufmgr_fake *bufmgr_fake = 1075 (drm_intel_bufmgr_fake *) bo->bufmgr; 1076 int ret; 1077 1078 pthread_mutex_lock(&bufmgr_fake->lock); 1079 ret = drm_intel_fake_bo_map_locked(bo, write_enable); 1080 pthread_mutex_unlock(&bufmgr_fake->lock); 1081 1082 return ret; 1083 } 1084 1085 static int 1086 drm_intel_fake_bo_unmap_locked(drm_intel_bo *bo) 1087 { 1088 drm_intel_bufmgr_fake *bufmgr_fake = 1089 (drm_intel_bufmgr_fake *) bo->bufmgr; 1090 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1091 1092 /* Static buffers are always mapped. */ 1093 if (bo_fake->is_static) 1094 return 0; 1095 1096 assert(bo_fake->map_count != 0); 1097 if (--bo_fake->map_count != 0) 1098 return 0; 1099 1100 DBG("drm_bo_unmap: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name, 1101 bo_fake->bo.size / 1024); 1102 1103 bo->virtual = NULL; 1104 1105 return 0; 1106 } 1107 1108 static int drm_intel_fake_bo_unmap(drm_intel_bo *bo) 1109 { 1110 drm_intel_bufmgr_fake *bufmgr_fake = 1111 (drm_intel_bufmgr_fake *) bo->bufmgr; 1112 int ret; 1113 1114 pthread_mutex_lock(&bufmgr_fake->lock); 1115 ret = drm_intel_fake_bo_unmap_locked(bo); 1116 pthread_mutex_unlock(&bufmgr_fake->lock); 1117 1118 return ret; 1119 } 1120 1121 static int 1122 drm_intel_fake_bo_subdata(drm_intel_bo *bo, unsigned long offset, 1123 unsigned long size, const void *data) 1124 { 1125 int ret; 1126 1127 if (size == 0 || data == NULL) 1128 return 0; 1129 1130 ret = drm_intel_bo_map(bo, 1); 1131 if (ret) 1132 return ret; 1133 memcpy((unsigned char *)bo->virtual + offset, data, size); 1134 drm_intel_bo_unmap(bo); 1135 return 0; 1136 } 1137 1138 static void 1139 drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake *bufmgr_fake) 1140 { 1141 struct block *block, *tmp; 1142 1143 bufmgr_fake->performed_rendering = 0; 1144 /* okay for ever BO that is on the HW kick it off. 1145 seriously not afraid of the POLICE right now */ 1146 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) { 1147 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 1148 1149 block->on_hardware = 0; 1150 free_block(bufmgr_fake, block, 0); 1151 bo_fake->block = NULL; 1152 bo_fake->validated = 0; 1153 if (!(bo_fake->flags & BM_NO_BACKING_STORE)) 1154 bo_fake->dirty = 1; 1155 } 1156 1157 } 1158 1159 static int 1160 drm_intel_fake_bo_validate(drm_intel_bo *bo) 1161 { 1162 drm_intel_bufmgr_fake *bufmgr_fake; 1163 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1164 1165 bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; 1166 1167 DBG("drm_bo_validate: (buf %d: %s, %lu kb)\n", bo_fake->id, 1168 bo_fake->name, bo_fake->bo.size / 1024); 1169 1170 /* Sanity check: Buffers should be unmapped before being validated. 1171 * This is not so much of a problem for bufmgr_fake, but TTM refuses, 1172 * and the problem is harder to debug there. 1173 */ 1174 assert(bo_fake->map_count == 0); 1175 1176 if (bo_fake->is_static) { 1177 /* Add it to the needs-fence list */ 1178 bufmgr_fake->need_fence = 1; 1179 return 0; 1180 } 1181 1182 /* Allocate the card memory */ 1183 if (!bo_fake->block && !evict_and_alloc_block(bo)) { 1184 bufmgr_fake->fail = 1; 1185 DBG("Failed to validate buf %d:%s\n", bo_fake->id, 1186 bo_fake->name); 1187 return -1; 1188 } 1189 1190 assert(bo_fake->block); 1191 assert(bo_fake->block->bo == &bo_fake->bo); 1192 1193 bo->offset = bo_fake->block->mem->ofs; 1194 1195 /* Upload the buffer contents if necessary */ 1196 if (bo_fake->dirty) { 1197 DBG("Upload dirty buf %d:%s, sz %lu offset 0x%x\n", bo_fake->id, 1198 bo_fake->name, bo->size, bo_fake->block->mem->ofs); 1199 1200 assert(!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))); 1201 1202 /* Actually, should be able to just wait for a fence on the 1203 * memory, which we would be tracking when we free it. Waiting 1204 * for idle is a sufficiently large hammer for now. 1205 */ 1206 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 1207 1208 /* we may never have mapped this BO so it might not have any 1209 * backing store if this happens it should be rare, but 0 the 1210 * card memory in any case */ 1211 if (bo_fake->backing_store) 1212 memcpy(bo_fake->block->virtual, bo_fake->backing_store, 1213 bo->size); 1214 else 1215 memset(bo_fake->block->virtual, 0, bo->size); 1216 1217 bo_fake->dirty = 0; 1218 } 1219 1220 bo_fake->block->fenced = 0; 1221 bo_fake->block->on_hardware = 1; 1222 DRMLISTDEL(bo_fake->block); 1223 DRMLISTADDTAIL(bo_fake->block, &bufmgr_fake->on_hardware); 1224 1225 bo_fake->validated = 1; 1226 bufmgr_fake->need_fence = 1; 1227 1228 return 0; 1229 } 1230 1231 static void 1232 drm_intel_fake_fence_validated(drm_intel_bufmgr *bufmgr) 1233 { 1234 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1235 unsigned int cookie; 1236 1237 cookie = _fence_emit_internal(bufmgr_fake); 1238 fence_blocks(bufmgr_fake, cookie); 1239 1240 DBG("drm_fence_validated: 0x%08x cookie\n", cookie); 1241 } 1242 1243 static void 1244 drm_intel_fake_destroy(drm_intel_bufmgr *bufmgr) 1245 { 1246 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1247 1248 pthread_mutex_destroy(&bufmgr_fake->lock); 1249 mmDestroy(bufmgr_fake->heap); 1250 free(bufmgr); 1251 } 1252 1253 static int 1254 drm_intel_fake_emit_reloc(drm_intel_bo *bo, uint32_t offset, 1255 drm_intel_bo *target_bo, uint32_t target_offset, 1256 uint32_t read_domains, uint32_t write_domain) 1257 { 1258 drm_intel_bufmgr_fake *bufmgr_fake = 1259 (drm_intel_bufmgr_fake *) bo->bufmgr; 1260 struct fake_buffer_reloc *r; 1261 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1262 drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *) target_bo; 1263 int i; 1264 1265 pthread_mutex_lock(&bufmgr_fake->lock); 1266 1267 assert(bo); 1268 assert(target_bo); 1269 1270 if (bo_fake->relocs == NULL) { 1271 bo_fake->relocs = 1272 malloc(sizeof(struct fake_buffer_reloc) * MAX_RELOCS); 1273 } 1274 1275 r = &bo_fake->relocs[bo_fake->nr_relocs++]; 1276 1277 assert(bo_fake->nr_relocs <= MAX_RELOCS); 1278 1279 drm_intel_fake_bo_reference_locked(target_bo); 1280 1281 if (!target_fake->is_static) { 1282 bo_fake->child_size += 1283 ALIGN(target_bo->size, target_fake->alignment); 1284 bo_fake->child_size += target_fake->child_size; 1285 } 1286 r->target_buf = target_bo; 1287 r->offset = offset; 1288 r->last_target_offset = target_bo->offset; 1289 r->delta = target_offset; 1290 r->read_domains = read_domains; 1291 r->write_domain = write_domain; 1292 1293 if (bufmgr_fake->debug) { 1294 /* Check that a conflicting relocation hasn't already been 1295 * emitted. 1296 */ 1297 for (i = 0; i < bo_fake->nr_relocs - 1; i++) { 1298 struct fake_buffer_reloc *r2 = &bo_fake->relocs[i]; 1299 1300 assert(r->offset != r2->offset); 1301 } 1302 } 1303 1304 pthread_mutex_unlock(&bufmgr_fake->lock); 1305 1306 return 0; 1307 } 1308 1309 /** 1310 * Incorporates the validation flags associated with each relocation into 1311 * the combined validation flags for the buffer on this batchbuffer submission. 1312 */ 1313 static void 1314 drm_intel_fake_calculate_domains(drm_intel_bo *bo) 1315 { 1316 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1317 int i; 1318 1319 for (i = 0; i < bo_fake->nr_relocs; i++) { 1320 struct fake_buffer_reloc *r = &bo_fake->relocs[i]; 1321 drm_intel_bo_fake *target_fake = 1322 (drm_intel_bo_fake *) r->target_buf; 1323 1324 /* Do the same for the tree of buffers we depend on */ 1325 drm_intel_fake_calculate_domains(r->target_buf); 1326 1327 target_fake->read_domains |= r->read_domains; 1328 target_fake->write_domain |= r->write_domain; 1329 } 1330 } 1331 1332 static int 1333 drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo *bo) 1334 { 1335 drm_intel_bufmgr_fake *bufmgr_fake = 1336 (drm_intel_bufmgr_fake *) bo->bufmgr; 1337 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1338 int i, ret; 1339 1340 assert(bo_fake->map_count == 0); 1341 1342 for (i = 0; i < bo_fake->nr_relocs; i++) { 1343 struct fake_buffer_reloc *r = &bo_fake->relocs[i]; 1344 drm_intel_bo_fake *target_fake = 1345 (drm_intel_bo_fake *) r->target_buf; 1346 uint32_t reloc_data; 1347 1348 /* Validate the target buffer if that hasn't been done. */ 1349 if (!target_fake->validated) { 1350 ret = 1351 drm_intel_fake_reloc_and_validate_buffer(r->target_buf); 1352 if (ret != 0) { 1353 if (bo->virtual != NULL) 1354 drm_intel_fake_bo_unmap_locked(bo); 1355 return ret; 1356 } 1357 } 1358 1359 /* Calculate the value of the relocation entry. */ 1360 if (r->target_buf->offset != r->last_target_offset) { 1361 reloc_data = r->target_buf->offset + r->delta; 1362 1363 if (bo->virtual == NULL) 1364 drm_intel_fake_bo_map_locked(bo, 1); 1365 1366 *(uint32_t *) ((uint8_t *) bo->virtual + r->offset) = 1367 reloc_data; 1368 1369 r->last_target_offset = r->target_buf->offset; 1370 } 1371 } 1372 1373 if (bo->virtual != NULL) 1374 drm_intel_fake_bo_unmap_locked(bo); 1375 1376 if (bo_fake->write_domain != 0) { 1377 if (!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))) { 1378 if (bo_fake->backing_store == 0) 1379 alloc_backing_store(bo); 1380 } 1381 bo_fake->card_dirty = 1; 1382 bufmgr_fake->performed_rendering = 1; 1383 } 1384 1385 return drm_intel_fake_bo_validate(bo); 1386 } 1387 1388 static void 1389 drm_intel_bo_fake_post_submit(drm_intel_bo *bo) 1390 { 1391 drm_intel_bufmgr_fake *bufmgr_fake = 1392 (drm_intel_bufmgr_fake *) bo->bufmgr; 1393 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; 1394 int i; 1395 1396 for (i = 0; i < bo_fake->nr_relocs; i++) { 1397 struct fake_buffer_reloc *r = &bo_fake->relocs[i]; 1398 drm_intel_bo_fake *target_fake = 1399 (drm_intel_bo_fake *) r->target_buf; 1400 1401 if (target_fake->validated) 1402 drm_intel_bo_fake_post_submit(r->target_buf); 1403 1404 DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n", 1405 bo_fake->name, (uint32_t) bo->offset, r->offset, 1406 target_fake->name, (uint32_t) r->target_buf->offset, 1407 r->delta); 1408 } 1409 1410 assert(bo_fake->map_count == 0); 1411 bo_fake->validated = 0; 1412 bo_fake->read_domains = 0; 1413 bo_fake->write_domain = 0; 1414 } 1415 1416 void 1417 drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr *bufmgr, 1418 int (*exec) (drm_intel_bo *bo, 1419 unsigned int used, 1420 void *priv), 1421 void *priv) 1422 { 1423 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1424 1425 bufmgr_fake->exec = exec; 1426 bufmgr_fake->exec_priv = priv; 1427 } 1428 1429 static int 1430 drm_intel_fake_bo_exec(drm_intel_bo *bo, int used, 1431 drm_clip_rect_t * cliprects, int num_cliprects, int DR4) 1432 { 1433 drm_intel_bufmgr_fake *bufmgr_fake = 1434 (drm_intel_bufmgr_fake *) bo->bufmgr; 1435 drm_intel_bo_fake *batch_fake = (drm_intel_bo_fake *) bo; 1436 struct drm_i915_batchbuffer batch; 1437 int ret; 1438 int retry_count = 0; 1439 1440 pthread_mutex_lock(&bufmgr_fake->lock); 1441 1442 bufmgr_fake->performed_rendering = 0; 1443 1444 drm_intel_fake_calculate_domains(bo); 1445 1446 batch_fake->read_domains = I915_GEM_DOMAIN_COMMAND; 1447 1448 /* we've ran out of RAM so blow the whole lot away and retry */ 1449 restart: 1450 ret = drm_intel_fake_reloc_and_validate_buffer(bo); 1451 if (bufmgr_fake->fail == 1) { 1452 if (retry_count == 0) { 1453 retry_count++; 1454 drm_intel_fake_kick_all_locked(bufmgr_fake); 1455 bufmgr_fake->fail = 0; 1456 goto restart; 1457 } else /* dump out the memory here */ 1458 mmDumpMemInfo(bufmgr_fake->heap); 1459 } 1460 1461 assert(ret == 0); 1462 1463 if (bufmgr_fake->exec != NULL) { 1464 ret = bufmgr_fake->exec(bo, used, bufmgr_fake->exec_priv); 1465 if (ret != 0) { 1466 pthread_mutex_unlock(&bufmgr_fake->lock); 1467 return ret; 1468 } 1469 } else { 1470 batch.start = bo->offset; 1471 batch.used = used; 1472 batch.cliprects = cliprects; 1473 batch.num_cliprects = num_cliprects; 1474 batch.DR1 = 0; 1475 batch.DR4 = DR4; 1476 1477 if (drmCommandWrite 1478 (bufmgr_fake->fd, DRM_I915_BATCHBUFFER, &batch, 1479 sizeof(batch))) { 1480 drmMsg("DRM_I915_BATCHBUFFER: %d\n", -errno); 1481 pthread_mutex_unlock(&bufmgr_fake->lock); 1482 return -errno; 1483 } 1484 } 1485 1486 drm_intel_fake_fence_validated(bo->bufmgr); 1487 1488 drm_intel_bo_fake_post_submit(bo); 1489 1490 pthread_mutex_unlock(&bufmgr_fake->lock); 1491 1492 return 0; 1493 } 1494 1495 /** 1496 * Return an error if the list of BOs will exceed the aperture size. 1497 * 1498 * This is a rough guess and likely to fail, as during the validate sequence we 1499 * may place a buffer in an inopportune spot early on and then fail to fit 1500 * a set smaller than the aperture. 1501 */ 1502 static int 1503 drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array, int count) 1504 { 1505 drm_intel_bufmgr_fake *bufmgr_fake = 1506 (drm_intel_bufmgr_fake *) bo_array[0]->bufmgr; 1507 unsigned int sz = 0; 1508 int i; 1509 1510 for (i = 0; i < count; i++) { 1511 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo_array[i]; 1512 1513 if (bo_fake == NULL) 1514 continue; 1515 1516 if (!bo_fake->is_static) 1517 sz += ALIGN(bo_array[i]->size, bo_fake->alignment); 1518 sz += bo_fake->child_size; 1519 } 1520 1521 if (sz > bufmgr_fake->size) { 1522 DBG("check_space: overflowed bufmgr size, %ukb vs %lukb\n", 1523 sz / 1024, bufmgr_fake->size / 1024); 1524 return -1; 1525 } 1526 1527 DBG("drm_check_space: sz %ukb vs bufgr %lukb\n", sz / 1024, 1528 bufmgr_fake->size / 1024); 1529 return 0; 1530 } 1531 1532 /** 1533 * Evicts all buffers, waiting for fences to pass and copying contents out 1534 * as necessary. 1535 * 1536 * Used by the X Server on LeaveVT, when the card memory is no longer our 1537 * own. 1538 */ 1539 void 1540 drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr *bufmgr) 1541 { 1542 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1543 struct block *block, *tmp; 1544 1545 pthread_mutex_lock(&bufmgr_fake->lock); 1546 1547 bufmgr_fake->need_fence = 1; 1548 bufmgr_fake->fail = 0; 1549 1550 /* Wait for hardware idle. We don't know where acceleration has been 1551 * happening, so we'll need to wait anyway before letting anything get 1552 * put on the card again. 1553 */ 1554 drm_intel_bufmgr_fake_wait_idle(bufmgr_fake); 1555 1556 /* Check that we hadn't released the lock without having fenced the last 1557 * set of buffers. 1558 */ 1559 assert(DRMLISTEMPTY(&bufmgr_fake->fenced)); 1560 assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); 1561 1562 DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { 1563 drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; 1564 /* Releases the memory, and memcpys dirty contents out if 1565 * necessary. 1566 */ 1567 free_block(bufmgr_fake, block, 0); 1568 bo_fake->block = NULL; 1569 } 1570 1571 pthread_mutex_unlock(&bufmgr_fake->lock); 1572 } 1573 1574 void 1575 drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr *bufmgr, 1576 volatile unsigned int 1577 *last_dispatch) 1578 { 1579 drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; 1580 1581 bufmgr_fake->last_dispatch = (volatile int *)last_dispatch; 1582 } 1583 1584 drm_intel_bufmgr * 1585 drm_intel_bufmgr_fake_init(int fd, unsigned long low_offset, 1586 void *low_virtual, unsigned long size, 1587 volatile unsigned int *last_dispatch) 1588 { 1589 drm_intel_bufmgr_fake *bufmgr_fake; 1590 1591 bufmgr_fake = calloc(1, sizeof(*bufmgr_fake)); 1592 1593 if (pthread_mutex_init(&bufmgr_fake->lock, NULL) != 0) { 1594 free(bufmgr_fake); 1595 return NULL; 1596 } 1597 1598 /* Initialize allocator */ 1599 DRMINITLISTHEAD(&bufmgr_fake->fenced); 1600 DRMINITLISTHEAD(&bufmgr_fake->on_hardware); 1601 DRMINITLISTHEAD(&bufmgr_fake->lru); 1602 1603 bufmgr_fake->low_offset = low_offset; 1604 bufmgr_fake->virtual = low_virtual; 1605 bufmgr_fake->size = size; 1606 bufmgr_fake->heap = mmInit(low_offset, size); 1607 1608 /* Hook in methods */ 1609 bufmgr_fake->bufmgr.bo_alloc = drm_intel_fake_bo_alloc; 1610 bufmgr_fake->bufmgr.bo_alloc_for_render = drm_intel_fake_bo_alloc; 1611 bufmgr_fake->bufmgr.bo_alloc_tiled = drm_intel_fake_bo_alloc_tiled; 1612 bufmgr_fake->bufmgr.bo_reference = drm_intel_fake_bo_reference; 1613 bufmgr_fake->bufmgr.bo_unreference = drm_intel_fake_bo_unreference; 1614 bufmgr_fake->bufmgr.bo_map = drm_intel_fake_bo_map; 1615 bufmgr_fake->bufmgr.bo_unmap = drm_intel_fake_bo_unmap; 1616 bufmgr_fake->bufmgr.bo_subdata = drm_intel_fake_bo_subdata; 1617 bufmgr_fake->bufmgr.bo_wait_rendering = 1618 drm_intel_fake_bo_wait_rendering; 1619 bufmgr_fake->bufmgr.bo_emit_reloc = drm_intel_fake_emit_reloc; 1620 bufmgr_fake->bufmgr.destroy = drm_intel_fake_destroy; 1621 bufmgr_fake->bufmgr.bo_exec = drm_intel_fake_bo_exec; 1622 bufmgr_fake->bufmgr.check_aperture_space = 1623 drm_intel_fake_check_aperture_space; 1624 bufmgr_fake->bufmgr.debug = 0; 1625 1626 bufmgr_fake->fd = fd; 1627 bufmgr_fake->last_dispatch = (volatile int *)last_dispatch; 1628 1629 return &bufmgr_fake->bufmgr; 1630 } 1631
