1 /************************************************************************** 2 * 3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA 4 * All Rights Reserved. 5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 26 * USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 **************************************************************************/ 29 /* 30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 31 */ 32 33 #ifdef HAVE_CONFIG_H 34 #include "config.h" 35 #endif 36 37 #include <drm/psb_ttm_placement_user.h> 38 #include <stdint.h> 39 #include <sys/time.h> 40 #include <errno.h> 41 #include <unistd.h> 42 #include <assert.h> 43 #include <string.h> 44 #include <sys/mman.h> 45 #include <xf86drm.h> 46 #include "wsbm_pool.h" 47 #include "wsbm_fencemgr.h" 48 #include "wsbm_priv.h" 49 #include "wsbm_manager.h" 50 51 #define WSBM_SLABPOOL_ALLOC_RETRIES 100 52 #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ 53 do { \ 54 (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ 55 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ 56 57 #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ 58 do { \ 59 (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ 60 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ 61 62 63 #ifdef DEBUG_FENCESIGNALED 64 static int createbuffer = 0; 65 static int fencesignaled = 0; 66 #endif 67 68 struct _WsbmSlab; 69 70 struct _WsbmSlabBuffer 71 { 72 struct _WsbmKernelBuf kBuf; 73 struct _WsbmBufStorage storage; 74 struct _WsbmCond event; 75 76 /* 77 * Remains constant after creation. 78 */ 79 80 int isSlabBuffer; 81 struct _WsbmSlab *parent; 82 uint32_t start; 83 void *virtual; 84 unsigned long requestedSize; 85 uint64_t mapHandle; 86 87 /* 88 * Protected by struct _WsbmSlabSizeHeader::mutex 89 */ 90 91 struct _WsbmListHead head; 92 93 /* 94 * Protected by this::mutex 95 */ 96 97 struct _WsbmFenceObject *fence; 98 uint32_t fenceType; 99 struct _WsbmAtomic writers; /* (Only upping) */ 100 int unFenced; 101 }; 102 103 struct _WsbmSlabPool; 104 struct _WsbmSlabKernelBO 105 { 106 107 /* 108 * Constant at creation 109 */ 110 111 struct _WsbmKernelBuf kBuf; 112 uint32_t pageAlignment; 113 void *virtual; 114 unsigned long actualSize; 115 uint64_t mapHandle; 116 117 /* 118 * Protected by struct _WsbmSlabCache::mutex 119 */ 120 121 struct _WsbmSlabPool *slabPool; 122 uint32_t proposedPlacement; 123 struct _WsbmListHead timeoutHead; 124 struct _WsbmListHead head; 125 struct timeval timeFreed; 126 }; 127 128 struct _WsbmSlab 129 { 130 struct _WsbmListHead head; 131 struct _WsbmListHead freeBuffers; 132 uint32_t numBuffers; 133 uint32_t numFree; 134 struct _WsbmSlabBuffer *buffers; 135 struct _WsbmSlabSizeHeader *header; 136 struct _WsbmSlabKernelBO *kbo; 137 }; 138 139 struct _WsbmSlabSizeHeader 140 { 141 /* 142 * Constant at creation. 143 */ 144 struct _WsbmSlabPool *slabPool; 145 uint32_t bufSize; 146 147 /* 148 * Protected by this::mutex 149 */ 150 151 struct _WsbmListHead slabs; 152 struct _WsbmListHead freeSlabs; 153 struct _WsbmListHead delayedBuffers; 154 uint32_t numDelayed; 155 struct _WsbmMutex mutex; 156 }; 157 158 struct _WsbmSlabCache 159 { 160 struct timeval slabTimeout; 161 struct timeval checkInterval; 162 struct timeval nextCheck; 163 struct _WsbmListHead timeoutList; 164 struct _WsbmListHead unCached; 165 struct _WsbmListHead cached; 166 struct _WsbmMutex mutex; 167 }; 168 169 struct _WsbmSlabPool 170 { 171 struct _WsbmBufferPool pool; 172 173 /* 174 * The data of this structure remains constant after 175 * initialization and thus needs no mutex protection. 176 */ 177 178 unsigned int devOffset; 179 struct _WsbmSlabCache *cache; 180 uint32_t proposedPlacement; 181 uint32_t validMask; 182 uint32_t *bucketSizes; 183 uint32_t numBuckets; 184 uint32_t pageSize; 185 int pageAlignment; 186 int maxSlabSize; 187 int desiredNumBuffers; 188 struct _WsbmSlabSizeHeader *headers; 189 }; 190 191 static inline struct _WsbmSlabPool * 192 slabPoolFromPool(struct _WsbmBufferPool *pool) 193 { 194 return containerOf(pool, struct _WsbmSlabPool, pool); 195 } 196 197 static inline struct _WsbmSlabPool * 198 slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf) 199 { 200 return slabPoolFromPool(sBuf->storage.pool); 201 } 202 203 static inline struct _WsbmSlabBuffer * 204 slabBuffer(struct _WsbmBufStorage *buf) 205 { 206 return containerOf(buf, struct _WsbmSlabBuffer, storage); 207 } 208 209 /* 210 * FIXME: Perhaps arrange timeout slabs in size buckets for fast 211 * retreival?? 212 */ 213 214 static inline int 215 wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2) 216 { 217 return ((arg1->tv_sec > arg2->tv_sec) || 218 ((arg1->tv_sec == arg2->tv_sec) && 219 (arg1->tv_usec > arg2->tv_usec))); 220 } 221 222 static inline void 223 wsbmTimeAdd(struct timeval *arg, struct timeval *add) 224 { 225 unsigned int sec; 226 227 arg->tv_sec += add->tv_sec; 228 arg->tv_usec += add->tv_usec; 229 sec = arg->tv_usec / 1000000; 230 arg->tv_sec += sec; 231 arg->tv_usec -= sec * 1000000; 232 } 233 234 static void 235 wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo) 236 { 237 struct ttm_pl_reference_req arg; 238 struct _WsbmSlabPool *slabPool; 239 240 if (!kbo) 241 return; 242 243 slabPool = kbo->slabPool; 244 arg.handle = kbo->kBuf.handle; 245 (void)munmap(kbo->virtual, kbo->actualSize); 246 (void)drmCommandWrite(slabPool->pool.fd, 247 slabPool->devOffset + TTM_PL_UNREF, &arg, 248 sizeof(arg)); 249 free(kbo); 250 } 251 252 static void 253 wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time) 254 { 255 struct _WsbmListHead *list, *next; 256 struct _WsbmSlabKernelBO *kbo; 257 258 if (!wsbmTimeAfterEq(time, &cache->nextCheck)) 259 return; 260 261 WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) { 262 kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead); 263 264 if (!wsbmTimeAfterEq(time, &kbo->timeFreed)) 265 break; 266 267 WSBMLISTDELINIT(&kbo->timeoutHead); 268 WSBMLISTDELINIT(&kbo->head); 269 wsbmFreeKernelBO(kbo); 270 } 271 272 cache->nextCheck = *time; 273 wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval); 274 } 275 276 /* 277 * Add a _SlabKernelBO to the free slab manager. 278 * This means that it is available for reuse, but if it's not 279 * reused in a while, it will be freed. 280 */ 281 282 static void 283 wsbmSetKernelBOFree(struct _WsbmSlabCache *cache, 284 struct _WsbmSlabKernelBO *kbo) 285 { 286 struct timeval time; 287 struct timeval timeFreed; 288 289 gettimeofday(&time, NULL); 290 timeFreed = time; 291 WSBM_MUTEX_LOCK(&cache->mutex); 292 wsbmTimeAdd(&timeFreed, &cache->slabTimeout); 293 kbo->timeFreed = timeFreed; 294 295 if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED) 296 WSBMLISTADD(&kbo->head, &cache->cached); 297 else 298 WSBMLISTADD(&kbo->head, &cache->unCached); 299 300 WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList); 301 wsbmFreeTimeoutKBOsLocked(cache, &time); 302 303 WSBM_MUTEX_UNLOCK(&cache->mutex); 304 } 305 306 /* 307 * Get a _SlabKernelBO for us to use as storage for a slab. 308 */ 309 310 static struct _WsbmSlabKernelBO * 311 wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header) 312 { 313 struct _WsbmSlabPool *slabPool = header->slabPool; 314 struct _WsbmSlabCache *cache = slabPool->cache; 315 struct _WsbmListHead *list, *head; 316 uint32_t size = header->bufSize * slabPool->desiredNumBuffers; 317 struct _WsbmSlabKernelBO *kbo; 318 struct _WsbmSlabKernelBO *kboTmp; 319 int ret; 320 321 /* 322 * FIXME: We should perhaps allow some variation in slabsize in order 323 * to efficiently reuse slabs. 324 */ 325 326 size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize; 327 if (size < header->bufSize) 328 size = header->bufSize; 329 size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1); 330 WSBM_MUTEX_LOCK(&cache->mutex); 331 332 kbo = NULL; 333 334 retry: 335 head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ? 336 &cache->cached : &cache->unCached; 337 338 WSBMLISTFOREACH(list, head) { 339 kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head); 340 341 if ((kboTmp->actualSize == size) && 342 (slabPool->pageAlignment == 0 || 343 (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) { 344 345 if (!kbo) 346 kbo = kboTmp; 347 348 if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0) 349 break; 350 351 } 352 } 353 354 if (kbo) { 355 WSBMLISTDELINIT(&kbo->head); 356 WSBMLISTDELINIT(&kbo->timeoutHead); 357 } 358 359 WSBM_MUTEX_UNLOCK(&cache->mutex); 360 361 if (kbo) { 362 uint32_t new_mask = 363 kbo->proposedPlacement ^ slabPool->proposedPlacement; 364 365 ret = 0; 366 if (new_mask) { 367 union ttm_pl_setstatus_arg arg; 368 struct ttm_pl_setstatus_req *req = &arg.req; 369 struct ttm_pl_rep *rep = &arg.rep; 370 371 req->handle = kbo->kBuf.handle; 372 req->set_placement = slabPool->proposedPlacement & new_mask; 373 req->clr_placement = ~slabPool->proposedPlacement & new_mask; 374 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, 375 slabPool->devOffset + TTM_PL_SETSTATUS, 376 arg, ret); 377 if (ret == 0) { 378 kbo->kBuf.gpuOffset = rep->gpu_offset; 379 kbo->kBuf.placement = rep->placement; 380 } 381 kbo->proposedPlacement = slabPool->proposedPlacement; 382 } 383 384 if (ret == 0) 385 return kbo; 386 387 wsbmFreeKernelBO(kbo); 388 kbo = NULL; 389 goto retry; 390 } 391 392 kbo = calloc(1, sizeof(*kbo)); 393 if (!kbo) 394 return NULL; 395 396 { 397 union ttm_pl_create_arg arg; 398 399 kbo->slabPool = slabPool; 400 WSBMINITLISTHEAD(&kbo->head); 401 WSBMINITLISTHEAD(&kbo->timeoutHead); 402 403 arg.req.size = size; 404 arg.req.placement = slabPool->proposedPlacement; 405 arg.req.page_alignment = slabPool->pageAlignment; 406 407 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd, 408 slabPool->devOffset + TTM_PL_CREATE, 409 arg, ret); 410 if (ret) 411 goto out_err0; 412 413 kbo->kBuf.gpuOffset = arg.rep.gpu_offset; 414 kbo->kBuf.placement = arg.rep.placement; 415 kbo->kBuf.handle = arg.rep.handle; 416 417 kbo->actualSize = arg.rep.bo_size; 418 kbo->mapHandle = arg.rep.map_handle; 419 kbo->proposedPlacement = slabPool->proposedPlacement; 420 } 421 422 kbo->virtual = mmap(0, kbo->actualSize, 423 PROT_READ | PROT_WRITE, MAP_SHARED, 424 slabPool->pool.fd, kbo->mapHandle); 425 426 if (kbo->virtual == MAP_FAILED) { 427 ret = -errno; 428 goto out_err1; 429 } 430 431 return kbo; 432 433 out_err1: 434 { 435 struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle }; 436 437 (void)drmCommandWrite(slabPool->pool.fd, 438 slabPool->devOffset + TTM_PL_UNREF, 439 &arg, sizeof(arg)); 440 } 441 out_err0: 442 free(kbo); 443 return NULL; 444 } 445 446 static int 447 wsbmAllocSlab(struct _WsbmSlabSizeHeader *header) 448 { 449 struct _WsbmSlab *slab; 450 struct _WsbmSlabBuffer *sBuf; 451 uint32_t numBuffers; 452 uint32_t ret; 453 uint32_t i; 454 455 slab = calloc(1, sizeof(*slab)); 456 if (!slab) 457 return -ENOMEM; 458 459 slab->kbo = wsbmAllocKernelBO(header); 460 if (!slab->kbo) { 461 ret = -ENOMEM; 462 goto out_err0; 463 } 464 465 numBuffers = slab->kbo->actualSize / header->bufSize; 466 467 slab->buffers = calloc(numBuffers, sizeof(*slab->buffers)); 468 if (!slab->buffers) { 469 ret = -ENOMEM; 470 goto out_err1; 471 } 472 473 WSBMINITLISTHEAD(&slab->head); 474 WSBMINITLISTHEAD(&slab->freeBuffers); 475 slab->numBuffers = numBuffers; 476 slab->numFree = 0; 477 slab->header = header; 478 479 sBuf = slab->buffers; 480 for (i = 0; i < numBuffers; ++i) { 481 ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool); 482 if (ret) 483 goto out_err2; 484 sBuf->parent = slab; 485 sBuf->start = i * header->bufSize; 486 sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual + 487 sBuf->start); 488 wsbmAtomicSet(&sBuf->writers, 0); 489 sBuf->isSlabBuffer = 1; 490 WSBM_COND_INIT(&sBuf->event); 491 WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers); 492 slab->numFree++; 493 sBuf++; 494 } 495 496 WSBMLISTADDTAIL(&slab->head, &header->slabs); 497 498 return 0; 499 500 out_err2: 501 sBuf = slab->buffers; 502 for (i = 0; i < numBuffers; ++i) { 503 if (sBuf->parent == slab) { 504 WSBM_COND_FREE(&sBuf->event); 505 wsbmBufStorageTakedown(&sBuf->storage); 506 } 507 sBuf++; 508 } 509 free(slab->buffers); 510 out_err1: 511 wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo); 512 out_err0: 513 free(slab); 514 return ret; 515 } 516 517 /* 518 * Delete a buffer from the slab header delayed list and put 519 * it on the slab free list. 520 */ 521 522 static void 523 wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf) 524 { 525 struct _WsbmSlab *slab = buf->parent; 526 struct _WsbmSlabSizeHeader *header = slab->header; 527 struct _WsbmListHead *list = &buf->head; 528 529 WSBMLISTDEL(list); 530 WSBMLISTADDTAIL(list, &slab->freeBuffers); 531 slab->numFree++; 532 533 if (slab->head.next == &slab->head) 534 WSBMLISTADDTAIL(&slab->head, &header->slabs); 535 536 if (slab->numFree == slab->numBuffers) { 537 list = &slab->head; 538 WSBMLISTDEL(list); 539 WSBMLISTADDTAIL(list, &header->freeSlabs); 540 } 541 542 if (header->slabs.next == &header->slabs || 543 slab->numFree != slab->numBuffers) { 544 545 struct _WsbmListHead *next; 546 struct _WsbmSlabCache *cache = header->slabPool->cache; 547 548 WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) { 549 uint32_t i; 550 struct _WsbmSlabBuffer *sBuf; 551 552 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); 553 554 WSBMLISTDELINIT(list); 555 556 sBuf = slab->buffers; 557 for (i = 0; i < slab->numBuffers; ++i) { 558 if (sBuf->parent == slab) { 559 WSBM_COND_FREE(&sBuf->event); 560 wsbmBufStorageTakedown(&sBuf->storage); 561 } 562 sBuf++; 563 } 564 wsbmSetKernelBOFree(cache, slab->kbo); 565 free(slab->buffers); 566 free(slab); 567 } 568 } 569 } 570 571 static void 572 wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait) 573 { 574 struct _WsbmListHead *list, *prev, *first, *head; 575 struct _WsbmSlabBuffer *sBuf; 576 struct _WsbmSlab *slab; 577 int firstWasSignaled = 1; 578 int signaled; 579 uint32_t i; 580 int ret; 581 582 /* 583 * Rerun the freeing test if the youngest tested buffer 584 * was signaled, since there might be more idle buffers 585 * in the delay list. 586 */ 587 588 while (firstWasSignaled) { 589 firstWasSignaled = 0; 590 signaled = 0; 591 first = header->delayedBuffers.next; 592 593 /* Only examine the oldest 1/3 of delayed buffers: 594 */ 595 if (header->numDelayed > 3) { 596 for (i = 0; i < header->numDelayed; i += 3) { 597 first = first->next; 598 } 599 } 600 601 /* 602 * No need to take the buffer mutex for each buffer we loop 603 * through since we're currently the only user. 604 */ 605 606 head = first->next; 607 WSBMLISTFOREACHPREVSAFE(list, prev, head) { 608 609 if (list == &header->delayedBuffers) 610 break; 611 612 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); 613 614 slab = sBuf->parent; 615 616 if (!signaled) { 617 if (wait) { 618 ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); 619 if (ret) 620 break; 621 signaled = 1; 622 wait = 0; 623 } else { 624 signaled = 625 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); 626 #ifdef DEBUG_FENCESIGNALED 627 fencesignaled++; 628 #endif 629 } 630 if (signaled) { 631 if (list == first) 632 firstWasSignaled = 1; 633 wsbmFenceUnreference(&sBuf->fence); 634 header->numDelayed--; 635 wsbmSlabFreeBufferLocked(sBuf); 636 } else 637 break; 638 } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { 639 wsbmFenceUnreference(&sBuf->fence); 640 header->numDelayed--; 641 wsbmSlabFreeBufferLocked(sBuf); 642 } 643 } 644 } 645 } 646 647 static struct _WsbmSlabBuffer * 648 wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header) 649 { 650 static struct _WsbmSlabBuffer *buf; 651 struct _WsbmSlab *slab; 652 struct _WsbmListHead *list; 653 int count = WSBM_SLABPOOL_ALLOC_RETRIES; 654 655 WSBM_MUTEX_LOCK(&header->mutex); 656 while (header->slabs.next == &header->slabs && count > 0) { 657 wsbmSlabCheckFreeLocked(header, 0); 658 if (header->slabs.next != &header->slabs) 659 break; 660 661 WSBM_MUTEX_UNLOCK(&header->mutex); 662 if (count != WSBM_SLABPOOL_ALLOC_RETRIES) 663 usleep(1000); 664 WSBM_MUTEX_LOCK(&header->mutex); 665 (void)wsbmAllocSlab(header); 666 count--; 667 } 668 669 list = header->slabs.next; 670 if (list == &header->slabs) { 671 WSBM_MUTEX_UNLOCK(&header->mutex); 672 return NULL; 673 } 674 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head); 675 if (--slab->numFree == 0) 676 WSBMLISTDELINIT(list); 677 678 list = slab->freeBuffers.next; 679 WSBMLISTDELINIT(list); 680 681 WSBM_MUTEX_UNLOCK(&header->mutex); 682 buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); 683 684 buf->storage.destroyContainer = NULL; 685 686 #ifdef DEBUG_FENCESIGNALED 687 createbuffer++; 688 #endif 689 return buf; 690 } 691 692 static struct _WsbmBufStorage * 693 pool_create(struct _WsbmBufferPool *pool, unsigned long size, 694 uint32_t placement, unsigned alignment) 695 { 696 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); 697 struct _WsbmSlabSizeHeader *header; 698 struct _WsbmSlabBuffer *sBuf; 699 uint32_t i; 700 int ret; 701 702 /* 703 * FIXME: Check for compatibility. 704 */ 705 706 header = slabPool->headers; 707 for (i = 0; i < slabPool->numBuckets; ++i) { 708 if (header->bufSize >= size) 709 break; 710 header++; 711 } 712 713 if (i < slabPool->numBuckets) { 714 sBuf = wsbmSlabAllocBuffer(header); 715 return ((sBuf) ? &sBuf->storage : NULL); 716 } 717 718 /* 719 * Fall back to allocate a buffer object directly from DRM. 720 * and wrap it in a wsbmBO structure. 721 */ 722 723 sBuf = calloc(1, sizeof(*sBuf)); 724 725 if (!sBuf) 726 return NULL; 727 728 if (alignment) { 729 if ((alignment < slabPool->pageSize) 730 && (slabPool->pageSize % alignment)) 731 goto out_err0; 732 if ((alignment > slabPool->pageSize) 733 && (alignment % slabPool->pageSize)) 734 goto out_err0; 735 } 736 737 ret = wsbmBufStorageInit(&sBuf->storage, pool); 738 if (ret) 739 goto out_err0; 740 741 ret = WSBM_COND_INIT(&sBuf->event); 742 if (ret) 743 goto out_err1; 744 745 { 746 union ttm_pl_create_arg arg; 747 748 arg.req.size = size; 749 arg.req.placement = placement; 750 arg.req.page_alignment = alignment / slabPool->pageSize; 751 752 DRMRESTARTCOMMANDWRITEREAD(pool->fd, 753 slabPool->devOffset + TTM_PL_CREATE, 754 arg, ret); 755 756 if (ret) 757 goto out_err2; 758 759 sBuf->kBuf.gpuOffset = arg.rep.gpu_offset; 760 sBuf->kBuf.placement = arg.rep.placement; 761 sBuf->kBuf.handle = arg.rep.handle; 762 sBuf->mapHandle = arg.rep.map_handle; 763 sBuf->requestedSize = size; 764 765 sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, 766 pool->fd, sBuf->mapHandle); 767 768 if (sBuf->virtual == MAP_FAILED) 769 goto out_err3; 770 } 771 772 wsbmAtomicSet(&sBuf->writers, 0); 773 return &sBuf->storage; 774 out_err3: 775 { 776 struct ttm_pl_reference_req arg; 777 778 arg.handle = sBuf->kBuf.handle; 779 (void)drmCommandWriteRead(pool->fd, 780 slabPool->devOffset + TTM_PL_UNREF, 781 &arg, sizeof(arg)); 782 } 783 out_err2: 784 WSBM_COND_FREE(&sBuf->event); 785 out_err1: 786 wsbmBufStorageTakedown(&sBuf->storage); 787 out_err0: 788 free(sBuf); 789 return NULL; 790 } 791 792 static void 793 pool_destroy(struct _WsbmBufStorage **p_buf) 794 { 795 struct _WsbmBufStorage *buf = *p_buf; 796 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 797 struct _WsbmSlab *slab; 798 struct _WsbmSlabSizeHeader *header; 799 800 *p_buf = NULL; 801 802 if (!sBuf->isSlabBuffer) { 803 struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf); 804 struct ttm_pl_reference_req arg; 805 806 if (sBuf->virtual != NULL) { 807 (void)munmap(sBuf->virtual, sBuf->requestedSize); 808 sBuf->virtual = NULL; 809 } 810 811 arg.handle = sBuf->kBuf.handle; 812 (void)drmCommandWrite(slabPool->pool.fd, 813 slabPool->devOffset + TTM_PL_UNREF, 814 &arg, sizeof(arg)); 815 816 WSBM_COND_FREE(&sBuf->event); 817 wsbmBufStorageTakedown(&sBuf->storage); 818 free(sBuf); 819 return; 820 } 821 822 slab = sBuf->parent; 823 header = slab->header; 824 825 /* 826 * No need to take the buffer mutex below since we're the only user. 827 */ 828 829 WSBM_MUTEX_LOCK(&header->mutex); 830 sBuf->unFenced = 0; 831 wsbmAtomicSet(&sBuf->writers, 0); 832 wsbmAtomicSet(&sBuf->storage.refCount, 1); 833 834 if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) { 835 WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers); 836 header->numDelayed++; 837 } else { 838 if (sBuf->fence) 839 wsbmFenceUnreference(&sBuf->fence); 840 wsbmSlabFreeBufferLocked(sBuf); 841 } 842 843 WSBM_MUTEX_UNLOCK(&header->mutex); 844 } 845 846 static void 847 waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy) 848 { 849 struct _WsbmBufStorage *storage = &sBuf->storage; 850 851 while (sBuf->unFenced || sBuf->fence != NULL) { 852 853 if (sBuf->unFenced) 854 WSBM_COND_WAIT(&sBuf->event, &storage->mutex); 855 856 if (sBuf->fence != NULL) { 857 if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) { 858 struct _WsbmFenceObject *fence = 859 wsbmFenceReference(sBuf->fence); 860 861 WSBM_MUTEX_UNLOCK(&storage->mutex); 862 (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy); 863 WSBM_MUTEX_LOCK(&storage->mutex); 864 if (sBuf->fence == fence) 865 wsbmFenceUnreference(&sBuf->fence); 866 867 wsbmFenceUnreference(&fence); 868 } else { 869 wsbmFenceUnreference(&sBuf->fence); 870 } 871 } 872 } 873 } 874 875 static int 876 pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) 877 { 878 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 879 880 WSBM_MUTEX_LOCK(&buf->mutex); 881 waitIdleLocked(sBuf, lazy); 882 WSBM_MUTEX_UNLOCK(&buf->mutex); 883 884 return 0; 885 } 886 887 static int 888 pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual) 889 { 890 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 891 892 *virtual = sBuf->virtual; 893 894 return 0; 895 } 896 897 static void 898 pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused))) 899 { 900 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 901 902 if (wsbmAtomicDecZero(&sBuf->writers)) 903 WSBM_COND_BROADCAST(&sBuf->event); 904 } 905 906 static int 907 pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode) 908 { 909 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 910 int ret = 0; 911 912 WSBM_MUTEX_LOCK(&buf->mutex); 913 if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) { 914 int signaled; 915 916 if (sBuf->unFenced) { 917 ret = -EBUSY; 918 goto out_unlock; 919 } 920 921 if (sBuf->isSlabBuffer) 922 signaled = (sBuf->fence == NULL) || 923 wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType); 924 else 925 signaled = (sBuf->fence == NULL) || 926 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType); 927 928 ret = 0; 929 if (signaled) { 930 wsbmFenceUnreference(&sBuf->fence); 931 wsbmAtomicInc(&sBuf->writers); 932 } else 933 ret = -EBUSY; 934 goto out_unlock; 935 } 936 waitIdleLocked(sBuf, 0); 937 wsbmAtomicInc(&sBuf->writers); 938 out_unlock: 939 WSBM_MUTEX_UNLOCK(&buf->mutex); 940 return ret; 941 } 942 943 static void 944 pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused))) 945 { 946 ; 947 } 948 949 static unsigned long 950 pool_poolOffset(struct _WsbmBufStorage *buf) 951 { 952 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 953 954 return sBuf->start; 955 } 956 957 static unsigned long 958 pool_size(struct _WsbmBufStorage *buf) 959 { 960 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 961 962 if (!sBuf->isSlabBuffer) 963 return sBuf->requestedSize; 964 965 return sBuf->parent->header->bufSize; 966 } 967 968 static struct _WsbmKernelBuf * 969 pool_kernel(struct _WsbmBufStorage *buf) 970 { 971 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 972 973 return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf; 974 } 975 976 static unsigned long 977 pool_offset(struct _WsbmBufStorage *buf) 978 { 979 return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf); 980 } 981 982 static void 983 pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence) 984 { 985 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 986 struct _WsbmKernelBuf *kBuf; 987 988 WSBM_MUTEX_LOCK(&buf->mutex); 989 if (sBuf->fence) 990 wsbmFenceUnreference(&sBuf->fence); 991 992 kBuf = pool_kernel(buf); 993 sBuf->fenceType = kBuf->fence_type_mask; 994 if (!wsbmFenceSignaledCached(fence, sBuf->fenceType)) 995 sBuf->fence = wsbmFenceReference(fence); 996 997 sBuf->unFenced = 0; 998 WSBM_COND_BROADCAST(&sBuf->event); 999 WSBM_MUTEX_UNLOCK(&buf->mutex); 1000 } 1001 1002 static int 1003 pool_validate(struct _WsbmBufStorage *buf, 1004 uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused))) 1005 { 1006 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 1007 1008 WSBM_MUTEX_LOCK(&buf->mutex); 1009 while (wsbmAtomicRead(&sBuf->writers) != 0) { 1010 WSBM_COND_WAIT(&sBuf->event, &buf->mutex); 1011 } 1012 1013 sBuf->unFenced = 1; 1014 WSBM_MUTEX_UNLOCK(&buf->mutex); 1015 return 0; 1016 } 1017 1018 static void 1019 pool_unvalidate(struct _WsbmBufStorage *buf) 1020 { 1021 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf); 1022 1023 WSBM_MUTEX_LOCK(&buf->mutex); 1024 if (sBuf->unFenced) { 1025 sBuf->unFenced = 0; 1026 WSBM_COND_BROADCAST(&sBuf->event); 1027 } 1028 WSBM_MUTEX_UNLOCK(&buf->mutex); 1029 } 1030 1031 struct _WsbmSlabCache * 1032 wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec) 1033 { 1034 struct _WsbmSlabCache *tmp; 1035 1036 tmp = calloc(1, sizeof(*tmp)); 1037 if (!tmp) 1038 return NULL; 1039 1040 WSBM_MUTEX_INIT(&tmp->mutex); 1041 WSBM_MUTEX_LOCK(&tmp->mutex); 1042 tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000; 1043 tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000; 1044 tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000; 1045 1046 tmp->checkInterval.tv_usec = checkIntervalMsec * 1000; 1047 tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000; 1048 tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000; 1049 1050 gettimeofday(&tmp->nextCheck, NULL); 1051 wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval); 1052 WSBMINITLISTHEAD(&tmp->timeoutList); 1053 WSBMINITLISTHEAD(&tmp->unCached); 1054 WSBMINITLISTHEAD(&tmp->cached); 1055 WSBM_MUTEX_UNLOCK(&tmp->mutex); 1056 1057 return tmp; 1058 } 1059 1060 void 1061 wsbmSlabCacheFinish(struct _WsbmSlabCache *cache) 1062 { 1063 struct timeval time; 1064 1065 time = cache->nextCheck; 1066 WSBM_MUTEX_LOCK(&cache->mutex); 1067 wsbmTimeAdd(&time, &cache->checkInterval); 1068 wsbmFreeTimeoutKBOsLocked(cache, &time); 1069 WSBM_MUTEX_UNLOCK(&cache->mutex); 1070 1071 assert(cache->timeoutList.next == &cache->timeoutList); 1072 assert(cache->unCached.next == &cache->unCached); 1073 assert(cache->cached.next == &cache->cached); 1074 1075 WSBM_MUTEX_FREE(&cache->mutex); 1076 free(cache); 1077 } 1078 1079 static void 1080 wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size, 1081 struct _WsbmSlabSizeHeader *header) 1082 { 1083 WSBM_MUTEX_INIT(&header->mutex); 1084 WSBM_MUTEX_LOCK(&header->mutex); 1085 1086 WSBMINITLISTHEAD(&header->slabs); 1087 WSBMINITLISTHEAD(&header->freeSlabs); 1088 WSBMINITLISTHEAD(&header->delayedBuffers); 1089 1090 header->numDelayed = 0; 1091 header->slabPool = slabPool; 1092 header->bufSize = size; 1093 1094 WSBM_MUTEX_UNLOCK(&header->mutex); 1095 } 1096 1097 static void 1098 wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header) 1099 { 1100 struct _WsbmListHead *list, *next; 1101 struct _WsbmSlabBuffer *sBuf; 1102 1103 WSBM_MUTEX_LOCK(&header->mutex); 1104 WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) { 1105 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head); 1106 1107 if (sBuf->fence) { 1108 (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0); 1109 wsbmFenceUnreference(&sBuf->fence); 1110 } 1111 header->numDelayed--; 1112 wsbmSlabFreeBufferLocked(sBuf); 1113 } 1114 WSBM_MUTEX_UNLOCK(&header->mutex); 1115 WSBM_MUTEX_FREE(&header->mutex); 1116 } 1117 1118 static void 1119 pool_takedown(struct _WsbmBufferPool *pool) 1120 { 1121 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool); 1122 unsigned int i; 1123 1124 for (i = 0; i < slabPool->numBuckets; ++i) { 1125 wsbmFinishSizeHeader(&slabPool->headers[i]); 1126 } 1127 1128 free(slabPool->headers); 1129 free(slabPool->bucketSizes); 1130 free(slabPool); 1131 } 1132 1133 struct _WsbmBufferPool * 1134 wsbmSlabPoolInit(int fd, 1135 uint32_t devOffset, 1136 uint32_t placement, 1137 uint32_t validMask, 1138 uint32_t smallestSize, 1139 uint32_t numSizes, 1140 uint32_t desiredNumBuffers, 1141 uint32_t maxSlabSize, 1142 uint32_t pageAlignment, struct _WsbmSlabCache *cache) 1143 { 1144 struct _WsbmBufferPool *pool; 1145 struct _WsbmSlabPool *slabPool; 1146 uint32_t i; 1147 1148 slabPool = calloc(1, sizeof(*slabPool)); 1149 if (!slabPool) 1150 return NULL; 1151 1152 pool = &slabPool->pool; 1153 1154 slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes)); 1155 if (!slabPool->bucketSizes) 1156 goto out_err0; 1157 1158 slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers)); 1159 if (!slabPool->headers) 1160 goto out_err1; 1161 1162 slabPool->devOffset = devOffset; 1163 slabPool->cache = cache; 1164 slabPool->proposedPlacement = placement; 1165 slabPool->validMask = validMask; 1166 slabPool->numBuckets = numSizes; 1167 slabPool->pageSize = getpagesize(); 1168 slabPool->pageAlignment = pageAlignment; 1169 slabPool->maxSlabSize = maxSlabSize; 1170 slabPool->desiredNumBuffers = desiredNumBuffers; 1171 1172 for (i = 0; i < slabPool->numBuckets; ++i) { 1173 slabPool->bucketSizes[i] = (smallestSize << i); 1174 wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i], 1175 &slabPool->headers[i]); 1176 } 1177 1178 pool->fd = fd; 1179 pool->map = &pool_map; 1180 pool->unmap = &pool_unmap; 1181 pool->destroy = &pool_destroy; 1182 pool->offset = &pool_offset; 1183 pool->poolOffset = &pool_poolOffset; 1184 pool->size = &pool_size; 1185 pool->create = &pool_create; 1186 pool->fence = &pool_fence; 1187 pool->kernel = &pool_kernel; 1188 pool->validate = &pool_validate; 1189 pool->unvalidate = &pool_unvalidate; 1190 pool->waitIdle = &pool_waitIdle; 1191 pool->takeDown = &pool_takedown; 1192 pool->releasefromcpu = &pool_releaseFromCpu; 1193 pool->syncforcpu = &pool_syncForCpu; 1194 1195 return pool; 1196 1197 out_err1: 1198 free(slabPool->bucketSizes); 1199 out_err0: 1200 free(slabPool); 1201 1202 return NULL; 1203 } 1204
