1 /************************************************************************** 2 * 3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA 4 * All Rights Reserved. 5 * Copyright 2009 Vmware, Inc., Palo Alto, CA., USA 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 26 * USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 **************************************************************************/ 29 /* 30 * Authors: Thomas Hellstrm <thomas-at-tungstengraphics-dot-com> 31 * Keith Whitwell <keithw-at-tungstengraphics-dot-com> 32 */ 33 34 #ifdef HAVE_CONFIG_H 35 #include "config.h" 36 #endif 37 38 #include <stdlib.h> 39 #include "errno.h" 40 #include "string.h" 41 #include "wsbm_pool.h" 42 #include "wsbm_manager.h" 43 #include "wsbm_fencemgr.h" 44 #include "wsbm_driver.h" 45 #include "wsbm_priv.h" 46 #include "wsbm_util.h" 47 #include "wsbm_atomic.h" 48 #include "assert.h" 49 50 #define WSBM_BODATA_SIZE_ACCEPT 4096 51 52 #define WSBM_BUFFER_COMPLEX 0 53 #define WSBM_BUFFER_SIMPLE 1 54 #define WSBM_BUFFER_REF 2 55 56 struct _ValidateList 57 { 58 unsigned numTarget; 59 unsigned numCurrent; 60 unsigned numOnList; 61 unsigned hashSize; 62 uint32_t hashMask; 63 int driverData; 64 struct _WsbmListHead list; 65 struct _WsbmListHead free; 66 struct _WsbmListHead *hashTable; 67 }; 68 69 struct _WsbmBufferObject 70 { 71 /* Left to the client to protect this data for now. */ 72 73 struct _WsbmAtomic refCount; 74 struct _WsbmBufStorage *storage; 75 76 uint32_t placement; 77 unsigned alignment; 78 unsigned bufferType; 79 struct _WsbmBufferPool *pool; 80 }; 81 82 struct _WsbmBufferList 83 { 84 int hasKernelBuffers; 85 86 struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */ 87 struct _ValidateList userBuffers; /* List of user-space buffers needing validation */ 88 }; 89 90 static struct _WsbmMutex bmMutex; 91 static struct _WsbmCond bmCond; 92 static int initialized = 0; 93 static void *commonData = NULL; 94 95 static int kernelReaders = 0; 96 static int kernelLocked = 0; 97 98 int 99 wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf) 100 { 101 int ret; 102 103 wsbmCurThreadFunc = tf; 104 wsbmCurVNodeFunc = vf; 105 106 ret = WSBM_MUTEX_INIT(&bmMutex); 107 if (ret) 108 return -ENOMEM; 109 ret = WSBM_COND_INIT(&bmCond); 110 if (ret) { 111 WSBM_MUTEX_FREE(&bmMutex); 112 return -ENOMEM; 113 } 114 115 initialized = 1; 116 return 0; 117 } 118 119 void 120 wsbmCommonDataSet(void *d) 121 { 122 commonData = d; 123 } 124 125 void * 126 wsbmCommonDataGet(void) 127 { 128 return commonData; 129 } 130 131 int 132 wsbmIsInitialized(void) 133 { 134 return initialized; 135 } 136 137 void 138 wsbmTakedown(void) 139 { 140 initialized = 0; 141 commonData = NULL; 142 WSBM_COND_FREE(&bmCond); 143 WSBM_MUTEX_FREE(&bmMutex); 144 } 145 146 static struct _ValidateNode * 147 validateListAddNode(struct _ValidateList *list, void *item, 148 uint32_t hash, uint64_t flags, uint64_t mask) 149 { 150 struct _ValidateNode *node; 151 struct _WsbmListHead *l; 152 struct _WsbmListHead *hashHead; 153 154 l = list->free.next; 155 if (l == &list->free) { 156 node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), 0); 157 if (!node) { 158 return NULL; 159 } 160 list->numCurrent++; 161 } else { 162 WSBMLISTDEL(l); 163 node = WSBMLISTENTRY(l, struct _ValidateNode, head); 164 } 165 node->buf = item; 166 node->set_flags = flags & mask; 167 node->clr_flags = (~flags) & mask; 168 node->listItem = list->numOnList; 169 WSBMLISTADDTAIL(&node->head, &list->list); 170 list->numOnList++; 171 hashHead = list->hashTable + hash; 172 WSBMLISTADDTAIL(&node->hashHead, hashHead); 173 174 return node; 175 } 176 177 static uint32_t 178 wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask) 179 { 180 uint32_t hash, i; 181 182 for (hash = 0, i = 0; i < len; ++i) { 183 hash += *key++; 184 hash += (hash << 10); 185 hash ^= (hash >> 6); 186 } 187 188 hash += (hash << 3); 189 hash ^= (hash >> 11); 190 hash += (hash << 15); 191 192 return hash & mask; 193 } 194 195 static void 196 validateFreeList(struct _ValidateList *list) 197 { 198 struct _ValidateNode *node; 199 struct _WsbmListHead *l; 200 201 l = list->list.next; 202 while (l != &list->list) { 203 WSBMLISTDEL(l); 204 node = WSBMLISTENTRY(l, struct _ValidateNode, head); 205 206 WSBMLISTDEL(&node->hashHead); 207 node->func->free(node); 208 l = list->list.next; 209 list->numCurrent--; 210 list->numOnList--; 211 } 212 213 l = list->free.next; 214 while (l != &list->free) { 215 WSBMLISTDEL(l); 216 node = WSBMLISTENTRY(l, struct _ValidateNode, head); 217 218 node->func->free(node); 219 l = list->free.next; 220 list->numCurrent--; 221 } 222 free(list->hashTable); 223 } 224 225 static int 226 validateListAdjustNodes(struct _ValidateList *list) 227 { 228 struct _ValidateNode *node; 229 struct _WsbmListHead *l; 230 int ret = 0; 231 232 while (list->numCurrent < list->numTarget) { 233 node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), list->driverData); 234 if (!node) { 235 ret = -ENOMEM; 236 break; 237 } 238 list->numCurrent++; 239 WSBMLISTADD(&node->head, &list->free); 240 } 241 242 while (list->numCurrent > list->numTarget) { 243 l = list->free.next; 244 if (l == &list->free) 245 break; 246 WSBMLISTDEL(l); 247 node = WSBMLISTENTRY(l, struct _ValidateNode, head); 248 249 node->func->free(node); 250 list->numCurrent--; 251 } 252 return ret; 253 } 254 255 static inline int 256 wsbmPot(unsigned int val) 257 { 258 unsigned int shift = 0; 259 while(val > (unsigned int)(1 << shift)) 260 shift++; 261 262 return shift; 263 } 264 265 266 267 static int 268 validateCreateList(int numTarget, struct _ValidateList *list, int driverData) 269 { 270 unsigned int i; 271 unsigned int shift = wsbmPot(numTarget); 272 int ret; 273 274 list->hashSize = (1 << shift); 275 list->hashMask = list->hashSize - 1; 276 277 list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable)); 278 if (!list->hashTable) 279 return -ENOMEM; 280 281 for (i = 0; i < list->hashSize; ++i) 282 WSBMINITLISTHEAD(&list->hashTable[i]); 283 284 WSBMINITLISTHEAD(&list->list); 285 WSBMINITLISTHEAD(&list->free); 286 list->numTarget = numTarget; 287 list->numCurrent = 0; 288 list->numOnList = 0; 289 list->driverData = driverData; 290 ret = validateListAdjustNodes(list); 291 if (ret != 0) 292 free(list->hashTable); 293 294 return ret; 295 } 296 297 static int 298 validateResetList(struct _ValidateList *list) 299 { 300 struct _WsbmListHead *l; 301 struct _ValidateNode *node; 302 int ret; 303 304 ret = validateListAdjustNodes(list); 305 if (ret) 306 return ret; 307 308 l = list->list.next; 309 while (l != &list->list) { 310 WSBMLISTDEL(l); 311 node = WSBMLISTENTRY(l, struct _ValidateNode, head); 312 313 WSBMLISTDEL(&node->hashHead); 314 WSBMLISTADD(l, &list->free); 315 list->numOnList--; 316 l = list->list.next; 317 } 318 return validateListAdjustNodes(list); 319 } 320 321 void 322 wsbmWriteLockKernelBO(void) 323 { 324 WSBM_MUTEX_LOCK(&bmMutex); 325 while (kernelReaders != 0) 326 WSBM_COND_WAIT(&bmCond, &bmMutex); 327 kernelLocked = 1; 328 } 329 330 void 331 wsbmWriteUnlockKernelBO(void) 332 { 333 kernelLocked = 0; 334 WSBM_MUTEX_UNLOCK(&bmMutex); 335 } 336 337 void 338 wsbmReadLockKernelBO(void) 339 { 340 WSBM_MUTEX_LOCK(&bmMutex); 341 if (kernelReaders++ == 0) 342 kernelLocked = 1; 343 WSBM_MUTEX_UNLOCK(&bmMutex); 344 } 345 346 void 347 wsbmReadUnlockKernelBO(void) 348 { 349 WSBM_MUTEX_LOCK(&bmMutex); 350 if (--kernelReaders == 0) { 351 kernelLocked = 0; 352 WSBM_COND_BROADCAST(&bmCond); 353 } 354 WSBM_MUTEX_UNLOCK(&bmMutex); 355 } 356 357 void 358 wsbmBOWaitIdle(struct _WsbmBufferObject *buf, int lazy) 359 { 360 struct _WsbmBufStorage *storage; 361 362 storage = buf->storage; 363 if (!storage) 364 return; 365 366 (void)storage->pool->waitIdle(storage, lazy); 367 } 368 369 void * 370 wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode) 371 { 372 struct _WsbmBufStorage *storage = buf->storage; 373 void *virtual; 374 int retval; 375 376 retval = storage->pool->map(storage, mode, &virtual); 377 378 return (retval == 0) ? virtual : NULL; 379 } 380 381 void 382 wsbmBOUnmap(struct _WsbmBufferObject *buf) 383 { 384 struct _WsbmBufStorage *storage = buf->storage; 385 386 if (!storage) 387 return; 388 389 storage->pool->unmap(storage); 390 } 391 392 int 393 wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode) 394 { 395 struct _WsbmBufStorage *storage = buf->storage; 396 397 return storage->pool->syncforcpu(storage, mode); 398 } 399 400 void 401 wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, unsigned mode) 402 { 403 struct _WsbmBufStorage *storage = buf->storage; 404 405 storage->pool->releasefromcpu(storage, mode); 406 } 407 408 unsigned long 409 wsbmBOOffsetHint(struct _WsbmBufferObject *buf) 410 { 411 struct _WsbmBufStorage *storage = buf->storage; 412 413 return storage->pool->offset(storage); 414 } 415 416 unsigned long 417 wsbmBOPoolOffset(struct _WsbmBufferObject *buf) 418 { 419 struct _WsbmBufStorage *storage = buf->storage; 420 421 return storage->pool->poolOffset(storage); 422 } 423 424 uint32_t 425 wsbmBOPlacementHint(struct _WsbmBufferObject * buf) 426 { 427 struct _WsbmBufStorage *storage = buf->storage; 428 429 assert(buf->storage != NULL); 430 431 return storage->pool->placement(storage); 432 } 433 434 struct _WsbmBufferObject * 435 wsbmBOReference(struct _WsbmBufferObject *buf) 436 { 437 if (buf->bufferType == WSBM_BUFFER_SIMPLE) { 438 wsbmAtomicInc(&buf->storage->refCount); 439 } else { 440 wsbmAtomicInc(&buf->refCount); 441 } 442 return buf; 443 } 444 445 int 446 wsbmBOSetStatus(struct _WsbmBufferObject *buf, 447 uint32_t setFlags, uint32_t clrFlags) 448 { 449 struct _WsbmBufStorage *storage = buf->storage; 450 451 if (!storage) 452 return 0; 453 454 if (storage->pool->setStatus == NULL) 455 return -EINVAL; 456 457 return storage->pool->setStatus(storage, setFlags, clrFlags); 458 } 459 460 void 461 wsbmBOUnreference(struct _WsbmBufferObject **p_buf) 462 { 463 struct _WsbmBufferObject *buf = *p_buf; 464 465 *p_buf = NULL; 466 467 if (!buf) 468 return; 469 470 if (buf->bufferType == WSBM_BUFFER_SIMPLE) { 471 struct _WsbmBufStorage *dummy = buf->storage; 472 473 wsbmBufStorageUnref(&dummy); 474 return; 475 } 476 477 if (wsbmAtomicDecZero(&buf->refCount)) { 478 wsbmBufStorageUnref(&buf->storage); 479 free(buf); 480 } 481 } 482 483 int 484 wsbmBOData(struct _WsbmBufferObject *buf, 485 unsigned size, const void *data, 486 struct _WsbmBufferPool *newPool, uint32_t placement) 487 { 488 void *virtual = NULL; 489 int newBuffer; 490 int retval = 0; 491 struct _WsbmBufStorage *storage; 492 int synced = 0; 493 uint32_t placement_diff; 494 struct _WsbmBufferPool *curPool; 495 496 if (buf->bufferType == WSBM_BUFFER_SIMPLE) 497 return -EINVAL; 498 499 storage = buf->storage; 500 501 if (newPool == NULL) 502 newPool = buf->pool; 503 504 if (newPool == NULL) 505 return -EINVAL; 506 507 newBuffer = (!storage || storage->pool != newPool || 508 storage->pool->size(storage) < size || 509 storage->pool->size(storage) > 510 size + WSBM_BODATA_SIZE_ACCEPT); 511 512 if (!placement) 513 placement = buf->placement; 514 515 if (newBuffer) { 516 if (buf->bufferType == WSBM_BUFFER_REF) 517 return -EINVAL; 518 519 wsbmBufStorageUnref(&buf->storage); 520 521 if (size == 0) { 522 buf->pool = newPool; 523 buf->placement = placement; 524 retval = 0; 525 goto out; 526 } 527 528 buf->storage = 529 newPool->create(newPool, size, placement, buf->alignment); 530 if (!buf->storage) { 531 retval = -ENOMEM; 532 goto out; 533 } 534 535 buf->placement = placement; 536 buf->pool = newPool; 537 } else if (wsbmAtomicRead(&storage->onList) || 538 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | 539 WSBM_SYNCCPU_DONT_BLOCK)) { 540 /* 541 * Buffer is busy. need to create a new one. 542 */ 543 544 struct _WsbmBufStorage *tmp_storage; 545 546 curPool = storage->pool; 547 548 tmp_storage = 549 curPool->create(curPool, size, placement, buf->alignment); 550 551 if (tmp_storage) { 552 wsbmBufStorageUnref(&buf->storage); 553 buf->storage = tmp_storage; 554 buf->placement = placement; 555 } else { 556 retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); 557 if (retval) 558 goto out; 559 synced = 1; 560 } 561 } else 562 synced = 1; 563 564 placement_diff = placement ^ buf->placement; 565 566 /* 567 * We might need to change buffer placement. 568 */ 569 570 storage = buf->storage; 571 curPool = storage->pool; 572 573 if (placement_diff) { 574 assert(curPool->setStatus != NULL); 575 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 576 retval = curPool->setStatus(storage, 577 placement_diff & placement, 578 placement_diff & ~placement); 579 if (retval) 580 goto out; 581 582 buf->placement = placement; 583 584 } 585 586 if (!synced) { 587 retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); 588 589 if (retval) 590 goto out; 591 synced = 1; 592 } 593 594 storage = buf->storage; 595 curPool = storage->pool; 596 597 if (data) { 598 retval = curPool->map(storage, WSBM_ACCESS_WRITE, &virtual); 599 if (retval) 600 goto out; 601 memcpy(virtual, data, size); 602 curPool->unmap(storage); 603 } 604 605 out: 606 607 if (synced) 608 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 609 610 return retval; 611 } 612 613 int 614 wsbmBODataUB(struct _WsbmBufferObject *buf, 615 unsigned size, const void *data, struct _WsbmBufferPool *newPool, 616 uint32_t placement, const unsigned long *user_ptr, int fd) 617 { 618 int newBuffer; 619 int retval = 0; 620 struct _WsbmBufStorage *storage; 621 int synced = 0; 622 uint32_t placement_diff; 623 struct _WsbmBufferPool *curPool; 624 extern struct _WsbmBufStorage * 625 ttm_pool_ub_create(struct _WsbmBufferPool *pool, 626 unsigned long size, uint32_t placement, unsigned alignment, 627 const unsigned long *user_ptr, int fd); 628 629 if (buf->bufferType == WSBM_BUFFER_SIMPLE) 630 return -EINVAL; 631 632 storage = buf->storage; 633 634 if (newPool == NULL) 635 newPool = buf->pool; 636 637 if (newPool == NULL) 638 return -EINVAL; 639 640 newBuffer = (!storage || storage->pool != newPool || 641 storage->pool->size(storage) < size || 642 storage->pool->size(storage) > 643 size + WSBM_BODATA_SIZE_ACCEPT); 644 645 if (!placement) 646 placement = buf->placement; 647 648 if (newBuffer) { 649 if (buf->bufferType == WSBM_BUFFER_REF) 650 return -EINVAL; 651 652 wsbmBufStorageUnref(&buf->storage); 653 654 if (size == 0) { 655 buf->pool = newPool; 656 buf->placement = placement; 657 retval = 0; 658 goto out; 659 } 660 661 buf->storage = 662 ttm_pool_ub_create(newPool, size, placement, buf->alignment, user_ptr, fd); 663 if (!buf->storage) { 664 retval = -ENOMEM; 665 goto out; 666 } 667 668 buf->placement = placement; 669 buf->pool = newPool; 670 } else if (wsbmAtomicRead(&storage->onList) || 671 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE | 672 WSBM_SYNCCPU_DONT_BLOCK)) { 673 /* 674 * Buffer is busy. need to create a new one. 675 * Actually such case will not be encountered for current ICS implementation 676 * TODO: maybe need refine the following code when such usage case is required 677 */ 678 679 struct _WsbmBufStorage *tmp_storage; 680 681 curPool = storage->pool; 682 683 tmp_storage = 684 ttm_pool_ub_create(curPool, size, placement, buf->alignment, user_ptr, fd); 685 686 if (tmp_storage) { 687 wsbmBufStorageUnref(&buf->storage); 688 buf->storage = tmp_storage; 689 buf->placement = placement; 690 } else { 691 retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); 692 if (retval) 693 goto out; 694 synced = 1; 695 } 696 } else { 697 synced = 1; 698 } 699 700 placement_diff = placement ^ buf->placement; 701 702 /* 703 * We might need to change buffer placement. 704 */ 705 706 storage = buf->storage; 707 curPool = storage->pool; 708 709 if (placement_diff) { 710 assert(curPool->setStatus != NULL); 711 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 712 retval = curPool->setStatus(storage, 713 placement_diff & placement, 714 placement_diff & ~placement); 715 if (retval) 716 goto out; 717 718 buf->placement = placement; 719 } 720 721 if (!synced) { 722 retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE); 723 if (retval) 724 goto out; 725 synced = 1; 726 } 727 728 storage = buf->storage; 729 curPool = storage->pool; 730 731 if (data) { 732 memcpy((unsigned long *) user_ptr, data, size); 733 } 734 735 out: 736 737 if (synced) 738 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 739 740 return retval; 741 } 742 743 static struct _WsbmBufStorage * 744 wsbmStorageClone(struct _WsbmBufferObject *buf) 745 { 746 struct _WsbmBufStorage *storage = buf->storage; 747 struct _WsbmBufferPool *pool = storage->pool; 748 749 return pool->create(pool, pool->size(storage), buf->placement, 750 buf->alignment); 751 } 752 753 struct _WsbmBufferObject * 754 wsbmBOClone(struct _WsbmBufferObject *buf, 755 int (*accelCopy) (struct _WsbmBufferObject *, 756 struct _WsbmBufferObject *)) 757 { 758 struct _WsbmBufferObject *newBuf; 759 int ret; 760 761 newBuf = malloc(sizeof(*newBuf)); 762 if (!newBuf) 763 return NULL; 764 765 *newBuf = *buf; 766 newBuf->storage = wsbmStorageClone(buf); 767 if (!newBuf->storage) 768 goto out_err0; 769 770 wsbmAtomicSet(&newBuf->refCount, 1); 771 if (!accelCopy || accelCopy(newBuf, buf) != 0) { 772 773 struct _WsbmBufferPool *pool = buf->storage->pool; 774 struct _WsbmBufStorage *storage = buf->storage; 775 struct _WsbmBufStorage *newStorage = newBuf->storage; 776 void *virtual; 777 void *nVirtual; 778 779 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); 780 if (ret) 781 goto out_err1; 782 ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); 783 if (ret) 784 goto out_err2; 785 ret = pool->map(newStorage, WSBM_ACCESS_WRITE, &nVirtual); 786 if (ret) 787 goto out_err3; 788 789 memcpy(nVirtual, virtual, pool->size(storage)); 790 pool->unmap(newBuf->storage); 791 pool->unmap(buf->storage); 792 pool->releasefromcpu(storage, WSBM_SYNCCPU_READ); 793 } 794 795 return newBuf; 796 out_err3: 797 buf->pool->unmap(buf->storage); 798 out_err2: 799 buf->pool->releasefromcpu(buf->storage, WSBM_SYNCCPU_READ); 800 out_err1: 801 wsbmBufStorageUnref(&newBuf->storage); 802 out_err0: 803 free(newBuf); 804 return 0; 805 } 806 807 int 808 wsbmBOSubData(struct _WsbmBufferObject *buf, 809 unsigned long offset, unsigned long size, const void *data, 810 int (*accelCopy) (struct _WsbmBufferObject *, 811 struct _WsbmBufferObject *)) 812 { 813 int ret = 0; 814 815 if (buf->bufferType == WSBM_BUFFER_SIMPLE) 816 return -EINVAL; 817 818 if (size && data) { 819 void *virtual; 820 struct _WsbmBufStorage *storage = buf->storage; 821 struct _WsbmBufferPool *pool = storage->pool; 822 823 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); 824 if (ret) 825 goto out; 826 827 if (wsbmAtomicRead(&storage->onList)) { 828 829 struct _WsbmBufferObject *newBuf; 830 831 /* 832 * Another context has this buffer on its validate list. 833 * This should be a very rare situation, but it can be valid, 834 * and therefore we must deal with it by cloning the storage. 835 */ 836 837 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 838 newBuf = wsbmBOClone(buf, accelCopy); 839 840 /* 841 * If clone fails we have the choice of either bailing. 842 * (The other context will be happy), or go on and update 843 * the old buffer anyway. (We will be happy). We choose the 844 * latter. 845 */ 846 847 if (newBuf) { 848 storage = newBuf->storage; 849 wsbmAtomicInc(&storage->refCount); 850 wsbmBufStorageUnref(&buf->storage); 851 buf->storage = storage; 852 wsbmBOUnreference(&newBuf); 853 pool = storage->pool; 854 } 855 856 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE); 857 if (ret) 858 goto out; 859 } 860 861 ret = pool->map(storage, WSBM_ACCESS_WRITE, &virtual); 862 if (ret) { 863 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 864 goto out; 865 } 866 867 memcpy((unsigned char *)virtual + offset, data, size); 868 pool->unmap(storage); 869 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 870 } 871 out: 872 return ret; 873 } 874 875 int 876 wsbmBOGetSubData(struct _WsbmBufferObject *buf, 877 unsigned long offset, unsigned long size, void *data) 878 { 879 int ret = 0; 880 881 if (size && data) { 882 void *virtual; 883 struct _WsbmBufStorage *storage = buf->storage; 884 struct _WsbmBufferPool *pool = storage->pool; 885 886 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ); 887 if (ret) 888 goto out; 889 ret = pool->map(storage, WSBM_ACCESS_READ, &virtual); 890 if (ret) { 891 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 892 goto out; 893 } 894 memcpy(data, (unsigned char *)virtual + offset, size); 895 pool->unmap(storage); 896 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE); 897 } 898 out: 899 return ret; 900 } 901 902 int 903 wsbmBOSetReferenced(struct _WsbmBufferObject *buf, unsigned long handle) 904 { 905 int ret = 0; 906 907 wsbmBufStorageUnref(&buf->storage); 908 if (buf->pool->createByReference == NULL) { 909 ret = -EINVAL; 910 goto out; 911 } 912 buf->storage = buf->pool->createByReference(buf->pool, handle); 913 if (!buf->storage) { 914 ret = -EINVAL; 915 goto out; 916 } 917 buf->bufferType = WSBM_BUFFER_REF; 918 out: 919 return ret; 920 } 921 922 void 923 wsbmBOFreeSimple(void *ptr) 924 { 925 free(ptr); 926 } 927 928 struct _WsbmBufferObject * 929 wsbmBOCreateSimple(struct _WsbmBufferPool *pool, 930 unsigned long size, 931 uint32_t placement, 932 unsigned alignment, size_t extra_size, size_t * offset) 933 { 934 struct _WsbmBufferObject *buf; 935 struct _WsbmBufStorage *storage; 936 937 *offset = (sizeof(*buf) + 15) & ~15; 938 939 if (extra_size) { 940 extra_size += *offset - sizeof(*buf); 941 } 942 943 buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf) + extra_size); 944 if (!buf) 945 return NULL; 946 947 storage = pool->create(pool, size, placement, alignment); 948 if (!storage) 949 goto out_err0; 950 951 storage->destroyContainer = &wsbmBOFreeSimple; 952 storage->destroyArg = buf; 953 954 buf->storage = storage; 955 buf->alignment = alignment; 956 buf->pool = pool; 957 buf->placement = placement; 958 buf->bufferType = WSBM_BUFFER_SIMPLE; 959 960 return buf; 961 962 out_err0: 963 free(buf); 964 return NULL; 965 } 966 967 int 968 wsbmGenBuffers(struct _WsbmBufferPool *pool, 969 unsigned n, 970 struct _WsbmBufferObject *buffers[], 971 unsigned alignment, uint32_t placement) 972 { 973 struct _WsbmBufferObject *buf; 974 unsigned int i; 975 976 placement = (placement) ? placement : 977 WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED; 978 979 for (i = 0; i < n; ++i) { 980 buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf)); 981 if (!buf) 982 return -ENOMEM; 983 984 wsbmAtomicSet(&buf->refCount, 1); 985 buf->placement = placement; 986 buf->alignment = alignment; 987 buf->pool = pool; 988 buf->bufferType = WSBM_BUFFER_COMPLEX; 989 buffers[i] = buf; 990 } 991 return 0; 992 } 993 994 void 995 wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[]) 996 { 997 unsigned int i; 998 999 for (i = 0; i < n; ++i) { 1000 wsbmBOUnreference(&buffers[i]); 1001 } 1002 } 1003 1004 /* 1005 * Note that lists are per-context and don't need mutex protection. 1006 */ 1007 1008 struct _WsbmBufferList * 1009 wsbmBOCreateList(int target, int hasKernelBuffers) 1010 { 1011 struct _WsbmBufferList *list = calloc(sizeof(*list), 1); 1012 int ret; 1013 1014 if (!list) 1015 return NULL; 1016 list->hasKernelBuffers = hasKernelBuffers; 1017 if (hasKernelBuffers) { 1018 ret = validateCreateList(target, &list->kernelBuffers, 0); 1019 if (ret) 1020 return NULL; 1021 } 1022 1023 ret = validateCreateList(target, &list->userBuffers, 1); 1024 if (ret) { 1025 validateFreeList(&list->kernelBuffers); 1026 return NULL; 1027 } 1028 1029 return list; 1030 } 1031 1032 int 1033 wsbmBOResetList(struct _WsbmBufferList *list) 1034 { 1035 int ret; 1036 1037 if (list->hasKernelBuffers) { 1038 ret = validateResetList(&list->kernelBuffers); 1039 if (ret) 1040 return ret; 1041 } 1042 ret = validateResetList(&list->userBuffers); 1043 return ret; 1044 } 1045 1046 void 1047 wsbmBOFreeList(struct _WsbmBufferList *list) 1048 { 1049 if (list->hasKernelBuffers) 1050 validateFreeList(&list->kernelBuffers); 1051 validateFreeList(&list->userBuffers); 1052 free(list); 1053 } 1054 1055 static int 1056 wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags, 1057 uint64_t mask, int *itemLoc, 1058 struct _ValidateNode **pnode, int *newItem) 1059 { 1060 struct _ValidateNode *node, *cur; 1061 struct _WsbmListHead *l; 1062 struct _WsbmListHead *hashHead; 1063 uint32_t hash; 1064 uint32_t count = 0; 1065 uint32_t key = (unsigned long) buf; 1066 1067 cur = NULL; 1068 hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask); 1069 hashHead = list->hashTable + hash; 1070 *newItem = 0; 1071 1072 for (l = hashHead->next; l != hashHead; l = l->next) { 1073 count++; 1074 node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead); 1075 1076 if (node->buf == buf) { 1077 cur = node; 1078 break; 1079 } 1080 } 1081 1082 if (!cur) { 1083 cur = validateListAddNode(list, buf, hash, flags, mask); 1084 if (!cur) 1085 return -ENOMEM; 1086 *newItem = 1; 1087 cur->func->clear(cur); 1088 } else { 1089 uint64_t set_flags = flags & mask; 1090 uint64_t clr_flags = (~flags) & mask; 1091 1092 if (((cur->clr_flags | clr_flags) & WSBM_PL_MASK_MEM) == 1093 WSBM_PL_MASK_MEM) { 1094 /* 1095 * No available memory type left. Bail. 1096 */ 1097 return -EINVAL; 1098 } 1099 1100 if ((cur->set_flags | set_flags) & 1101 (cur->clr_flags | clr_flags) & ~WSBM_PL_MASK_MEM) { 1102 /* 1103 * Conflicting flags. Bail. 1104 */ 1105 return -EINVAL; 1106 } 1107 1108 cur->set_flags &= ~(clr_flags & WSBM_PL_MASK_MEM); 1109 cur->set_flags |= (set_flags & ~WSBM_PL_MASK_MEM); 1110 cur->clr_flags |= clr_flags; 1111 } 1112 *itemLoc = cur->listItem; 1113 if (pnode) 1114 *pnode = cur; 1115 return 0; 1116 } 1117 1118 int 1119 wsbmBOAddListItem(struct _WsbmBufferList *list, 1120 struct _WsbmBufferObject *buf, 1121 uint64_t flags, uint64_t mask, int *itemLoc, 1122 struct _ValidateNode **node) 1123 { 1124 int newItem; 1125 struct _WsbmBufStorage *storage = buf->storage; 1126 int ret; 1127 int dummy; 1128 struct _ValidateNode *dummyNode; 1129 1130 if (list->hasKernelBuffers) { 1131 ret = wsbmAddValidateItem(&list->kernelBuffers, 1132 storage->pool->kernel(storage), 1133 flags, mask, itemLoc, node, &dummy); 1134 if (ret) 1135 goto out_unlock; 1136 } else { 1137 *node = NULL; 1138 *itemLoc = -1000; 1139 } 1140 1141 ret = wsbmAddValidateItem(&list->userBuffers, storage, 1142 flags, mask, &dummy, &dummyNode, &newItem); 1143 if (ret) 1144 goto out_unlock; 1145 1146 if (newItem) { 1147 wsbmAtomicInc(&storage->refCount); 1148 wsbmAtomicInc(&storage->onList); 1149 } 1150 1151 out_unlock: 1152 return ret; 1153 } 1154 1155 void 1156 wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence) 1157 { 1158 struct _WsbmBufStorage *storage; 1159 1160 storage = buf->storage; 1161 if (storage->pool->fence) 1162 storage->pool->fence(storage, fence); 1163 1164 } 1165 1166 int 1167 wsbmBOOnList(const struct _WsbmBufferObject *buf) 1168 { 1169 if (buf->storage == NULL) 1170 return 0; 1171 return wsbmAtomicRead(&buf->storage->onList); 1172 } 1173 1174 int 1175 wsbmBOUnrefUserList(struct _WsbmBufferList *list) 1176 { 1177 struct _WsbmBufStorage *storage; 1178 void *curBuf; 1179 1180 curBuf = validateListIterator(&list->userBuffers); 1181 1182 while (curBuf) { 1183 storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); 1184 wsbmAtomicDec(&storage->onList); 1185 wsbmBufStorageUnref(&storage); 1186 curBuf = validateListNext(&list->userBuffers, curBuf); 1187 } 1188 1189 return wsbmBOResetList(list); 1190 } 1191 1192 1193 int 1194 wsbmBOFenceUserList(struct _WsbmBufferList *list, 1195 struct _WsbmFenceObject *fence) 1196 { 1197 struct _WsbmBufStorage *storage; 1198 void *curBuf; 1199 1200 curBuf = validateListIterator(&list->userBuffers); 1201 1202 /* 1203 * User-space fencing callbacks. 1204 */ 1205 1206 while (curBuf) { 1207 storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf); 1208 1209 storage->pool->fence(storage, fence); 1210 wsbmAtomicDec(&storage->onList); 1211 wsbmBufStorageUnref(&storage); 1212 curBuf = validateListNext(&list->userBuffers, curBuf); 1213 } 1214 1215 return wsbmBOResetList(list); 1216 } 1217 1218 int 1219 wsbmBOValidateUserList(struct _WsbmBufferList *list) 1220 { 1221 void *curBuf; 1222 struct _WsbmBufStorage *storage; 1223 struct _ValidateNode *node; 1224 int ret; 1225 1226 curBuf = validateListIterator(&list->userBuffers); 1227 1228 /* 1229 * User-space validation callbacks. 1230 */ 1231 1232 while (curBuf) { 1233 node = validateListNode(curBuf); 1234 storage = (struct _WsbmBufStorage *)node->buf; 1235 if (storage->pool->validate) { 1236 ret = storage->pool->validate(storage, node->set_flags, 1237 node->clr_flags); 1238 if (ret) 1239 return ret; 1240 } 1241 curBuf = validateListNext(&list->userBuffers, curBuf); 1242 } 1243 return 0; 1244 } 1245 1246 int 1247 wsbmBOUnvalidateUserList(struct _WsbmBufferList *list) 1248 { 1249 void *curBuf; 1250 struct _WsbmBufStorage *storage; 1251 struct _ValidateNode *node; 1252 1253 curBuf = validateListIterator(&list->userBuffers); 1254 1255 /* 1256 * User-space validation callbacks. 1257 */ 1258 1259 while (curBuf) { 1260 node = validateListNode(curBuf); 1261 storage = (struct _WsbmBufStorage *)node->buf; 1262 if (storage->pool->unvalidate) { 1263 storage->pool->unvalidate(storage); 1264 } 1265 wsbmAtomicDec(&storage->onList); 1266 wsbmBufStorageUnref(&storage); 1267 curBuf = validateListNext(&list->userBuffers, curBuf); 1268 } 1269 return wsbmBOResetList(list); 1270 } 1271 1272 void 1273 wsbmPoolTakeDown(struct _WsbmBufferPool *pool) 1274 { 1275 pool->takeDown(pool); 1276 1277 } 1278 1279 unsigned long 1280 wsbmBOSize(struct _WsbmBufferObject *buf) 1281 { 1282 unsigned long size; 1283 struct _WsbmBufStorage *storage; 1284 1285 storage = buf->storage; 1286 size = storage->pool->size(storage); 1287 1288 return size; 1289 1290 } 1291 1292 struct _ValidateList * 1293 wsbmGetKernelValidateList(struct _WsbmBufferList *list) 1294 { 1295 return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL; 1296 } 1297 1298 struct _ValidateList * 1299 wsbmGetUserValidateList(struct _WsbmBufferList *list) 1300 { 1301 return &list->userBuffers; 1302 } 1303 1304 struct _ValidateNode * 1305 validateListNode(void *iterator) 1306 { 1307 struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; 1308 1309 return WSBMLISTENTRY(l, struct _ValidateNode, head); 1310 } 1311 1312 void * 1313 validateListIterator(struct _ValidateList *list) 1314 { 1315 void *ret = list->list.next; 1316 1317 if (ret == &list->list) 1318 return NULL; 1319 return ret; 1320 } 1321 1322 void * 1323 validateListNext(struct _ValidateList *list, void *iterator) 1324 { 1325 void *ret; 1326 1327 struct _WsbmListHead *l = (struct _WsbmListHead *)iterator; 1328 1329 ret = l->next; 1330 if (ret == &list->list) 1331 return NULL; 1332 return ret; 1333 } 1334 1335 uint32_t 1336 wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf) 1337 { 1338 return kBuf->handle; 1339 } 1340 1341 extern void 1342 wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf, 1343 uint64_t gpuOffset, uint32_t placement, 1344 uint32_t fence_type_mask) 1345 { 1346 kBuf->gpuOffset = gpuOffset; 1347 kBuf->placement = placement; 1348 kBuf->fence_type_mask = fence_type_mask; 1349 } 1350 1351 extern struct _WsbmKernelBuf * 1352 wsbmKBuf(const struct _WsbmBufferObject *buf) 1353 { 1354 struct _WsbmBufStorage *storage = buf->storage; 1355 1356 return storage->pool->kernel(storage); 1357 } 1358
