1 /************************************************************************** 2 * 3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, Tx., USA 4 * All Rights Reserved. 5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA 6 * All Rights Reserved. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the 10 * "Software"), to deal in the Software without restriction, including 11 * without limitation the rights to use, copy, modify, merge, publish, 12 * distribute, sub license, and/or sell copies of the Software, and to 13 * permit persons to whom the Software is furnished to do so, subject to 14 * the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the 17 * next paragraph) shall be included in all copies or substantial portions 18 * of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 26 * USE OR OTHER DEALINGS IN THE SOFTWARE. 27 * 28 **************************************************************************/ 29 /* 30 * Authors: Thomas Hellstrm <thomas-at-tungstengraphics-dot-com> 31 */ 32 #ifdef HAVE_CONFIG_H 33 #include "config.h" 34 #endif 35 36 #include <xf86drm.h> 37 #include <stdlib.h> 38 #include <unistd.h> 39 #include <errno.h> 40 #include <sys/mman.h> 41 #include <drm/psb_ttm_placement_user.h> 42 #include "wsbm_pool.h" 43 #include "assert.h" 44 #include "wsbm_priv.h" 45 #include "wsbm_manager.h" 46 47 #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \ 48 do { \ 49 (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \ 50 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ 51 52 #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \ 53 do { \ 54 (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \ 55 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \ 56 57 /* 58 * Buffer pool implementation using DRM buffer objects as wsbm buffer objects. 59 */ 60 61 struct _TTMBuffer 62 { 63 struct _WsbmBufStorage buf; 64 struct _WsbmCond event; 65 66 /* 67 * Remains constant after creation. 68 */ 69 70 uint64_t requestedSize; 71 uint64_t mapHandle; 72 uint64_t realSize; 73 74 /* 75 * Protected by the kernel lock. 76 */ 77 78 struct _WsbmKernelBuf kBuf; 79 80 /* 81 * Protected by the mutex. 82 */ 83 84 void *virtual; 85 int syncInProgress; 86 unsigned readers; 87 unsigned writers; 88 }; 89 90 struct _TTMPool 91 { 92 struct _WsbmBufferPool pool; 93 unsigned int pageSize; 94 unsigned int devOffset; 95 }; 96 97 static inline struct _TTMPool * 98 ttmGetPool(struct _TTMBuffer *dBuf) 99 { 100 return containerOf(dBuf->buf.pool, struct _TTMPool, pool); 101 } 102 103 static inline struct _TTMBuffer * 104 ttmBuffer(struct _WsbmBufStorage *buf) 105 { 106 return containerOf(buf, struct _TTMBuffer, buf); 107 } 108 109 static struct _WsbmBufStorage * 110 pool_create(struct _WsbmBufferPool *pool, 111 unsigned long size, uint32_t placement, unsigned alignment) 112 { 113 struct _TTMBuffer *dBuf = (struct _TTMBuffer *) 114 calloc(1, sizeof(*dBuf)); 115 struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); 116 int ret; 117 unsigned pageSize = ttmPool->pageSize; 118 union ttm_pl_create_arg arg; 119 120 if (!dBuf) 121 return NULL; 122 123 if ((alignment > pageSize) && (alignment % pageSize)) 124 goto out_err0; 125 126 ret = wsbmBufStorageInit(&dBuf->buf, pool); 127 if (ret) 128 goto out_err0; 129 130 ret = WSBM_COND_INIT(&dBuf->event); 131 if (ret) 132 goto out_err1; 133 134 arg.req.size = size; 135 arg.req.placement = placement; 136 arg.req.page_alignment = alignment / pageSize; 137 138 DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE, 139 arg, ret); 140 141 if (ret) 142 goto out_err2; 143 144 dBuf->requestedSize = size; 145 dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; 146 dBuf->mapHandle = arg.rep.map_handle; 147 dBuf->realSize = arg.rep.bo_size; 148 dBuf->kBuf.placement = arg.rep.placement; 149 dBuf->kBuf.handle = arg.rep.handle; 150 151 return &dBuf->buf; 152 153 out_err2: 154 WSBM_COND_FREE(&dBuf->event); 155 out_err1: 156 wsbmBufStorageTakedown(&dBuf->buf); 157 out_err0: 158 free(dBuf); 159 return NULL; 160 } 161 162 static struct _WsbmBufStorage * 163 pool_reference(struct _WsbmBufferPool *pool, unsigned handle) 164 { 165 struct _TTMBuffer *dBuf = (struct _TTMBuffer *)calloc(1, sizeof(*dBuf)); 166 struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); 167 union ttm_pl_reference_arg arg; 168 int ret; 169 170 if (!dBuf) 171 return NULL; 172 173 ret = wsbmBufStorageInit(&dBuf->buf, pool); 174 if (ret) 175 goto out_err0; 176 177 ret = WSBM_COND_INIT(&dBuf->event); 178 if (ret) 179 goto out_err1; 180 181 arg.req.handle = handle; 182 ret = drmCommandWriteRead(pool->fd, ttmPool->devOffset + TTM_PL_REFERENCE, 183 &arg, sizeof(arg)); 184 185 if (ret) 186 goto out_err2; 187 188 dBuf->requestedSize = arg.rep.bo_size; 189 dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; 190 dBuf->mapHandle = arg.rep.map_handle; 191 dBuf->realSize = arg.rep.bo_size; 192 dBuf->kBuf.placement = arg.rep.placement; 193 dBuf->kBuf.handle = arg.rep.handle; 194 dBuf->kBuf.fence_type_mask = arg.rep.sync_object_arg; 195 196 return &dBuf->buf; 197 198 out_err2: 199 WSBM_COND_FREE(&dBuf->event); 200 out_err1: 201 wsbmBufStorageTakedown(&dBuf->buf); 202 out_err0: 203 free(dBuf); 204 return NULL; 205 } 206 207 static void 208 pool_destroy(struct _WsbmBufStorage **buf) 209 { 210 struct _TTMBuffer *dBuf = ttmBuffer(*buf); 211 struct _TTMPool *ttmPool = ttmGetPool(dBuf); 212 struct ttm_pl_reference_req arg; 213 214 *buf = NULL; 215 if (dBuf->virtual != NULL) { 216 (void)munmap(dBuf->virtual, dBuf->requestedSize); 217 dBuf->virtual = NULL; 218 } 219 arg.handle = dBuf->kBuf.handle; 220 (void)drmCommandWrite(dBuf->buf.pool->fd, 221 ttmPool->devOffset + TTM_PL_UNREF, 222 &arg, sizeof(arg)); 223 224 WSBM_COND_FREE(&dBuf->event); 225 wsbmBufStorageTakedown(&dBuf->buf); 226 free(dBuf); 227 } 228 229 static int 230 syncforcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) 231 { 232 uint32_t kmode = 0; 233 struct _TTMBuffer *dBuf = ttmBuffer(buf); 234 struct _TTMPool *ttmPool = ttmGetPool(dBuf); 235 unsigned int readers; 236 unsigned int writers; 237 int ret = 0; 238 239 while (dBuf->syncInProgress) 240 WSBM_COND_WAIT(&dBuf->event, &buf->mutex); 241 242 readers = dBuf->readers; 243 writers = dBuf->writers; 244 245 if ((mode & WSBM_SYNCCPU_READ) && (++dBuf->readers == 1)) 246 kmode |= TTM_PL_SYNCCPU_MODE_READ; 247 248 if ((mode & WSBM_SYNCCPU_WRITE) && (++dBuf->writers == 1)) 249 kmode |= TTM_PL_SYNCCPU_MODE_WRITE; 250 251 if (kmode) { 252 struct ttm_pl_synccpu_arg arg; 253 254 if (mode & WSBM_SYNCCPU_DONT_BLOCK) 255 kmode |= TTM_PL_SYNCCPU_MODE_NO_BLOCK; 256 257 dBuf->syncInProgress = 1; 258 259 /* 260 * This might be a lengthy wait, so 261 * release the mutex. 262 */ 263 264 WSBM_MUTEX_UNLOCK(&buf->mutex); 265 266 arg.handle = dBuf->kBuf.handle; 267 arg.access_mode = kmode; 268 arg.op = TTM_PL_SYNCCPU_OP_GRAB; 269 270 DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, 271 ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); 272 273 WSBM_MUTEX_LOCK(&buf->mutex); 274 dBuf->syncInProgress = 0; 275 WSBM_COND_BROADCAST(&dBuf->event); 276 277 if (ret) { 278 dBuf->readers = readers; 279 dBuf->writers = writers; 280 } 281 } 282 283 return ret; 284 } 285 286 static int 287 releasefromcpu_locked(struct _WsbmBufStorage *buf, unsigned mode) 288 { 289 uint32_t kmode = 0; 290 struct _TTMBuffer *dBuf = ttmBuffer(buf); 291 struct _TTMPool *ttmPool = ttmGetPool(dBuf); 292 int ret = 0; 293 294 while (dBuf->syncInProgress) 295 WSBM_COND_WAIT(&dBuf->event, &buf->mutex); 296 297 if ((mode & WSBM_SYNCCPU_READ) && (--dBuf->readers == 0)) 298 kmode |= TTM_PL_SYNCCPU_MODE_READ; 299 300 if ((mode & WSBM_SYNCCPU_WRITE) && (--dBuf->writers == 0)) 301 kmode |= TTM_PL_SYNCCPU_MODE_WRITE; 302 303 if (kmode) { 304 struct ttm_pl_synccpu_arg arg; 305 306 arg.handle = dBuf->kBuf.handle; 307 arg.access_mode = kmode; 308 arg.op = TTM_PL_SYNCCPU_OP_RELEASE; 309 310 DRMRESTARTCOMMANDWRITE(dBuf->buf.pool->fd, 311 ttmPool->devOffset + TTM_PL_SYNCCPU, arg, ret); 312 313 } 314 315 return ret; 316 } 317 318 static int 319 pool_syncforcpu(struct _WsbmBufStorage *buf, unsigned mode) 320 { 321 int ret; 322 323 WSBM_MUTEX_LOCK(&buf->mutex); 324 ret = syncforcpu_locked(buf, mode); 325 WSBM_MUTEX_UNLOCK(&buf->mutex); 326 return ret; 327 } 328 329 static void 330 pool_releasefromcpu(struct _WsbmBufStorage *buf, unsigned mode) 331 { 332 WSBM_MUTEX_LOCK(&buf->mutex); 333 (void)releasefromcpu_locked(buf, mode); 334 WSBM_MUTEX_UNLOCK(&buf->mutex); 335 } 336 337 #ifdef ANDROID 338 339 /* No header but syscall provided by bionic */ 340 void* __mmap2(void*, size_t, int, int, int, size_t); 341 #define MMAP2_SHIFT 12 // 2**12 == 4096 342 343 static void* _temp_mmap(void *addr, size_t size, int prot, int flags, int fd, long long offset) 344 { 345 return __mmap2(addr, size, prot, flags, fd, (unsigned long)(offset >> MMAP2_SHIFT)); 346 } 347 348 #endif 349 350 static int 351 pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual) 352 { 353 struct _TTMBuffer *dBuf = ttmBuffer(buf); 354 void *virt; 355 int ret = 0; 356 357 WSBM_MUTEX_LOCK(&buf->mutex); 358 359 /* 360 * mmaps are expensive, so we only really unmap if 361 * we destroy the buffer. 362 */ 363 364 if (dBuf->virtual == NULL) { 365 #if defined(__LP64__) || defined(_LP64) || defined(__LP64) 366 virt = mmap(0, dBuf->requestedSize, 367 PROT_READ | PROT_WRITE, MAP_SHARED, 368 buf->pool->fd, dBuf->mapHandle); 369 #else 370 virt = _temp_mmap(0, dBuf->requestedSize, 371 PROT_READ | PROT_WRITE, MAP_SHARED, 372 buf->pool->fd, dBuf->mapHandle); 373 #endif 374 if (virt == MAP_FAILED) { 375 ret = -errno; 376 goto out_unlock; 377 } 378 dBuf->virtual = virt; 379 } 380 381 *virtual = dBuf->virtual; 382 out_unlock: 383 384 WSBM_MUTEX_UNLOCK(&buf->mutex); 385 386 return ret; 387 } 388 389 static void 390 pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused))) 391 { 392 ; 393 } 394 395 static unsigned long 396 pool_offset(struct _WsbmBufStorage *buf) 397 { 398 struct _TTMBuffer *dBuf = ttmBuffer(buf); 399 400 return dBuf->kBuf.gpuOffset; 401 } 402 403 static unsigned long 404 pool_poolOffset(struct _WsbmBufStorage *buf __attribute__ ((unused))) 405 { 406 return 0; 407 } 408 409 static uint32_t 410 pool_placement(struct _WsbmBufStorage *buf) 411 { 412 struct _TTMBuffer *dBuf = ttmBuffer(buf); 413 414 return dBuf->kBuf.placement; 415 } 416 417 static unsigned long 418 pool_size(struct _WsbmBufStorage *buf) 419 { 420 struct _TTMBuffer *dBuf = ttmBuffer(buf); 421 422 return dBuf->realSize; 423 } 424 425 static void 426 pool_fence(struct _WsbmBufStorage *buf __attribute__ ((unused)), 427 struct _WsbmFenceObject *fence __attribute__ ((unused))) 428 { 429 /* 430 * Noop. The kernel handles all fencing. 431 */ 432 } 433 434 static int 435 pool_waitIdle(struct _WsbmBufStorage *buf, int lazy) 436 { 437 struct _TTMBuffer *dBuf = ttmBuffer(buf); 438 struct _TTMPool *ttmPool = ttmGetPool(dBuf); 439 struct ttm_pl_waitidle_arg req; 440 struct _WsbmBufferPool *pool = buf->pool; 441 int ret; 442 443 req.handle = dBuf->kBuf.handle; 444 req.mode = (lazy) ? TTM_PL_WAITIDLE_MODE_LAZY : 0; 445 446 DRMRESTARTCOMMANDWRITE(pool->fd, ttmPool->devOffset + TTM_PL_WAITIDLE, 447 req, ret); 448 449 return ret; 450 } 451 452 static void 453 pool_takedown(struct _WsbmBufferPool *pool) 454 { 455 struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); 456 457 free(ttmPool); 458 } 459 460 static int 461 pool_setStatus(struct _WsbmBufStorage *buf, uint32_t set_placement, 462 uint32_t clr_placement) 463 { 464 struct _TTMBuffer *dBuf = ttmBuffer(buf); 465 struct _TTMPool *ttmPool = ttmGetPool(dBuf); 466 union ttm_pl_setstatus_arg arg; 467 struct ttm_pl_setstatus_req *req = &arg.req; 468 struct ttm_pl_rep *rep = &arg.rep; 469 struct _WsbmBufferPool *pool = buf->pool; 470 int ret; 471 472 req->handle = dBuf->kBuf.handle; 473 req->set_placement = set_placement; 474 req->clr_placement = clr_placement; 475 476 DRMRESTARTCOMMANDWRITEREAD(pool->fd, 477 ttmPool->devOffset + TTM_PL_SETSTATUS, 478 arg, ret); 479 480 if (!ret) { 481 dBuf->kBuf.gpuOffset = rep->gpu_offset; 482 dBuf->kBuf.placement = rep->placement; 483 } 484 485 return ret; 486 } 487 488 static struct _WsbmKernelBuf * 489 pool_kernel(struct _WsbmBufStorage *buf) 490 { 491 return (void *)&ttmBuffer(buf)->kBuf; 492 } 493 494 struct _WsbmBufferPool * 495 wsbmTTMPoolInit(int fd, unsigned int devOffset) 496 { 497 struct _TTMPool *ttmPool; 498 struct _WsbmBufferPool *pool; 499 500 ttmPool = (struct _TTMPool *)calloc(1, sizeof(*ttmPool)); 501 502 if (!ttmPool) 503 return NULL; 504 505 ttmPool->pageSize = getpagesize(); 506 ttmPool->devOffset = devOffset; 507 pool = &ttmPool->pool; 508 509 pool->fd = fd; 510 pool->map = &pool_map; 511 pool->unmap = &pool_unmap; 512 pool->syncforcpu = &pool_syncforcpu; 513 pool->releasefromcpu = &pool_releasefromcpu; 514 pool->destroy = &pool_destroy; 515 pool->offset = &pool_offset; 516 pool->poolOffset = &pool_poolOffset; 517 pool->placement = &pool_placement; 518 pool->size = &pool_size; 519 pool->create = &pool_create; 520 pool->fence = &pool_fence; 521 pool->kernel = &pool_kernel; 522 pool->validate = NULL; 523 pool->unvalidate = NULL; 524 pool->waitIdle = &pool_waitIdle; 525 pool->takeDown = &pool_takedown; 526 pool->createByReference = &pool_reference; 527 pool->setStatus = &pool_setStatus; 528 return pool; 529 } 530 531 struct _WsbmBufStorage * 532 ttm_pool_ub_create(struct _WsbmBufferPool *pool, unsigned long size, uint32_t placement, unsigned alignment, const unsigned long *user_ptr, int fd) 533 { 534 struct _TTMBuffer *dBuf = (struct _TTMBuffer *) 535 calloc(1, sizeof(*dBuf)); 536 struct _TTMPool *ttmPool = containerOf(pool, struct _TTMPool, pool); 537 int ret; 538 unsigned pageSize = ttmPool->pageSize; 539 union ttm_pl_create_ub_arg arg; 540 541 if (!dBuf) 542 return NULL; 543 544 if ((alignment > pageSize) && (alignment % pageSize)) 545 goto out_err0; 546 547 ret = wsbmBufStorageInit(&dBuf->buf, pool); 548 if (ret) 549 goto out_err0; 550 551 ret = WSBM_COND_INIT(&dBuf->event); 552 if (ret) 553 goto out_err1; 554 555 arg.req.size = size; 556 arg.req.placement = placement; 557 arg.req.page_alignment = alignment / pageSize; 558 arg.req.user_address = (unsigned long)user_ptr; 559 arg.req.fd = fd; 560 561 DRMRESTARTCOMMANDWRITEREAD(pool->fd, ttmPool->devOffset + TTM_PL_CREATE_UB, 562 arg, ret); 563 if (ret) 564 goto out_err2; 565 566 dBuf->requestedSize = size; 567 dBuf->kBuf.gpuOffset = arg.rep.gpu_offset; 568 dBuf->mapHandle = arg.rep.map_handle; 569 dBuf->realSize = arg.rep.bo_size; 570 dBuf->kBuf.placement = arg.rep.placement; 571 dBuf->kBuf.handle = arg.rep.handle; 572 573 return &dBuf->buf; 574 575 out_err2: 576 WSBM_COND_FREE(&dBuf->event); 577 out_err1: 578 wsbmBufStorageTakedown(&dBuf->buf); 579 out_err0: 580 free(dBuf); 581 return NULL; 582 } 583 584
