1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */ 2 3 /* 4 * Copyright (C) 2013 Rob Clark <robclark (at) freedesktop.org> 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Rob Clark <robclark (at) freedesktop.org> 27 */ 28 29 #ifdef HAVE_CONFIG_H 30 # include <config.h> 31 #endif 32 33 #include <assert.h> 34 #include <inttypes.h> 35 36 #include "freedreno_ringbuffer.h" 37 #include "msm_priv.h" 38 39 /* represents a single cmd buffer in the submit ioctl. Each cmd buffer has 40 * a backing bo, and a reloc table. 41 */ 42 struct msm_cmd { 43 struct list_head list; 44 45 struct fd_ringbuffer *ring; 46 struct fd_bo *ring_bo; 47 48 /* reloc's table: */ 49 struct drm_msm_gem_submit_reloc *relocs; 50 uint32_t nr_relocs, max_relocs; 51 52 uint32_t size; 53 }; 54 55 struct msm_ringbuffer { 56 struct fd_ringbuffer base; 57 58 /* submit ioctl related tables: 59 * Note that bos and cmds are tracked by the parent ringbuffer, since 60 * that is global to the submit ioctl call. The reloc's table is tracked 61 * per cmd-buffer. 62 */ 63 struct { 64 /* bo's table: */ 65 struct drm_msm_gem_submit_bo *bos; 66 uint32_t nr_bos, max_bos; 67 68 /* cmd's table: */ 69 struct drm_msm_gem_submit_cmd *cmds; 70 uint32_t nr_cmds, max_cmds; 71 } submit; 72 73 /* should have matching entries in submit.bos: */ 74 /* Note, only in parent ringbuffer */ 75 struct fd_bo **bos; 76 uint32_t nr_bos, max_bos; 77 78 /* should have matching entries in submit.cmds: */ 79 struct msm_cmd **cmds; 80 uint32_t nr_cmds, max_cmds; 81 82 /* List of physical cmdstream buffers (msm_cmd) assocated with this 83 * logical fd_ringbuffer. 84 * 85 * Note that this is different from msm_ringbuffer::cmds (which 86 * shadows msm_ringbuffer::submit::cmds for tracking submit ioctl 87 * related stuff, and *only* is tracked in the parent ringbuffer. 88 * And only has "completed" cmd buffers (ie. we already know the 89 * size) added via get_cmd(). 90 */ 91 struct list_head cmd_list; 92 93 int is_growable; 94 unsigned cmd_count; 95 96 unsigned seqno; 97 98 /* maps fd_bo to idx: */ 99 void *bo_table; 100 }; 101 102 static inline struct msm_ringbuffer * to_msm_ringbuffer(struct fd_ringbuffer *x) 103 { 104 return (struct msm_ringbuffer *)x; 105 } 106 107 #define INIT_SIZE 0x1000 108 109 static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER; 110 drm_private extern pthread_mutex_t table_lock; 111 112 static void ring_bo_del(struct fd_device *dev, struct fd_bo *bo) 113 { 114 int ret; 115 116 pthread_mutex_lock(&table_lock); 117 ret = fd_bo_cache_free(&to_msm_device(dev)->ring_cache, bo); 118 pthread_mutex_unlock(&table_lock); 119 120 if (ret == 0) 121 return; 122 123 fd_bo_del(bo); 124 } 125 126 static struct fd_bo * ring_bo_new(struct fd_device *dev, uint32_t size) 127 { 128 struct fd_bo *bo; 129 130 bo = fd_bo_cache_alloc(&to_msm_device(dev)->ring_cache, &size, 0); 131 if (bo) 132 return bo; 133 134 bo = fd_bo_new(dev, size, 0); 135 if (!bo) 136 return NULL; 137 138 /* keep ringbuffer bo's out of the normal bo cache: */ 139 bo->bo_reuse = FALSE; 140 141 return bo; 142 } 143 144 static void ring_cmd_del(struct msm_cmd *cmd) 145 { 146 if (cmd->ring_bo) 147 ring_bo_del(cmd->ring->pipe->dev, cmd->ring_bo); 148 list_del(&cmd->list); 149 to_msm_ringbuffer(cmd->ring)->cmd_count--; 150 free(cmd->relocs); 151 free(cmd); 152 } 153 154 static struct msm_cmd * ring_cmd_new(struct fd_ringbuffer *ring, uint32_t size) 155 { 156 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 157 struct msm_cmd *cmd = calloc(1, sizeof(*cmd)); 158 159 if (!cmd) 160 return NULL; 161 162 cmd->ring = ring; 163 cmd->ring_bo = ring_bo_new(ring->pipe->dev, size); 164 if (!cmd->ring_bo) 165 goto fail; 166 167 list_addtail(&cmd->list, &msm_ring->cmd_list); 168 msm_ring->cmd_count++; 169 170 return cmd; 171 172 fail: 173 ring_cmd_del(cmd); 174 return NULL; 175 } 176 177 static void *grow(void *ptr, uint32_t nr, uint32_t *max, uint32_t sz) 178 { 179 if ((nr + 1) > *max) { 180 if ((*max * 2) < (nr + 1)) 181 *max = nr + 5; 182 else 183 *max = *max * 2; 184 ptr = realloc(ptr, *max * sz); 185 } 186 return ptr; 187 } 188 189 #define APPEND(x, name) ({ \ 190 (x)->name = grow((x)->name, (x)->nr_ ## name, &(x)->max_ ## name, sizeof((x)->name[0])); \ 191 (x)->nr_ ## name ++; \ 192 }) 193 194 static struct msm_cmd *current_cmd(struct fd_ringbuffer *ring) 195 { 196 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 197 assert(!LIST_IS_EMPTY(&msm_ring->cmd_list)); 198 return LIST_LAST_ENTRY(&msm_ring->cmd_list, struct msm_cmd, list); 199 } 200 201 static uint32_t append_bo(struct fd_ringbuffer *ring, struct fd_bo *bo) 202 { 203 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 204 uint32_t idx; 205 206 idx = APPEND(&msm_ring->submit, bos); 207 idx = APPEND(msm_ring, bos); 208 209 msm_ring->submit.bos[idx].flags = 0; 210 msm_ring->submit.bos[idx].handle = bo->handle; 211 msm_ring->submit.bos[idx].presumed = to_msm_bo(bo)->presumed; 212 213 msm_ring->bos[idx] = fd_bo_ref(bo); 214 215 return idx; 216 } 217 218 /* add (if needed) bo, return idx: */ 219 static uint32_t bo2idx(struct fd_ringbuffer *ring, struct fd_bo *bo, uint32_t flags) 220 { 221 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 222 struct msm_bo *msm_bo = to_msm_bo(bo); 223 uint32_t idx; 224 pthread_mutex_lock(&idx_lock); 225 if (msm_bo->current_ring_seqno == msm_ring->seqno) { 226 idx = msm_bo->idx; 227 } else { 228 void *val; 229 230 if (!msm_ring->bo_table) 231 msm_ring->bo_table = drmHashCreate(); 232 233 if (!drmHashLookup(msm_ring->bo_table, bo->handle, &val)) { 234 /* found */ 235 idx = (uint32_t)(uintptr_t)val; 236 } else { 237 idx = append_bo(ring, bo); 238 val = (void *)(uintptr_t)idx; 239 drmHashInsert(msm_ring->bo_table, bo->handle, val); 240 } 241 msm_bo->current_ring_seqno = msm_ring->seqno; 242 msm_bo->idx = idx; 243 } 244 pthread_mutex_unlock(&idx_lock); 245 if (flags & FD_RELOC_READ) 246 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_READ; 247 if (flags & FD_RELOC_WRITE) 248 msm_ring->submit.bos[idx].flags |= MSM_SUBMIT_BO_WRITE; 249 return idx; 250 } 251 252 static int check_cmd_bo(struct fd_ringbuffer *ring, 253 struct drm_msm_gem_submit_cmd *cmd, struct fd_bo *bo) 254 { 255 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 256 return msm_ring->submit.bos[cmd->submit_idx].handle == bo->handle; 257 } 258 259 /* Ensure that submit has corresponding entry in cmds table for the 260 * target cmdstream buffer: 261 */ 262 static void get_cmd(struct fd_ringbuffer *ring, struct msm_cmd *target_cmd, 263 uint32_t submit_offset, uint32_t size, uint32_t type) 264 { 265 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 266 struct drm_msm_gem_submit_cmd *cmd; 267 uint32_t i; 268 269 /* figure out if we already have a cmd buf: */ 270 for (i = 0; i < msm_ring->submit.nr_cmds; i++) { 271 cmd = &msm_ring->submit.cmds[i]; 272 if ((cmd->submit_offset == submit_offset) && 273 (cmd->size == size) && 274 (cmd->type == type) && 275 check_cmd_bo(ring, cmd, target_cmd->ring_bo)) 276 return; 277 } 278 279 /* create cmd buf if not: */ 280 i = APPEND(&msm_ring->submit, cmds); 281 APPEND(msm_ring, cmds); 282 msm_ring->cmds[i] = target_cmd; 283 cmd = &msm_ring->submit.cmds[i]; 284 cmd->type = type; 285 cmd->submit_idx = bo2idx(ring, target_cmd->ring_bo, FD_RELOC_READ); 286 cmd->submit_offset = submit_offset; 287 cmd->size = size; 288 cmd->pad = 0; 289 290 target_cmd->size = size; 291 } 292 293 static void * msm_ringbuffer_hostptr(struct fd_ringbuffer *ring) 294 { 295 return fd_bo_map(current_cmd(ring)->ring_bo); 296 } 297 298 static uint32_t find_next_reloc_idx(struct msm_cmd *msm_cmd, 299 uint32_t start, uint32_t offset) 300 { 301 uint32_t i; 302 303 /* a binary search would be more clever.. */ 304 for (i = start; i < msm_cmd->nr_relocs; i++) { 305 struct drm_msm_gem_submit_reloc *reloc = &msm_cmd->relocs[i]; 306 if (reloc->submit_offset >= offset) 307 return i; 308 } 309 310 return i; 311 } 312 313 static void delete_cmds(struct msm_ringbuffer *msm_ring) 314 { 315 struct msm_cmd *cmd, *tmp; 316 317 LIST_FOR_EACH_ENTRY_SAFE(cmd, tmp, &msm_ring->cmd_list, list) { 318 ring_cmd_del(cmd); 319 } 320 } 321 322 static void flush_reset(struct fd_ringbuffer *ring) 323 { 324 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 325 unsigned i; 326 327 for (i = 0; i < msm_ring->nr_bos; i++) { 328 struct msm_bo *msm_bo = to_msm_bo(msm_ring->bos[i]); 329 msm_bo->current_ring_seqno = 0; 330 fd_bo_del(&msm_bo->base); 331 } 332 333 /* for each of the cmd buffers, clear their reloc's: */ 334 for (i = 0; i < msm_ring->submit.nr_cmds; i++) { 335 struct msm_cmd *target_cmd = msm_ring->cmds[i]; 336 target_cmd->nr_relocs = 0; 337 } 338 339 msm_ring->submit.nr_cmds = 0; 340 msm_ring->submit.nr_bos = 0; 341 msm_ring->nr_cmds = 0; 342 msm_ring->nr_bos = 0; 343 344 if (msm_ring->bo_table) { 345 drmHashDestroy(msm_ring->bo_table); 346 msm_ring->bo_table = NULL; 347 } 348 349 if (msm_ring->is_growable) { 350 delete_cmds(msm_ring); 351 } else { 352 /* in old mode, just reset the # of relocs: */ 353 current_cmd(ring)->nr_relocs = 0; 354 } 355 } 356 357 static void finalize_current_cmd(struct fd_ringbuffer *ring, uint32_t *last_start) 358 { 359 uint32_t submit_offset, size, type; 360 struct fd_ringbuffer *parent; 361 362 if (ring->parent) { 363 parent = ring->parent; 364 type = MSM_SUBMIT_CMD_IB_TARGET_BUF; 365 } else { 366 parent = ring; 367 type = MSM_SUBMIT_CMD_BUF; 368 } 369 370 submit_offset = offset_bytes(last_start, ring->start); 371 size = offset_bytes(ring->cur, last_start); 372 373 get_cmd(parent, current_cmd(ring), submit_offset, size, type); 374 } 375 376 static void dump_submit(struct msm_ringbuffer *msm_ring) 377 { 378 uint32_t i, j; 379 380 for (i = 0; i < msm_ring->submit.nr_bos; i++) { 381 struct drm_msm_gem_submit_bo *bo = &msm_ring->submit.bos[i]; 382 ERROR_MSG(" bos[%d]: handle=%u, flags=%x", i, bo->handle, bo->flags); 383 } 384 for (i = 0; i < msm_ring->submit.nr_cmds; i++) { 385 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i]; 386 struct drm_msm_gem_submit_reloc *relocs = U642VOID(cmd->relocs); 387 ERROR_MSG(" cmd[%d]: type=%u, submit_idx=%u, submit_offset=%u, size=%u", 388 i, cmd->type, cmd->submit_idx, cmd->submit_offset, cmd->size); 389 for (j = 0; j < cmd->nr_relocs; j++) { 390 struct drm_msm_gem_submit_reloc *r = &relocs[j]; 391 ERROR_MSG(" reloc[%d]: submit_offset=%u, or=%08x, shift=%d, reloc_idx=%u" 392 ", reloc_offset=%"PRIu64, j, r->submit_offset, r->or, r->shift, 393 r->reloc_idx, r->reloc_offset); 394 } 395 } 396 } 397 398 static int msm_ringbuffer_flush(struct fd_ringbuffer *ring, uint32_t *last_start, 399 int in_fence_fd, int *out_fence_fd) 400 { 401 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 402 struct drm_msm_gem_submit req = { 403 .flags = to_msm_pipe(ring->pipe)->pipe, 404 }; 405 uint32_t i; 406 int ret; 407 408 if (in_fence_fd != -1) { 409 req.flags |= MSM_SUBMIT_FENCE_FD_IN | MSM_SUBMIT_NO_IMPLICIT; 410 req.fence_fd = in_fence_fd; 411 } 412 413 if (out_fence_fd) { 414 req.flags |= MSM_SUBMIT_FENCE_FD_OUT; 415 } 416 417 finalize_current_cmd(ring, last_start); 418 419 /* needs to be after get_cmd() as that could create bos/cmds table: */ 420 req.bos = VOID2U64(msm_ring->submit.bos), 421 req.nr_bos = msm_ring->submit.nr_bos; 422 req.cmds = VOID2U64(msm_ring->submit.cmds), 423 req.nr_cmds = msm_ring->submit.nr_cmds; 424 425 /* for each of the cmd's fix up their reloc's: */ 426 for (i = 0; i < msm_ring->submit.nr_cmds; i++) { 427 struct drm_msm_gem_submit_cmd *cmd = &msm_ring->submit.cmds[i]; 428 struct msm_cmd *msm_cmd = msm_ring->cmds[i]; 429 uint32_t a = find_next_reloc_idx(msm_cmd, 0, cmd->submit_offset); 430 uint32_t b = find_next_reloc_idx(msm_cmd, a, cmd->submit_offset + cmd->size); 431 cmd->relocs = VOID2U64(&msm_cmd->relocs[a]); 432 cmd->nr_relocs = (b > a) ? b - a : 0; 433 } 434 435 DEBUG_MSG("nr_cmds=%u, nr_bos=%u", req.nr_cmds, req.nr_bos); 436 437 ret = drmCommandWriteRead(ring->pipe->dev->fd, DRM_MSM_GEM_SUBMIT, 438 &req, sizeof(req)); 439 if (ret) { 440 ERROR_MSG("submit failed: %d (%s)", ret, strerror(errno)); 441 dump_submit(msm_ring); 442 } else if (!ret) { 443 /* update timestamp on all rings associated with submit: */ 444 for (i = 0; i < msm_ring->submit.nr_cmds; i++) { 445 struct msm_cmd *msm_cmd = msm_ring->cmds[i]; 446 msm_cmd->ring->last_timestamp = req.fence; 447 } 448 449 if (out_fence_fd) { 450 *out_fence_fd = req.fence_fd; 451 } 452 } 453 454 flush_reset(ring); 455 456 return ret; 457 } 458 459 static void msm_ringbuffer_grow(struct fd_ringbuffer *ring, uint32_t size) 460 { 461 assert(to_msm_ringbuffer(ring)->is_growable); 462 finalize_current_cmd(ring, ring->last_start); 463 ring_cmd_new(ring, size); 464 } 465 466 static void msm_ringbuffer_reset(struct fd_ringbuffer *ring) 467 { 468 flush_reset(ring); 469 } 470 471 static void msm_ringbuffer_emit_reloc(struct fd_ringbuffer *ring, 472 const struct fd_reloc *r) 473 { 474 struct fd_ringbuffer *parent = ring->parent ? ring->parent : ring; 475 struct msm_bo *msm_bo = to_msm_bo(r->bo); 476 struct drm_msm_gem_submit_reloc *reloc; 477 struct msm_cmd *cmd = current_cmd(ring); 478 uint32_t idx = APPEND(cmd, relocs); 479 uint32_t addr; 480 481 reloc = &cmd->relocs[idx]; 482 483 reloc->reloc_idx = bo2idx(parent, r->bo, r->flags); 484 reloc->reloc_offset = r->offset; 485 reloc->or = r->or; 486 reloc->shift = r->shift; 487 reloc->submit_offset = offset_bytes(ring->cur, ring->start); 488 489 addr = msm_bo->presumed; 490 if (reloc->shift < 0) 491 addr >>= -reloc->shift; 492 else 493 addr <<= reloc->shift; 494 (*ring->cur++) = addr | r->or; 495 496 if (ring->pipe->gpu_id >= 500) { 497 struct drm_msm_gem_submit_reloc *reloc_hi; 498 499 idx = APPEND(cmd, relocs); 500 501 reloc_hi = &cmd->relocs[idx]; 502 503 reloc_hi->reloc_idx = reloc->reloc_idx; 504 reloc_hi->reloc_offset = r->offset; 505 reloc_hi->or = r->orhi; 506 reloc_hi->shift = r->shift - 32; 507 reloc_hi->submit_offset = offset_bytes(ring->cur, ring->start); 508 509 addr = msm_bo->presumed >> 32; 510 if (reloc_hi->shift < 0) 511 addr >>= -reloc_hi->shift; 512 else 513 addr <<= reloc_hi->shift; 514 (*ring->cur++) = addr | r->orhi; 515 } 516 } 517 518 static uint32_t msm_ringbuffer_emit_reloc_ring(struct fd_ringbuffer *ring, 519 struct fd_ringbuffer *target, uint32_t cmd_idx, 520 uint32_t submit_offset, uint32_t size) 521 { 522 struct msm_cmd *cmd = NULL; 523 uint32_t idx = 0; 524 525 LIST_FOR_EACH_ENTRY(cmd, &to_msm_ringbuffer(target)->cmd_list, list) { 526 if (idx == cmd_idx) 527 break; 528 idx++; 529 } 530 531 assert(cmd && (idx == cmd_idx)); 532 533 if (idx < (to_msm_ringbuffer(target)->cmd_count - 1)) { 534 /* All but the last cmd buffer is fully "baked" (ie. already has 535 * done get_cmd() to add it to the cmds table). But in this case, 536 * the size we get is invalid (since it is calculated from the 537 * last cmd buffer): 538 */ 539 size = cmd->size; 540 } else { 541 get_cmd(ring, cmd, submit_offset, size, MSM_SUBMIT_CMD_IB_TARGET_BUF); 542 } 543 544 msm_ringbuffer_emit_reloc(ring, &(struct fd_reloc){ 545 .bo = cmd->ring_bo, 546 .flags = FD_RELOC_READ, 547 .offset = submit_offset, 548 }); 549 550 return size; 551 } 552 553 static uint32_t msm_ringbuffer_cmd_count(struct fd_ringbuffer *ring) 554 { 555 return to_msm_ringbuffer(ring)->cmd_count; 556 } 557 558 static void msm_ringbuffer_destroy(struct fd_ringbuffer *ring) 559 { 560 struct msm_ringbuffer *msm_ring = to_msm_ringbuffer(ring); 561 562 flush_reset(ring); 563 delete_cmds(msm_ring); 564 565 free(msm_ring->submit.cmds); 566 free(msm_ring->submit.bos); 567 free(msm_ring->bos); 568 free(msm_ring->cmds); 569 free(msm_ring); 570 } 571 572 static const struct fd_ringbuffer_funcs funcs = { 573 .hostptr = msm_ringbuffer_hostptr, 574 .flush = msm_ringbuffer_flush, 575 .grow = msm_ringbuffer_grow, 576 .reset = msm_ringbuffer_reset, 577 .emit_reloc = msm_ringbuffer_emit_reloc, 578 .emit_reloc_ring = msm_ringbuffer_emit_reloc_ring, 579 .cmd_count = msm_ringbuffer_cmd_count, 580 .destroy = msm_ringbuffer_destroy, 581 }; 582 583 drm_private struct fd_ringbuffer * msm_ringbuffer_new(struct fd_pipe *pipe, 584 uint32_t size) 585 { 586 struct msm_ringbuffer *msm_ring; 587 struct fd_ringbuffer *ring = NULL; 588 589 msm_ring = calloc(1, sizeof(*msm_ring)); 590 if (!msm_ring) { 591 ERROR_MSG("allocation failed"); 592 goto fail; 593 } 594 595 if (size == 0) { 596 assert(pipe->dev->version >= FD_VERSION_UNLIMITED_CMDS); 597 size = INIT_SIZE; 598 msm_ring->is_growable = TRUE; 599 } 600 601 list_inithead(&msm_ring->cmd_list); 602 msm_ring->seqno = ++to_msm_device(pipe->dev)->ring_cnt; 603 604 ring = &msm_ring->base; 605 ring->funcs = &funcs; 606 ring->size = size; 607 ring->pipe = pipe; /* needed in ring_cmd_new() */ 608 609 ring_cmd_new(ring, size); 610 611 return ring; 612 fail: 613 if (ring) 614 fd_ringbuffer_del(ring); 615 return NULL; 616 } 617
