1 /* 2 * Copyright 2008 Jrme Glisse 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 14 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 15 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 16 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS 17 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 20 * USE OR OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * The above copyright notice and this permission notice (including the 23 * next paragraph) shall be included in all copies or substantial portions 24 * of the Software. 25 */ 26 /* 27 * Authors: 28 * Aapo Tahkola <aet (at) rasterburn.org> 29 * Nicolai Haehnle <prefect_ (at) gmx.net> 30 * Jrme Glisse <glisse (at) freedesktop.org> 31 */ 32 #ifdef HAVE_CONFIG_H 33 #include "config.h" 34 #endif 35 #include <assert.h> 36 #include <errno.h> 37 #include <stdlib.h> 38 #include <string.h> 39 #include <pthread.h> 40 #include <sys/ioctl.h> 41 #include "radeon_cs.h" 42 #include "radeon_cs_int.h" 43 #include "radeon_bo_int.h" 44 #include "radeon_cs_gem.h" 45 #include "radeon_bo_gem.h" 46 #include "drm.h" 47 #include "libdrm_macros.h" 48 #include "xf86drm.h" 49 #include "xf86atomic.h" 50 #include "radeon_drm.h" 51 52 /* Add LIBDRM_RADEON_BOF_FILES to libdrm_radeon_la_SOURCES when building with BOF_DUMP */ 53 #define CS_BOF_DUMP 0 54 #if CS_BOF_DUMP 55 #include "bof.h" 56 #endif 57 58 struct radeon_cs_manager_gem { 59 struct radeon_cs_manager base; 60 uint32_t device_id; 61 unsigned nbof; 62 }; 63 64 #pragma pack(1) 65 struct cs_reloc_gem { 66 uint32_t handle; 67 uint32_t read_domain; 68 uint32_t write_domain; 69 uint32_t flags; 70 }; 71 72 #pragma pack() 73 #define RELOC_SIZE (sizeof(struct cs_reloc_gem) / sizeof(uint32_t)) 74 75 struct cs_gem { 76 struct radeon_cs_int base; 77 struct drm_radeon_cs cs; 78 struct drm_radeon_cs_chunk chunks[2]; 79 unsigned nrelocs; 80 uint32_t *relocs; 81 struct radeon_bo_int **relocs_bo; 82 }; 83 84 static pthread_mutex_t id_mutex = PTHREAD_MUTEX_INITIALIZER; 85 static uint32_t cs_id_source = 0; 86 87 /** 88 * result is undefined if called with ~0 89 */ 90 static uint32_t get_first_zero(const uint32_t n) 91 { 92 /* __builtin_ctz returns number of trailing zeros. */ 93 return 1 << __builtin_ctz(~n); 94 } 95 96 /** 97 * Returns a free id for cs. 98 * If there is no free id we return zero 99 **/ 100 static uint32_t generate_id(void) 101 { 102 uint32_t r = 0; 103 pthread_mutex_lock( &id_mutex ); 104 /* check for free ids */ 105 if (cs_id_source != ~r) { 106 /* find first zero bit */ 107 r = get_first_zero(cs_id_source); 108 109 /* set id as reserved */ 110 cs_id_source |= r; 111 } 112 pthread_mutex_unlock( &id_mutex ); 113 return r; 114 } 115 116 /** 117 * Free the id for later reuse 118 **/ 119 static void free_id(uint32_t id) 120 { 121 pthread_mutex_lock( &id_mutex ); 122 123 cs_id_source &= ~id; 124 125 pthread_mutex_unlock( &id_mutex ); 126 } 127 128 static struct radeon_cs_int *cs_gem_create(struct radeon_cs_manager *csm, 129 uint32_t ndw) 130 { 131 struct cs_gem *csg; 132 133 /* max cmd buffer size is 64Kb */ 134 if (ndw > (64 * 1024 / 4)) { 135 return NULL; 136 } 137 csg = (struct cs_gem*)calloc(1, sizeof(struct cs_gem)); 138 if (csg == NULL) { 139 return NULL; 140 } 141 csg->base.csm = csm; 142 csg->base.ndw = 64 * 1024 / 4; 143 csg->base.packets = (uint32_t*)calloc(1, 64 * 1024); 144 if (csg->base.packets == NULL) { 145 free(csg); 146 return NULL; 147 } 148 csg->base.relocs_total_size = 0; 149 csg->base.crelocs = 0; 150 csg->base.id = generate_id(); 151 csg->nrelocs = 4096 / (4 * 4) ; 152 csg->relocs_bo = (struct radeon_bo_int**)calloc(1, 153 csg->nrelocs*sizeof(void*)); 154 if (csg->relocs_bo == NULL) { 155 free(csg->base.packets); 156 free(csg); 157 return NULL; 158 } 159 csg->base.relocs = csg->relocs = (uint32_t*)calloc(1, 4096); 160 if (csg->relocs == NULL) { 161 free(csg->relocs_bo); 162 free(csg->base.packets); 163 free(csg); 164 return NULL; 165 } 166 csg->chunks[0].chunk_id = RADEON_CHUNK_ID_IB; 167 csg->chunks[0].length_dw = 0; 168 csg->chunks[0].chunk_data = (uint64_t)(uintptr_t)csg->base.packets; 169 csg->chunks[1].chunk_id = RADEON_CHUNK_ID_RELOCS; 170 csg->chunks[1].length_dw = 0; 171 csg->chunks[1].chunk_data = (uint64_t)(uintptr_t)csg->relocs; 172 return (struct radeon_cs_int*)csg; 173 } 174 175 static int cs_gem_write_reloc(struct radeon_cs_int *cs, 176 struct radeon_bo *bo, 177 uint32_t read_domain, 178 uint32_t write_domain, 179 uint32_t flags) 180 { 181 struct radeon_bo_int *boi = (struct radeon_bo_int *)bo; 182 struct cs_gem *csg = (struct cs_gem*)cs; 183 struct cs_reloc_gem *reloc; 184 uint32_t idx; 185 unsigned i; 186 187 assert(boi->space_accounted); 188 189 /* check domains */ 190 if ((read_domain && write_domain) || (!read_domain && !write_domain)) { 191 /* in one CS a bo can only be in read or write domain but not 192 * in read & write domain at the same sime 193 */ 194 return -EINVAL; 195 } 196 if (read_domain == RADEON_GEM_DOMAIN_CPU) { 197 return -EINVAL; 198 } 199 if (write_domain == RADEON_GEM_DOMAIN_CPU) { 200 return -EINVAL; 201 } 202 /* use bit field hash function to determine 203 if this bo is for sure not in this cs.*/ 204 if ((atomic_read((atomic_t *)radeon_gem_get_reloc_in_cs(bo)) & cs->id)) { 205 /* check if bo is already referenced. 206 * Scanning from end to begin reduces cycles with mesa because 207 * it often relocates same shared dma bo again. */ 208 for(i = cs->crelocs; i != 0;) { 209 --i; 210 idx = i * RELOC_SIZE; 211 reloc = (struct cs_reloc_gem*)&csg->relocs[idx]; 212 if (reloc->handle == bo->handle) { 213 /* Check domains must be in read or write. As we check already 214 * checked that in argument one of the read or write domain was 215 * set we only need to check that if previous reloc as the read 216 * domain set then the read_domain should also be set for this 217 * new relocation. 218 */ 219 /* the DDX expects to read and write from same pixmap */ 220 if (write_domain && (reloc->read_domain & write_domain)) { 221 reloc->read_domain = 0; 222 reloc->write_domain = write_domain; 223 } else if (read_domain & reloc->write_domain) { 224 reloc->read_domain = 0; 225 } else { 226 if (write_domain != reloc->write_domain) 227 return -EINVAL; 228 if (read_domain != reloc->read_domain) 229 return -EINVAL; 230 } 231 232 reloc->read_domain |= read_domain; 233 reloc->write_domain |= write_domain; 234 /* update flags */ 235 reloc->flags |= (flags & reloc->flags); 236 /* write relocation packet */ 237 radeon_cs_write_dword((struct radeon_cs *)cs, 0xc0001000); 238 radeon_cs_write_dword((struct radeon_cs *)cs, idx); 239 return 0; 240 } 241 } 242 } 243 /* new relocation */ 244 if (csg->base.crelocs >= csg->nrelocs) { 245 /* allocate more memory (TODO: should use a slab allocatore maybe) */ 246 uint32_t *tmp, size; 247 size = ((csg->nrelocs + 1) * sizeof(struct radeon_bo*)); 248 tmp = (uint32_t*)realloc(csg->relocs_bo, size); 249 if (tmp == NULL) { 250 return -ENOMEM; 251 } 252 csg->relocs_bo = (struct radeon_bo_int **)tmp; 253 size = ((csg->nrelocs + 1) * RELOC_SIZE * 4); 254 tmp = (uint32_t*)realloc(csg->relocs, size); 255 if (tmp == NULL) { 256 return -ENOMEM; 257 } 258 cs->relocs = csg->relocs = tmp; 259 csg->nrelocs += 1; 260 csg->chunks[1].chunk_data = (uint64_t)(uintptr_t)csg->relocs; 261 } 262 csg->relocs_bo[csg->base.crelocs] = boi; 263 idx = (csg->base.crelocs++) * RELOC_SIZE; 264 reloc = (struct cs_reloc_gem*)&csg->relocs[idx]; 265 reloc->handle = bo->handle; 266 reloc->read_domain = read_domain; 267 reloc->write_domain = write_domain; 268 reloc->flags = flags; 269 csg->chunks[1].length_dw += RELOC_SIZE; 270 radeon_bo_ref(bo); 271 /* bo might be referenced from another context so have to use atomic opertions */ 272 atomic_add((atomic_t *)radeon_gem_get_reloc_in_cs(bo), cs->id); 273 cs->relocs_total_size += boi->size; 274 radeon_cs_write_dword((struct radeon_cs *)cs, 0xc0001000); 275 radeon_cs_write_dword((struct radeon_cs *)cs, idx); 276 return 0; 277 } 278 279 static int cs_gem_begin(struct radeon_cs_int *cs, 280 uint32_t ndw, 281 const char *file, 282 const char *func, 283 int line) 284 { 285 286 if (cs->section_ndw) { 287 fprintf(stderr, "CS already in a section(%s,%s,%d)\n", 288 cs->section_file, cs->section_func, cs->section_line); 289 fprintf(stderr, "CS can't start section(%s,%s,%d)\n", 290 file, func, line); 291 return -EPIPE; 292 } 293 cs->section_ndw = ndw; 294 cs->section_cdw = 0; 295 cs->section_file = file; 296 cs->section_func = func; 297 cs->section_line = line; 298 299 if (cs->cdw + ndw > cs->ndw) { 300 uint32_t tmp, *ptr; 301 302 /* round up the required size to a multiple of 1024 */ 303 tmp = (cs->cdw + ndw + 0x3FF) & (~0x3FF); 304 ptr = (uint32_t*)realloc(cs->packets, 4 * tmp); 305 if (ptr == NULL) { 306 return -ENOMEM; 307 } 308 cs->packets = ptr; 309 cs->ndw = tmp; 310 } 311 return 0; 312 } 313 314 static int cs_gem_end(struct radeon_cs_int *cs, 315 const char *file, 316 const char *func, 317 int line) 318 319 { 320 if (!cs->section_ndw) { 321 fprintf(stderr, "CS no section to end at (%s,%s,%d)\n", 322 file, func, line); 323 return -EPIPE; 324 } 325 if (cs->section_ndw != cs->section_cdw) { 326 fprintf(stderr, "CS section size missmatch start at (%s,%s,%d) %d vs %d\n", 327 cs->section_file, cs->section_func, cs->section_line, cs->section_ndw, cs->section_cdw); 328 fprintf(stderr, "CS section end at (%s,%s,%d)\n", 329 file, func, line); 330 331 /* We must reset the section even when there is error. */ 332 cs->section_ndw = 0; 333 return -EPIPE; 334 } 335 cs->section_ndw = 0; 336 return 0; 337 } 338 339 #if CS_BOF_DUMP 340 static void cs_gem_dump_bof(struct radeon_cs_int *cs) 341 { 342 struct cs_gem *csg = (struct cs_gem*)cs; 343 struct radeon_cs_manager_gem *csm; 344 bof_t *bcs, *blob, *array, *bo, *size, *handle, *device_id, *root; 345 char tmp[256]; 346 unsigned i; 347 348 csm = (struct radeon_cs_manager_gem *)cs->csm; 349 root = device_id = bcs = blob = array = bo = size = handle = NULL; 350 root = bof_object(); 351 if (root == NULL) 352 goto out_err; 353 device_id = bof_int32(csm->device_id); 354 if (device_id == NULL) 355 return; 356 if (bof_object_set(root, "device_id", device_id)) 357 goto out_err; 358 bof_decref(device_id); 359 device_id = NULL; 360 /* dump relocs */ 361 blob = bof_blob(csg->nrelocs * 16, csg->relocs); 362 if (blob == NULL) 363 goto out_err; 364 if (bof_object_set(root, "reloc", blob)) 365 goto out_err; 366 bof_decref(blob); 367 blob = NULL; 368 /* dump cs */ 369 blob = bof_blob(cs->cdw * 4, cs->packets); 370 if (blob == NULL) 371 goto out_err; 372 if (bof_object_set(root, "pm4", blob)) 373 goto out_err; 374 bof_decref(blob); 375 blob = NULL; 376 /* dump bo */ 377 array = bof_array(); 378 if (array == NULL) 379 goto out_err; 380 for (i = 0; i < csg->base.crelocs; i++) { 381 bo = bof_object(); 382 if (bo == NULL) 383 goto out_err; 384 size = bof_int32(csg->relocs_bo[i]->size); 385 if (size == NULL) 386 goto out_err; 387 if (bof_object_set(bo, "size", size)) 388 goto out_err; 389 bof_decref(size); 390 size = NULL; 391 handle = bof_int32(csg->relocs_bo[i]->handle); 392 if (handle == NULL) 393 goto out_err; 394 if (bof_object_set(bo, "handle", handle)) 395 goto out_err; 396 bof_decref(handle); 397 handle = NULL; 398 radeon_bo_map((struct radeon_bo*)csg->relocs_bo[i], 0); 399 blob = bof_blob(csg->relocs_bo[i]->size, csg->relocs_bo[i]->ptr); 400 radeon_bo_unmap((struct radeon_bo*)csg->relocs_bo[i]); 401 if (blob == NULL) 402 goto out_err; 403 if (bof_object_set(bo, "data", blob)) 404 goto out_err; 405 bof_decref(blob); 406 blob = NULL; 407 if (bof_array_append(array, bo)) 408 goto out_err; 409 bof_decref(bo); 410 bo = NULL; 411 } 412 if (bof_object_set(root, "bo", array)) 413 goto out_err; 414 sprintf(tmp, "d-0x%04X-%08d.bof", csm->device_id, csm->nbof++); 415 bof_dump_file(root, tmp); 416 out_err: 417 bof_decref(blob); 418 bof_decref(array); 419 bof_decref(bo); 420 bof_decref(size); 421 bof_decref(handle); 422 bof_decref(device_id); 423 bof_decref(root); 424 } 425 #endif 426 427 static int cs_gem_emit(struct radeon_cs_int *cs) 428 { 429 struct cs_gem *csg = (struct cs_gem*)cs; 430 uint64_t chunk_array[2]; 431 unsigned i; 432 int r; 433 434 while (cs->cdw & 7) 435 radeon_cs_write_dword((struct radeon_cs *)cs, 0x80000000); 436 437 #if CS_BOF_DUMP 438 cs_gem_dump_bof(cs); 439 #endif 440 csg->chunks[0].length_dw = cs->cdw; 441 442 chunk_array[0] = (uint64_t)(uintptr_t)&csg->chunks[0]; 443 chunk_array[1] = (uint64_t)(uintptr_t)&csg->chunks[1]; 444 445 csg->cs.num_chunks = 2; 446 csg->cs.chunks = (uint64_t)(uintptr_t)chunk_array; 447 448 r = drmCommandWriteRead(cs->csm->fd, DRM_RADEON_CS, 449 &csg->cs, sizeof(struct drm_radeon_cs)); 450 for (i = 0; i < csg->base.crelocs; i++) { 451 csg->relocs_bo[i]->space_accounted = 0; 452 /* bo might be referenced from another context so have to use atomic opertions */ 453 atomic_dec((atomic_t *)radeon_gem_get_reloc_in_cs((struct radeon_bo*)csg->relocs_bo[i]), cs->id); 454 radeon_bo_unref((struct radeon_bo *)csg->relocs_bo[i]); 455 csg->relocs_bo[i] = NULL; 456 } 457 458 cs->csm->read_used = 0; 459 cs->csm->vram_write_used = 0; 460 cs->csm->gart_write_used = 0; 461 return r; 462 } 463 464 static int cs_gem_destroy(struct radeon_cs_int *cs) 465 { 466 struct cs_gem *csg = (struct cs_gem*)cs; 467 468 free_id(cs->id); 469 free(csg->relocs_bo); 470 free(cs->relocs); 471 free(cs->packets); 472 free(cs); 473 return 0; 474 } 475 476 static int cs_gem_erase(struct radeon_cs_int *cs) 477 { 478 struct cs_gem *csg = (struct cs_gem*)cs; 479 unsigned i; 480 481 if (csg->relocs_bo) { 482 for (i = 0; i < csg->base.crelocs; i++) { 483 if (csg->relocs_bo[i]) { 484 /* bo might be referenced from another context so have to use atomic opertions */ 485 atomic_dec((atomic_t *)radeon_gem_get_reloc_in_cs((struct radeon_bo*)csg->relocs_bo[i]), cs->id); 486 radeon_bo_unref((struct radeon_bo *)csg->relocs_bo[i]); 487 csg->relocs_bo[i] = NULL; 488 } 489 } 490 } 491 cs->relocs_total_size = 0; 492 cs->cdw = 0; 493 cs->section_ndw = 0; 494 cs->crelocs = 0; 495 csg->chunks[0].length_dw = 0; 496 csg->chunks[1].length_dw = 0; 497 return 0; 498 } 499 500 static int cs_gem_need_flush(struct radeon_cs_int *cs) 501 { 502 return 0; //(cs->relocs_total_size > (32*1024*1024)); 503 } 504 505 static void cs_gem_print(struct radeon_cs_int *cs, FILE *file) 506 { 507 struct radeon_cs_manager_gem *csm; 508 unsigned int i; 509 510 csm = (struct radeon_cs_manager_gem *)cs->csm; 511 fprintf(file, "VENDORID:DEVICEID 0x%04X:0x%04X\n", 0x1002, csm->device_id); 512 for (i = 0; i < cs->cdw; i++) { 513 fprintf(file, "0x%08X\n", cs->packets[i]); 514 } 515 } 516 517 static const struct radeon_cs_funcs radeon_cs_gem_funcs = { 518 .cs_create = cs_gem_create, 519 .cs_write_reloc = cs_gem_write_reloc, 520 .cs_begin = cs_gem_begin, 521 .cs_end = cs_gem_end, 522 .cs_emit = cs_gem_emit, 523 .cs_destroy = cs_gem_destroy, 524 .cs_erase = cs_gem_erase, 525 .cs_need_flush = cs_gem_need_flush, 526 .cs_print = cs_gem_print, 527 }; 528 529 static int radeon_get_device_id(int fd, uint32_t *device_id) 530 { 531 struct drm_radeon_info info = {}; 532 int r; 533 534 *device_id = 0; 535 info.request = RADEON_INFO_DEVICE_ID; 536 info.value = (uintptr_t)device_id; 537 r = drmCommandWriteRead(fd, DRM_RADEON_INFO, &info, 538 sizeof(struct drm_radeon_info)); 539 return r; 540 } 541 542 struct radeon_cs_manager *radeon_cs_manager_gem_ctor(int fd) 543 { 544 struct radeon_cs_manager_gem *csm; 545 546 csm = calloc(1, sizeof(struct radeon_cs_manager_gem)); 547 if (csm == NULL) { 548 return NULL; 549 } 550 csm->base.funcs = &radeon_cs_gem_funcs; 551 csm->base.fd = fd; 552 radeon_get_device_id(fd, &csm->device_id); 553 return &csm->base; 554 } 555 556 void radeon_cs_manager_gem_dtor(struct radeon_cs_manager *csm) 557 { 558 free(csm); 559 } 560
