1 /* 2 * Mesa 3-D graphics library 3 * 4 * Copyright (C) 2012-2013 LunarG, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included 14 * in all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Chia-I Wu <olv (at) lunarg.com> 26 */ 27 28 #include "util/u_prim.h" 29 #include "core/intel_winsys.h" 30 31 #include "ilo_render.h" 32 #include "ilo_blit.h" 33 #include "ilo_context.h" 34 #include "ilo_cp.h" 35 #include "ilo_query.h" 36 #include "ilo_shader.h" 37 #include "ilo_state.h" 38 #include "ilo_draw.h" 39 40 static void 41 ilo_draw_set_owner(struct ilo_context *ilo) 42 { 43 ilo_cp_set_owner(ilo->cp, INTEL_RING_RENDER, &ilo->draw.cp_owner); 44 } 45 46 static uint64_t 47 query_timestamp_to_ns(const struct ilo_context *ilo, uint64_t timestamp) 48 { 49 /* see ilo_get_timestamp() */ 50 return (timestamp & 0xffffffff) * 80; 51 } 52 53 /** 54 * Process the bo and accumulate the result. The bo is emptied. 55 */ 56 static void 57 query_process_bo(const struct ilo_context *ilo, struct ilo_query *q) 58 { 59 const uint64_t *vals; 60 uint64_t tmp; 61 int i; 62 63 if (!q->used) 64 return; 65 66 vals = intel_bo_map(q->bo, false); 67 if (!vals) { 68 q->used = 0; 69 return; 70 } 71 72 switch (q->type) { 73 case PIPE_QUERY_OCCLUSION_COUNTER: 74 case PIPE_QUERY_OCCLUSION_PREDICATE: 75 case PIPE_QUERY_TIME_ELAPSED: 76 case PIPE_QUERY_PRIMITIVES_GENERATED: 77 case PIPE_QUERY_PRIMITIVES_EMITTED: 78 assert(q->stride == sizeof(*vals) * 2); 79 80 tmp = 0; 81 for (i = 0; i < q->used; i++) 82 tmp += vals[2 * i + 1] - vals[2 * i]; 83 84 if (q->type == PIPE_QUERY_TIME_ELAPSED) 85 tmp = query_timestamp_to_ns(ilo, tmp); 86 87 q->result.u64 += tmp; 88 break; 89 case PIPE_QUERY_TIMESTAMP: 90 assert(q->stride == sizeof(*vals)); 91 92 q->result.u64 = query_timestamp_to_ns(ilo, vals[q->used - 1]); 93 break; 94 case PIPE_QUERY_PIPELINE_STATISTICS: 95 assert(q->stride == sizeof(*vals) * 22); 96 97 for (i = 0; i < q->used; i++) { 98 struct pipe_query_data_pipeline_statistics *stats = 99 &q->result.pipeline_statistics; 100 const uint64_t *begin = vals + 22 * i; 101 const uint64_t *end = begin + 11; 102 103 stats->ia_vertices += end[0] - begin[0]; 104 stats->ia_primitives += end[1] - begin[1]; 105 stats->vs_invocations += end[2] - begin[2]; 106 stats->gs_invocations += end[3] - begin[3]; 107 stats->gs_primitives += end[4] - begin[4]; 108 stats->c_invocations += end[5] - begin[5]; 109 stats->c_primitives += end[6] - begin[6]; 110 stats->ps_invocations += end[7] - begin[7]; 111 stats->hs_invocations += end[8] - begin[8]; 112 stats->ds_invocations += end[9] - begin[9]; 113 stats->cs_invocations += end[10] - begin[10]; 114 } 115 break; 116 default: 117 break; 118 } 119 120 intel_bo_unmap(q->bo); 121 122 q->used = 0; 123 } 124 125 static void 126 query_begin_bo(struct ilo_context *ilo, struct ilo_query *q) 127 { 128 /* bo is full */ 129 if (q->used >= q->count) 130 query_process_bo(ilo, q); 131 132 /* write the beginning value to the bo */ 133 if (q->in_pairs) 134 ilo_render_emit_query(ilo->render, q, q->stride * q->used); 135 } 136 137 static void 138 query_end_bo(struct ilo_context *ilo, struct ilo_query *q) 139 { 140 uint32_t offset; 141 142 assert(q->used < q->count); 143 144 offset = q->stride * q->used; 145 if (q->in_pairs) 146 offset += q->stride >> 1; 147 148 q->used++; 149 150 /* write the ending value to the bo */ 151 ilo_render_emit_query(ilo->render, q, offset); 152 } 153 154 bool 155 ilo_init_draw_query(struct ilo_context *ilo, struct ilo_query *q) 156 { 157 unsigned bo_size; 158 159 switch (q->type) { 160 case PIPE_QUERY_OCCLUSION_COUNTER: 161 case PIPE_QUERY_OCCLUSION_PREDICATE: 162 case PIPE_QUERY_TIME_ELAPSED: 163 case PIPE_QUERY_PRIMITIVES_GENERATED: 164 case PIPE_QUERY_PRIMITIVES_EMITTED: 165 q->stride = sizeof(uint64_t); 166 q->in_pairs = true; 167 break; 168 case PIPE_QUERY_TIMESTAMP: 169 q->stride = sizeof(uint64_t); 170 q->in_pairs = false; 171 break; 172 case PIPE_QUERY_PIPELINE_STATISTICS: 173 q->stride = sizeof(uint64_t) * 11; 174 q->in_pairs = true; 175 break; 176 default: 177 return false; 178 break; 179 } 180 181 q->cmd_len = ilo_render_get_query_len(ilo->render, q->type); 182 183 /* double cmd_len and stride if in pairs */ 184 q->cmd_len <<= q->in_pairs; 185 q->stride <<= q->in_pairs; 186 187 bo_size = (q->stride > 4096) ? q->stride : 4096; 188 q->bo = intel_winsys_alloc_bo(ilo->winsys, "query", bo_size, false); 189 if (!q->bo) 190 return false; 191 192 q->count = bo_size / q->stride; 193 194 return true; 195 } 196 197 void 198 ilo_begin_draw_query(struct ilo_context *ilo, struct ilo_query *q) 199 { 200 ilo_draw_set_owner(ilo); 201 202 /* need to submit first */ 203 if (!ilo_builder_validate(&ilo->cp->builder, 1, &q->bo) || 204 ilo_cp_space(ilo->cp) < q->cmd_len) { 205 ilo_cp_submit(ilo->cp, "out of aperture or space"); 206 207 assert(ilo_builder_validate(&ilo->cp->builder, 1, &q->bo)); 208 assert(ilo_cp_space(ilo->cp) >= q->cmd_len); 209 210 ilo_draw_set_owner(ilo); 211 } 212 213 /* reserve the space for ending/pausing the query */ 214 ilo->draw.cp_owner.reserve += q->cmd_len >> q->in_pairs; 215 216 query_begin_bo(ilo, q); 217 218 if (q->in_pairs) 219 list_add(&q->list, &ilo->draw.queries); 220 } 221 222 void 223 ilo_end_draw_query(struct ilo_context *ilo, struct ilo_query *q) 224 { 225 ilo_draw_set_owner(ilo); 226 227 /* reclaim the reserved space */ 228 ilo->draw.cp_owner.reserve -= q->cmd_len >> q->in_pairs; 229 assert(ilo->draw.cp_owner.reserve >= 0); 230 231 query_end_bo(ilo, q); 232 233 list_delinit(&q->list); 234 } 235 236 /** 237 * Process the raw query data. 238 */ 239 void 240 ilo_process_draw_query(struct ilo_context *ilo, struct ilo_query *q) 241 { 242 query_process_bo(ilo, q); 243 } 244 245 static void 246 ilo_draw_own_cp(struct ilo_cp *cp, void *data) 247 { 248 struct ilo_context *ilo = data; 249 250 /* multiply by 2 for both resuming and pausing */ 251 if (ilo_cp_space(ilo->cp) < ilo->draw.cp_owner.reserve * 2) { 252 ilo_cp_submit(ilo->cp, "out of space"); 253 assert(ilo_cp_space(ilo->cp) >= ilo->draw.cp_owner.reserve * 2); 254 } 255 256 while (true) { 257 struct ilo_builder_snapshot snapshot; 258 struct ilo_query *q; 259 260 ilo_builder_batch_snapshot(&ilo->cp->builder, &snapshot); 261 262 /* resume queries */ 263 LIST_FOR_EACH_ENTRY(q, &ilo->draw.queries, list) 264 query_begin_bo(ilo, q); 265 266 if (!ilo_builder_validate(&ilo->cp->builder, 0, NULL)) { 267 ilo_builder_batch_restore(&ilo->cp->builder, &snapshot); 268 269 if (ilo_builder_batch_used(&ilo->cp->builder)) { 270 ilo_cp_submit(ilo->cp, "out of aperture"); 271 continue; 272 } 273 } 274 275 break; 276 } 277 278 assert(ilo_cp_space(ilo->cp) >= ilo->draw.cp_owner.reserve); 279 } 280 281 static void 282 ilo_draw_release_cp(struct ilo_cp *cp, void *data) 283 { 284 struct ilo_context *ilo = data; 285 struct ilo_query *q; 286 287 assert(ilo_cp_space(ilo->cp) >= ilo->draw.cp_owner.reserve); 288 289 /* pause queries */ 290 LIST_FOR_EACH_ENTRY(q, &ilo->draw.queries, list) 291 query_end_bo(ilo, q); 292 } 293 294 static bool 295 draw_vbo(struct ilo_context *ilo, const struct ilo_state_vector *vec) 296 { 297 bool need_flush = false; 298 bool success = true; 299 int max_len, before_space; 300 301 /* on Gen7 and Gen7.5, we need SOL_RESET to reset the SO write offsets */ 302 if (ilo_dev_gen(ilo->dev) >= ILO_GEN(7) && 303 ilo_dev_gen(ilo->dev) <= ILO_GEN(7.5) && 304 (vec->dirty & ILO_DIRTY_SO) && vec->so.enabled && 305 !vec->so.append_bitmask) { 306 ilo_cp_submit(ilo->cp, "SOL_RESET"); 307 ilo_cp_set_one_off_flags(ilo->cp, INTEL_EXEC_GEN7_SOL_RESET); 308 } 309 310 if (ilo_builder_batch_used(&ilo->cp->builder)) { 311 /* 312 * Without a better tracking mechanism, when the framebuffer changes, we 313 * have to assume that the old framebuffer may be sampled from. If that 314 * happens in the middle of a batch buffer, we need to insert manual 315 * flushes. 316 */ 317 need_flush = (vec->dirty & ILO_DIRTY_FB); 318 319 /* same to SO target changes */ 320 need_flush |= (vec->dirty & ILO_DIRTY_SO); 321 } 322 323 ilo_draw_set_owner(ilo); 324 325 /* make sure there is enough room first */ 326 max_len = ilo_render_get_draw_len(ilo->render, vec); 327 if (need_flush) 328 max_len += ilo_render_get_flush_len(ilo->render); 329 330 if (max_len > ilo_cp_space(ilo->cp)) { 331 ilo_cp_submit(ilo->cp, "out of space"); 332 need_flush = false; 333 assert(max_len <= ilo_cp_space(ilo->cp)); 334 } 335 336 /* space available before emission */ 337 before_space = ilo_cp_space(ilo->cp); 338 339 if (need_flush) 340 ilo_render_emit_flush(ilo->render); 341 342 while (true) { 343 struct ilo_builder_snapshot snapshot; 344 345 ilo_builder_batch_snapshot(&ilo->cp->builder, &snapshot); 346 347 ilo_render_emit_draw(ilo->render, vec); 348 349 if (!ilo_builder_validate(&ilo->cp->builder, 0, NULL)) { 350 ilo_builder_batch_restore(&ilo->cp->builder, &snapshot); 351 352 /* flush and try again */ 353 if (ilo_builder_batch_used(&ilo->cp->builder)) { 354 ilo_cp_submit(ilo->cp, "out of aperture"); 355 continue; 356 } 357 358 success = false; 359 } 360 361 break; 362 } 363 364 /* sanity check size estimation */ 365 assert(before_space - ilo_cp_space(ilo->cp) <= max_len); 366 367 return success; 368 } 369 370 void 371 ilo_draw_rectlist(struct ilo_context *ilo) 372 { 373 int max_len, before_space; 374 bool need_flush; 375 376 need_flush = ilo_builder_batch_used(&ilo->cp->builder); 377 378 ilo_draw_set_owner(ilo); 379 380 max_len = ilo_render_get_rectlist_len(ilo->render, ilo->blitter); 381 max_len += ilo_render_get_flush_len(ilo->render) * 2; 382 383 if (max_len > ilo_cp_space(ilo->cp)) { 384 ilo_cp_submit(ilo->cp, "out of space"); 385 need_flush = false; 386 assert(max_len <= ilo_cp_space(ilo->cp)); 387 } 388 389 before_space = ilo_cp_space(ilo->cp); 390 391 /* 392 * From the Sandy Bridge PRM, volume 2 part 1, page 313: 393 * 394 * "If other rendering operations have preceded this clear, a 395 * PIPE_CONTROL with write cache flush enabled and Z-inhibit 396 * disabled must be issued before the rectangle primitive used for 397 * the depth buffer clear operation." 398 * 399 * From the Sandy Bridge PRM, volume 2 part 1, page 314: 400 * 401 * "Depth buffer clear pass must be followed by a PIPE_CONTROL 402 * command with DEPTH_STALL bit set and Then followed by Depth 403 * FLUSH" 404 * 405 * But the pipeline has to be flushed both before and after not only 406 * because of these workarounds. We need them for reasons such as 407 * 408 * - we may sample from a texture that was rendered to 409 * - we may sample from the fb shortly after 410 * 411 * Skip checking blitter->op and do the flushes. 412 */ 413 if (need_flush) 414 ilo_render_emit_flush(ilo->render); 415 416 while (true) { 417 struct ilo_builder_snapshot snapshot; 418 419 ilo_builder_batch_snapshot(&ilo->cp->builder, &snapshot); 420 421 ilo_render_emit_rectlist(ilo->render, ilo->blitter); 422 423 if (!ilo_builder_validate(&ilo->cp->builder, 0, NULL)) { 424 ilo_builder_batch_restore(&ilo->cp->builder, &snapshot); 425 426 /* flush and try again */ 427 if (ilo_builder_batch_used(&ilo->cp->builder)) { 428 ilo_cp_submit(ilo->cp, "out of aperture"); 429 continue; 430 } 431 } 432 433 break; 434 } 435 436 ilo_render_invalidate_hw(ilo->render); 437 438 ilo_render_emit_flush(ilo->render); 439 440 /* sanity check size estimation */ 441 assert(before_space - ilo_cp_space(ilo->cp) <= max_len); 442 } 443 444 static void 445 draw_vbo_with_sw_restart(struct ilo_context *ilo, 446 const struct pipe_draw_info *info) 447 { 448 const struct ilo_ib_state *ib = &ilo->state_vector.ib; 449 const struct ilo_vma *vma; 450 union { 451 const void *ptr; 452 const uint8_t *u8; 453 const uint16_t *u16; 454 const uint32_t *u32; 455 } u; 456 457 /* we will draw with IB mapped */ 458 if (ib->state.buffer) { 459 vma = ilo_resource_get_vma(ib->state.buffer); 460 u.ptr = intel_bo_map(vma->bo, false); 461 if (u.ptr) 462 u.u8 += vma->bo_offset + ib->state.offset; 463 } else { 464 vma = NULL; 465 u.ptr = ib->state.user_buffer; 466 } 467 468 if (!u.ptr) 469 return; 470 471 #define DRAW_VBO_WITH_SW_RESTART(pipe, info, ptr) do { \ 472 const unsigned end = (info)->start + (info)->count; \ 473 struct pipe_draw_info subinfo; \ 474 unsigned i; \ 475 \ 476 subinfo = *(info); \ 477 subinfo.primitive_restart = false; \ 478 for (i = (info)->start; i < end; i++) { \ 479 if ((ptr)[i] == (info)->restart_index) { \ 480 subinfo.count = i - subinfo.start; \ 481 if (subinfo.count) \ 482 (pipe)->draw_vbo(pipe, &subinfo); \ 483 subinfo.start = i + 1; \ 484 } \ 485 } \ 486 subinfo.count = i - subinfo.start; \ 487 if (subinfo.count) \ 488 (pipe)->draw_vbo(pipe, &subinfo); \ 489 } while (0) 490 491 switch (ib->state.index_size) { 492 case 1: 493 DRAW_VBO_WITH_SW_RESTART(&ilo->base, info, u.u8); 494 break; 495 case 2: 496 DRAW_VBO_WITH_SW_RESTART(&ilo->base, info, u.u16); 497 break; 498 case 4: 499 DRAW_VBO_WITH_SW_RESTART(&ilo->base, info, u.u32); 500 break; 501 default: 502 assert(!"unsupported index size"); 503 break; 504 } 505 506 #undef DRAW_VBO_WITH_SW_RESTART 507 508 if (vma) 509 intel_bo_unmap(vma->bo); 510 } 511 512 static bool 513 draw_vbo_need_sw_restart(const struct ilo_context *ilo, 514 const struct pipe_draw_info *info) 515 { 516 /* the restart index is fixed prior to GEN7.5 */ 517 if (ilo_dev_gen(ilo->dev) < ILO_GEN(7.5)) { 518 const unsigned cut_index = 519 (ilo->state_vector.ib.state.index_size == 1) ? 0xff : 520 (ilo->state_vector.ib.state.index_size == 2) ? 0xffff : 521 (ilo->state_vector.ib.state.index_size == 4) ? 0xffffffff : 0; 522 523 if (info->restart_index < cut_index) 524 return true; 525 } 526 527 switch (info->mode) { 528 case PIPE_PRIM_POINTS: 529 case PIPE_PRIM_LINES: 530 case PIPE_PRIM_LINE_STRIP: 531 case PIPE_PRIM_TRIANGLES: 532 case PIPE_PRIM_TRIANGLE_STRIP: 533 /* these never need software fallback */ 534 return false; 535 case PIPE_PRIM_LINE_LOOP: 536 case PIPE_PRIM_POLYGON: 537 case PIPE_PRIM_QUAD_STRIP: 538 case PIPE_PRIM_QUADS: 539 case PIPE_PRIM_TRIANGLE_FAN: 540 /* these need software fallback prior to GEN7.5 */ 541 return (ilo_dev_gen(ilo->dev) < ILO_GEN(7.5)); 542 default: 543 /* the rest always needs software fallback */ 544 return true; 545 } 546 } 547 548 static void 549 ilo_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info) 550 { 551 struct ilo_context *ilo = ilo_context(pipe); 552 int vs_scratch_size, gs_scratch_size, fs_scratch_size; 553 554 if (ilo_debug & ILO_DEBUG_DRAW) { 555 if (info->indexed) { 556 ilo_printf("indexed draw %s: " 557 "index start %d, count %d, vertex range [%d, %d]\n", 558 u_prim_name(info->mode), info->start, info->count, 559 info->min_index, info->max_index); 560 } 561 else { 562 ilo_printf("draw %s: vertex start %d, count %d\n", 563 u_prim_name(info->mode), info->start, info->count); 564 } 565 566 ilo_state_vector_dump_dirty(&ilo->state_vector); 567 } 568 569 if (ilo_skip_rendering(ilo)) 570 return; 571 572 if (info->primitive_restart && info->indexed && 573 draw_vbo_need_sw_restart(ilo, info)) { 574 draw_vbo_with_sw_restart(ilo, info); 575 return; 576 } 577 578 ilo_finalize_3d_states(ilo, info); 579 580 /* upload kernels */ 581 ilo_shader_cache_upload(ilo->shader_cache, &ilo->cp->builder); 582 583 /* prepare scratch spaces */ 584 ilo_shader_cache_get_max_scratch_sizes(ilo->shader_cache, 585 &vs_scratch_size, &gs_scratch_size, &fs_scratch_size); 586 ilo_render_prepare_scratch_spaces(ilo->render, 587 vs_scratch_size, gs_scratch_size, fs_scratch_size); 588 589 ilo_blit_resolve_framebuffer(ilo); 590 591 /* If draw_vbo ever fails, return immediately. */ 592 if (!draw_vbo(ilo, &ilo->state_vector)) 593 return; 594 595 /* clear dirty status */ 596 ilo->state_vector.dirty = 0x0; 597 598 /* avoid dangling pointer reference */ 599 ilo->state_vector.draw = NULL; 600 601 if (ilo_debug & ILO_DEBUG_NOCACHE) 602 ilo_render_emit_flush(ilo->render); 603 } 604 605 static void 606 ilo_texture_barrier(struct pipe_context *pipe, unsigned flags) 607 { 608 struct ilo_context *ilo = ilo_context(pipe); 609 610 if (ilo->cp->ring != INTEL_RING_RENDER) 611 return; 612 613 ilo_render_emit_flush(ilo->render); 614 615 /* don't know why */ 616 if (ilo_dev_gen(ilo->dev) >= ILO_GEN(7)) 617 ilo_cp_submit(ilo->cp, "texture barrier"); 618 } 619 620 static void 621 ilo_get_sample_position(struct pipe_context *pipe, 622 unsigned sample_count, 623 unsigned sample_index, 624 float *out_value) 625 { 626 struct ilo_context *ilo = ilo_context(pipe); 627 628 ilo_render_get_sample_position(ilo->render, 629 sample_count, sample_index, 630 &out_value[0], &out_value[1]); 631 } 632 633 void 634 ilo_init_draw(struct ilo_context *ilo) 635 { 636 ilo->draw.cp_owner.own = ilo_draw_own_cp; 637 ilo->draw.cp_owner.release = ilo_draw_release_cp; 638 ilo->draw.cp_owner.data = (void *) ilo; 639 ilo->draw.cp_owner.reserve = 0; 640 641 list_inithead(&ilo->draw.queries); 642 } 643 644 /** 645 * Initialize 3D-related functions. 646 */ 647 void 648 ilo_init_draw_functions(struct ilo_context *ilo) 649 { 650 ilo->base.draw_vbo = ilo_draw_vbo; 651 ilo->base.texture_barrier = ilo_texture_barrier; 652 ilo->base.get_sample_position = ilo_get_sample_position; 653 } 654
