1 /************************************************************************** 2 * 3 * Copyright 2009-2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 /** 29 * @file 30 * Depth/stencil testing to LLVM IR translation. 31 * 32 * To be done accurately/efficiently the depth/stencil test must be done with 33 * the same type/format of the depth/stencil buffer, which implies massaging 34 * the incoming depths to fit into place. Using a more straightforward 35 * type/format for depth/stencil values internally and only convert when 36 * flushing would avoid this, but it would most likely result in depth fighting 37 * artifacts. 38 * 39 * Since we're using linear layout for everything, but we need to deal with 40 * 2x2 quads, we need to load/store multiple values and swizzle them into 41 * place (we could avoid this by doing depth/stencil testing in linear format, 42 * which would be easy for late depth/stencil test as we could do that after 43 * the fragment shader loop just as we do for color buffers, but more tricky 44 * for early depth test as we'd need both masks and interpolated depth in 45 * linear format). 46 * 47 * 48 * @author Jose Fonseca <jfonseca (at) vmware.com> 49 * @author Brian Paul <jfonseca (at) vmware.com> 50 */ 51 52 #include "pipe/p_state.h" 53 #include "util/u_format.h" 54 #include "util/u_cpu_detect.h" 55 56 #include "gallivm/lp_bld_type.h" 57 #include "gallivm/lp_bld_arit.h" 58 #include "gallivm/lp_bld_bitarit.h" 59 #include "gallivm/lp_bld_const.h" 60 #include "gallivm/lp_bld_conv.h" 61 #include "gallivm/lp_bld_logic.h" 62 #include "gallivm/lp_bld_flow.h" 63 #include "gallivm/lp_bld_intr.h" 64 #include "gallivm/lp_bld_debug.h" 65 #include "gallivm/lp_bld_swizzle.h" 66 #include "gallivm/lp_bld_pack.h" 67 68 #include "lp_bld_depth.h" 69 70 71 /** Used to select fields from pipe_stencil_state */ 72 enum stencil_op { 73 S_FAIL_OP, 74 Z_FAIL_OP, 75 Z_PASS_OP 76 }; 77 78 79 80 /** 81 * Do the stencil test comparison (compare FB stencil values against ref value). 82 * This will be used twice when generating two-sided stencil code. 83 * \param stencil the front/back stencil state 84 * \param stencilRef the stencil reference value, replicated as a vector 85 * \param stencilVals vector of stencil values from framebuffer 86 * \return vector mask of pass/fail values (~0 or 0) 87 */ 88 static LLVMValueRef 89 lp_build_stencil_test_single(struct lp_build_context *bld, 90 const struct pipe_stencil_state *stencil, 91 LLVMValueRef stencilRef, 92 LLVMValueRef stencilVals) 93 { 94 LLVMBuilderRef builder = bld->gallivm->builder; 95 const unsigned stencilMax = 255; /* XXX fix */ 96 struct lp_type type = bld->type; 97 LLVMValueRef res; 98 99 /* 100 * SSE2 has intrinsics for signed comparisons, but not unsigned ones. Values 101 * are between 0..255 so ensure we generate the fastest comparisons for 102 * wider elements. 103 */ 104 if (type.width <= 8) { 105 assert(!type.sign); 106 } else { 107 assert(type.sign); 108 } 109 110 assert(stencil->enabled); 111 112 if (stencil->valuemask != stencilMax) { 113 /* compute stencilRef = stencilRef & valuemask */ 114 LLVMValueRef valuemask = lp_build_const_int_vec(bld->gallivm, type, stencil->valuemask); 115 stencilRef = LLVMBuildAnd(builder, stencilRef, valuemask, ""); 116 /* compute stencilVals = stencilVals & valuemask */ 117 stencilVals = LLVMBuildAnd(builder, stencilVals, valuemask, ""); 118 } 119 120 res = lp_build_cmp(bld, stencil->func, stencilRef, stencilVals); 121 122 return res; 123 } 124 125 126 /** 127 * Do the one or two-sided stencil test comparison. 128 * \sa lp_build_stencil_test_single 129 * \param front_facing an integer vector mask, indicating front (~0) or back 130 * (0) facing polygon. If NULL, assume front-facing. 131 */ 132 static LLVMValueRef 133 lp_build_stencil_test(struct lp_build_context *bld, 134 const struct pipe_stencil_state stencil[2], 135 LLVMValueRef stencilRefs[2], 136 LLVMValueRef stencilVals, 137 LLVMValueRef front_facing) 138 { 139 LLVMValueRef res; 140 141 assert(stencil[0].enabled); 142 143 /* do front face test */ 144 res = lp_build_stencil_test_single(bld, &stencil[0], 145 stencilRefs[0], stencilVals); 146 147 if (stencil[1].enabled && front_facing != NULL) { 148 /* do back face test */ 149 LLVMValueRef back_res; 150 151 back_res = lp_build_stencil_test_single(bld, &stencil[1], 152 stencilRefs[1], stencilVals); 153 154 res = lp_build_select(bld, front_facing, res, back_res); 155 } 156 157 return res; 158 } 159 160 161 /** 162 * Apply the stencil operator (add/sub/keep/etc) to the given vector 163 * of stencil values. 164 * \return new stencil values vector 165 */ 166 static LLVMValueRef 167 lp_build_stencil_op_single(struct lp_build_context *bld, 168 const struct pipe_stencil_state *stencil, 169 enum stencil_op op, 170 LLVMValueRef stencilRef, 171 LLVMValueRef stencilVals) 172 173 { 174 LLVMBuilderRef builder = bld->gallivm->builder; 175 struct lp_type type = bld->type; 176 LLVMValueRef res; 177 LLVMValueRef max = lp_build_const_int_vec(bld->gallivm, type, 0xff); 178 unsigned stencil_op; 179 180 assert(type.sign); 181 182 switch (op) { 183 case S_FAIL_OP: 184 stencil_op = stencil->fail_op; 185 break; 186 case Z_FAIL_OP: 187 stencil_op = stencil->zfail_op; 188 break; 189 case Z_PASS_OP: 190 stencil_op = stencil->zpass_op; 191 break; 192 default: 193 assert(0 && "Invalid stencil_op mode"); 194 stencil_op = PIPE_STENCIL_OP_KEEP; 195 } 196 197 switch (stencil_op) { 198 case PIPE_STENCIL_OP_KEEP: 199 res = stencilVals; 200 /* we can return early for this case */ 201 return res; 202 case PIPE_STENCIL_OP_ZERO: 203 res = bld->zero; 204 break; 205 case PIPE_STENCIL_OP_REPLACE: 206 res = stencilRef; 207 break; 208 case PIPE_STENCIL_OP_INCR: 209 res = lp_build_add(bld, stencilVals, bld->one); 210 res = lp_build_min(bld, res, max); 211 break; 212 case PIPE_STENCIL_OP_DECR: 213 res = lp_build_sub(bld, stencilVals, bld->one); 214 res = lp_build_max(bld, res, bld->zero); 215 break; 216 case PIPE_STENCIL_OP_INCR_WRAP: 217 res = lp_build_add(bld, stencilVals, bld->one); 218 res = LLVMBuildAnd(builder, res, max, ""); 219 break; 220 case PIPE_STENCIL_OP_DECR_WRAP: 221 res = lp_build_sub(bld, stencilVals, bld->one); 222 res = LLVMBuildAnd(builder, res, max, ""); 223 break; 224 case PIPE_STENCIL_OP_INVERT: 225 res = LLVMBuildNot(builder, stencilVals, ""); 226 res = LLVMBuildAnd(builder, res, max, ""); 227 break; 228 default: 229 assert(0 && "bad stencil op mode"); 230 res = bld->undef; 231 } 232 233 return res; 234 } 235 236 237 /** 238 * Do the one or two-sided stencil test op/update. 239 */ 240 static LLVMValueRef 241 lp_build_stencil_op(struct lp_build_context *bld, 242 const struct pipe_stencil_state stencil[2], 243 enum stencil_op op, 244 LLVMValueRef stencilRefs[2], 245 LLVMValueRef stencilVals, 246 LLVMValueRef mask, 247 LLVMValueRef front_facing) 248 249 { 250 LLVMBuilderRef builder = bld->gallivm->builder; 251 LLVMValueRef res; 252 253 assert(stencil[0].enabled); 254 255 /* do front face op */ 256 res = lp_build_stencil_op_single(bld, &stencil[0], op, 257 stencilRefs[0], stencilVals); 258 259 if (stencil[1].enabled && front_facing != NULL) { 260 /* do back face op */ 261 LLVMValueRef back_res; 262 263 back_res = lp_build_stencil_op_single(bld, &stencil[1], op, 264 stencilRefs[1], stencilVals); 265 266 res = lp_build_select(bld, front_facing, res, back_res); 267 } 268 269 if (stencil[0].writemask != 0xff || 270 (stencil[1].enabled && front_facing != NULL && stencil[1].writemask != 0xff)) { 271 /* mask &= stencil[0].writemask */ 272 LLVMValueRef writemask = lp_build_const_int_vec(bld->gallivm, bld->type, 273 stencil[0].writemask); 274 if (stencil[1].enabled && stencil[1].writemask != stencil[0].writemask && front_facing != NULL) { 275 LLVMValueRef back_writemask = lp_build_const_int_vec(bld->gallivm, bld->type, 276 stencil[1].writemask); 277 writemask = lp_build_select(bld, front_facing, writemask, back_writemask); 278 } 279 280 mask = LLVMBuildAnd(builder, mask, writemask, ""); 281 /* res = (res & mask) | (stencilVals & ~mask) */ 282 res = lp_build_select_bitwise(bld, mask, res, stencilVals); 283 } 284 else { 285 /* res = mask ? res : stencilVals */ 286 res = lp_build_select(bld, mask, res, stencilVals); 287 } 288 289 return res; 290 } 291 292 293 294 /** 295 * Return a type that matches the depth/stencil format. 296 */ 297 struct lp_type 298 lp_depth_type(const struct util_format_description *format_desc, 299 unsigned length) 300 { 301 struct lp_type type; 302 unsigned z_swizzle; 303 304 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS); 305 assert(format_desc->block.width == 1); 306 assert(format_desc->block.height == 1); 307 308 memset(&type, 0, sizeof type); 309 type.width = format_desc->block.bits; 310 311 z_swizzle = format_desc->swizzle[0]; 312 if (z_swizzle < 4) { 313 if (format_desc->channel[z_swizzle].type == UTIL_FORMAT_TYPE_FLOAT) { 314 type.floating = TRUE; 315 assert(z_swizzle == 0); 316 assert(format_desc->channel[z_swizzle].size == 32); 317 } 318 else if(format_desc->channel[z_swizzle].type == UTIL_FORMAT_TYPE_UNSIGNED) { 319 assert(format_desc->block.bits <= 32); 320 assert(format_desc->channel[z_swizzle].normalized); 321 if (format_desc->channel[z_swizzle].size < format_desc->block.bits) { 322 /* Prefer signed integers when possible, as SSE has less support 323 * for unsigned comparison; 324 */ 325 type.sign = TRUE; 326 } 327 } 328 else 329 assert(0); 330 } 331 332 type.length = length; 333 334 return type; 335 } 336 337 338 /** 339 * Compute bitmask and bit shift to apply to the incoming fragment Z values 340 * and the Z buffer values needed before doing the Z comparison. 341 * 342 * Note that we leave the Z bits in the position that we find them 343 * in the Z buffer (typically 0xffffff00 or 0x00ffffff). That lets us 344 * get by with fewer bit twiddling steps. 345 */ 346 static boolean 347 get_z_shift_and_mask(const struct util_format_description *format_desc, 348 unsigned *shift, unsigned *width, unsigned *mask) 349 { 350 unsigned total_bits; 351 unsigned z_swizzle; 352 353 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS); 354 assert(format_desc->block.width == 1); 355 assert(format_desc->block.height == 1); 356 357 /* 64bit d/s format is special already extracted 32 bits */ 358 total_bits = format_desc->block.bits > 32 ? 32 : format_desc->block.bits; 359 360 z_swizzle = format_desc->swizzle[0]; 361 362 if (z_swizzle == PIPE_SWIZZLE_NONE) 363 return FALSE; 364 365 *width = format_desc->channel[z_swizzle].size; 366 /* & 31 is for the same reason as the 32-bit limit above */ 367 *shift = format_desc->channel[z_swizzle].shift & 31; 368 369 if (*width == total_bits) { 370 *mask = 0xffffffff; 371 } else { 372 *mask = ((1 << *width) - 1) << *shift; 373 } 374 375 return TRUE; 376 } 377 378 379 /** 380 * Compute bitmask and bit shift to apply to the framebuffer pixel values 381 * to put the stencil bits in the least significant position. 382 * (i.e. 0x000000ff) 383 */ 384 static boolean 385 get_s_shift_and_mask(const struct util_format_description *format_desc, 386 unsigned *shift, unsigned *mask) 387 { 388 unsigned s_swizzle; 389 unsigned sz; 390 391 s_swizzle = format_desc->swizzle[1]; 392 393 if (s_swizzle == PIPE_SWIZZLE_NONE) 394 return FALSE; 395 396 /* just special case 64bit d/s format */ 397 if (format_desc->block.bits > 32) { 398 /* XXX big-endian? */ 399 assert(format_desc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT); 400 *shift = 0; 401 *mask = 0xff; 402 return TRUE; 403 } 404 405 *shift = format_desc->channel[s_swizzle].shift; 406 sz = format_desc->channel[s_swizzle].size; 407 *mask = (1U << sz) - 1U; 408 409 return TRUE; 410 } 411 412 413 /** 414 * Perform the occlusion test and increase the counter. 415 * Test the depth mask. Add the number of channel which has none zero mask 416 * into the occlusion counter. e.g. maskvalue is {-1, -1, -1, -1}. 417 * The counter will add 4. 418 * TODO: could get that out of the fs loop. 419 * 420 * \param type holds element type of the mask vector. 421 * \param maskvalue is the depth test mask. 422 * \param counter is a pointer of the uint32 counter. 423 */ 424 void 425 lp_build_occlusion_count(struct gallivm_state *gallivm, 426 struct lp_type type, 427 LLVMValueRef maskvalue, 428 LLVMValueRef counter) 429 { 430 LLVMBuilderRef builder = gallivm->builder; 431 LLVMContextRef context = gallivm->context; 432 LLVMValueRef countmask = lp_build_const_int_vec(gallivm, type, 1); 433 LLVMValueRef count, newcount; 434 435 assert(type.length <= 16); 436 assert(type.floating); 437 438 if(util_cpu_caps.has_sse && type.length == 4) { 439 const char *movmskintr = "llvm.x86.sse.movmsk.ps"; 440 const char *popcntintr = "llvm.ctpop.i32"; 441 LLVMValueRef bits = LLVMBuildBitCast(builder, maskvalue, 442 lp_build_vec_type(gallivm, type), ""); 443 bits = lp_build_intrinsic_unary(builder, movmskintr, 444 LLVMInt32TypeInContext(context), bits); 445 count = lp_build_intrinsic_unary(builder, popcntintr, 446 LLVMInt32TypeInContext(context), bits); 447 count = LLVMBuildZExt(builder, count, LLVMIntTypeInContext(context, 64), ""); 448 } 449 else if(util_cpu_caps.has_avx && type.length == 8) { 450 const char *movmskintr = "llvm.x86.avx.movmsk.ps.256"; 451 const char *popcntintr = "llvm.ctpop.i32"; 452 LLVMValueRef bits = LLVMBuildBitCast(builder, maskvalue, 453 lp_build_vec_type(gallivm, type), ""); 454 bits = lp_build_intrinsic_unary(builder, movmskintr, 455 LLVMInt32TypeInContext(context), bits); 456 count = lp_build_intrinsic_unary(builder, popcntintr, 457 LLVMInt32TypeInContext(context), bits); 458 count = LLVMBuildZExt(builder, count, LLVMIntTypeInContext(context, 64), ""); 459 } 460 else { 461 unsigned i; 462 LLVMValueRef countv = LLVMBuildAnd(builder, maskvalue, countmask, "countv"); 463 LLVMTypeRef counttype = LLVMIntTypeInContext(context, type.length * 8); 464 LLVMTypeRef i8vntype = LLVMVectorType(LLVMInt8TypeInContext(context), type.length * 4); 465 LLVMValueRef shufflev, countd; 466 LLVMValueRef shuffles[16]; 467 const char *popcntintr = NULL; 468 469 countv = LLVMBuildBitCast(builder, countv, i8vntype, ""); 470 471 for (i = 0; i < type.length; i++) { 472 shuffles[i] = lp_build_const_int32(gallivm, 4*i); 473 } 474 475 shufflev = LLVMConstVector(shuffles, type.length); 476 countd = LLVMBuildShuffleVector(builder, countv, LLVMGetUndef(i8vntype), shufflev, ""); 477 countd = LLVMBuildBitCast(builder, countd, counttype, "countd"); 478 479 /* 480 * XXX FIXME 481 * this is bad on cpus without popcount (on x86 supported by intel 482 * nehalem, amd barcelona, and up - not tied to sse42). 483 * Would be much faster to just sum the 4 elements of the vector with 484 * some horizontal add (shuffle/add/shuffle/add after the initial and). 485 */ 486 switch (type.length) { 487 case 4: 488 popcntintr = "llvm.ctpop.i32"; 489 break; 490 case 8: 491 popcntintr = "llvm.ctpop.i64"; 492 break; 493 case 16: 494 popcntintr = "llvm.ctpop.i128"; 495 break; 496 default: 497 assert(0); 498 } 499 count = lp_build_intrinsic_unary(builder, popcntintr, counttype, countd); 500 501 if (type.length > 8) { 502 count = LLVMBuildTrunc(builder, count, LLVMIntTypeInContext(context, 64), ""); 503 } 504 else if (type.length < 8) { 505 count = LLVMBuildZExt(builder, count, LLVMIntTypeInContext(context, 64), ""); 506 } 507 } 508 newcount = LLVMBuildLoad(builder, counter, "origcount"); 509 newcount = LLVMBuildAdd(builder, newcount, count, "newcount"); 510 LLVMBuildStore(builder, newcount, counter); 511 } 512 513 514 /** 515 * Load depth/stencil values. 516 * The stored values are linear, swizzle them. 517 * 518 * \param type the data type of the fragment depth/stencil values 519 * \param format_desc description of the depth/stencil surface 520 * \param is_1d whether this resource has only one dimension 521 * \param loop_counter the current loop iteration 522 * \param depth_ptr pointer to the depth/stencil values of this 4x4 block 523 * \param depth_stride stride of the depth/stencil buffer 524 * \param z_fb contains z values loaded from fb (may include padding) 525 * \param s_fb contains s values loaded from fb (may include padding) 526 */ 527 void 528 lp_build_depth_stencil_load_swizzled(struct gallivm_state *gallivm, 529 struct lp_type z_src_type, 530 const struct util_format_description *format_desc, 531 boolean is_1d, 532 LLVMValueRef depth_ptr, 533 LLVMValueRef depth_stride, 534 LLVMValueRef *z_fb, 535 LLVMValueRef *s_fb, 536 LLVMValueRef loop_counter) 537 { 538 LLVMBuilderRef builder = gallivm->builder; 539 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 4]; 540 LLVMValueRef zs_dst1, zs_dst2; 541 LLVMValueRef zs_dst_ptr; 542 LLVMValueRef depth_offset1, depth_offset2; 543 LLVMTypeRef load_ptr_type; 544 unsigned depth_bytes = format_desc->block.bits / 8; 545 struct lp_type zs_type = lp_depth_type(format_desc, z_src_type.length); 546 struct lp_type zs_load_type = zs_type; 547 548 zs_load_type.length = zs_load_type.length / 2; 549 load_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, zs_load_type), 0); 550 551 if (z_src_type.length == 4) { 552 unsigned i; 553 LLVMValueRef looplsb = LLVMBuildAnd(builder, loop_counter, 554 lp_build_const_int32(gallivm, 1), ""); 555 LLVMValueRef loopmsb = LLVMBuildAnd(builder, loop_counter, 556 lp_build_const_int32(gallivm, 2), ""); 557 LLVMValueRef offset2 = LLVMBuildMul(builder, loopmsb, 558 depth_stride, ""); 559 depth_offset1 = LLVMBuildMul(builder, looplsb, 560 lp_build_const_int32(gallivm, depth_bytes * 2), ""); 561 depth_offset1 = LLVMBuildAdd(builder, depth_offset1, offset2, ""); 562 563 /* just concatenate the loaded 2x2 values into 4-wide vector */ 564 for (i = 0; i < 4; i++) { 565 shuffles[i] = lp_build_const_int32(gallivm, i); 566 } 567 } 568 else { 569 unsigned i; 570 LLVMValueRef loopx2 = LLVMBuildShl(builder, loop_counter, 571 lp_build_const_int32(gallivm, 1), ""); 572 assert(z_src_type.length == 8); 573 depth_offset1 = LLVMBuildMul(builder, loopx2, depth_stride, ""); 574 /* 575 * We load 2x4 values, and need to swizzle them (order 576 * 0,1,4,5,2,3,6,7) - not so hot with avx unfortunately. 577 */ 578 for (i = 0; i < 8; i++) { 579 shuffles[i] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2); 580 } 581 } 582 583 depth_offset2 = LLVMBuildAdd(builder, depth_offset1, depth_stride, ""); 584 585 /* Load current z/stencil values from z/stencil buffer */ 586 zs_dst_ptr = LLVMBuildGEP(builder, depth_ptr, &depth_offset1, 1, ""); 587 zs_dst_ptr = LLVMBuildBitCast(builder, zs_dst_ptr, load_ptr_type, ""); 588 zs_dst1 = LLVMBuildLoad(builder, zs_dst_ptr, ""); 589 if (is_1d) { 590 zs_dst2 = lp_build_undef(gallivm, zs_load_type); 591 } 592 else { 593 zs_dst_ptr = LLVMBuildGEP(builder, depth_ptr, &depth_offset2, 1, ""); 594 zs_dst_ptr = LLVMBuildBitCast(builder, zs_dst_ptr, load_ptr_type, ""); 595 zs_dst2 = LLVMBuildLoad(builder, zs_dst_ptr, ""); 596 } 597 598 *z_fb = LLVMBuildShuffleVector(builder, zs_dst1, zs_dst2, 599 LLVMConstVector(shuffles, zs_type.length), ""); 600 *s_fb = *z_fb; 601 602 if (format_desc->block.bits < z_src_type.width) { 603 /* Extend destination ZS values (e.g., when reading from Z16_UNORM) */ 604 *z_fb = LLVMBuildZExt(builder, *z_fb, 605 lp_build_int_vec_type(gallivm, z_src_type), ""); 606 } 607 608 else if (format_desc->block.bits > 32) { 609 /* rely on llvm to handle too wide vector we have here nicely */ 610 unsigned i; 611 struct lp_type typex2 = zs_type; 612 struct lp_type s_type = zs_type; 613 LLVMValueRef shuffles1[LP_MAX_VECTOR_LENGTH / 4]; 614 LLVMValueRef shuffles2[LP_MAX_VECTOR_LENGTH / 4]; 615 LLVMValueRef tmp; 616 617 typex2.width = typex2.width / 2; 618 typex2.length = typex2.length * 2; 619 s_type.width = s_type.width / 2; 620 s_type.floating = 0; 621 622 tmp = LLVMBuildBitCast(builder, *z_fb, 623 lp_build_vec_type(gallivm, typex2), ""); 624 625 for (i = 0; i < zs_type.length; i++) { 626 shuffles1[i] = lp_build_const_int32(gallivm, i * 2); 627 shuffles2[i] = lp_build_const_int32(gallivm, i * 2 + 1); 628 } 629 *z_fb = LLVMBuildShuffleVector(builder, tmp, tmp, 630 LLVMConstVector(shuffles1, zs_type.length), ""); 631 *s_fb = LLVMBuildShuffleVector(builder, tmp, tmp, 632 LLVMConstVector(shuffles2, zs_type.length), ""); 633 *s_fb = LLVMBuildBitCast(builder, *s_fb, 634 lp_build_vec_type(gallivm, s_type), ""); 635 lp_build_name(*s_fb, "s_dst"); 636 } 637 638 lp_build_name(*z_fb, "z_dst"); 639 lp_build_name(*s_fb, "s_dst"); 640 lp_build_name(*z_fb, "z_dst"); 641 } 642 643 /** 644 * Store depth/stencil values. 645 * Incoming values are swizzled (typically n 2x2 quads), stored linear. 646 * If there's a mask it will do select/store otherwise just store. 647 * 648 * \param type the data type of the fragment depth/stencil values 649 * \param format_desc description of the depth/stencil surface 650 * \param is_1d whether this resource has only one dimension 651 * \param mask the alive/dead pixel mask for the quad (vector) 652 * \param z_fb z values read from fb (with padding) 653 * \param s_fb s values read from fb (with padding) 654 * \param loop_counter the current loop iteration 655 * \param depth_ptr pointer to the depth/stencil values of this 4x4 block 656 * \param depth_stride stride of the depth/stencil buffer 657 * \param z_value the depth values to store (with padding) 658 * \param s_value the stencil values to store (with padding) 659 */ 660 void 661 lp_build_depth_stencil_write_swizzled(struct gallivm_state *gallivm, 662 struct lp_type z_src_type, 663 const struct util_format_description *format_desc, 664 boolean is_1d, 665 struct lp_build_mask_context *mask, 666 LLVMValueRef z_fb, 667 LLVMValueRef s_fb, 668 LLVMValueRef loop_counter, 669 LLVMValueRef depth_ptr, 670 LLVMValueRef depth_stride, 671 LLVMValueRef z_value, 672 LLVMValueRef s_value) 673 { 674 struct lp_build_context z_bld; 675 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 4]; 676 LLVMBuilderRef builder = gallivm->builder; 677 LLVMValueRef mask_value = NULL; 678 LLVMValueRef zs_dst1, zs_dst2; 679 LLVMValueRef zs_dst_ptr1, zs_dst_ptr2; 680 LLVMValueRef depth_offset1, depth_offset2; 681 LLVMTypeRef load_ptr_type; 682 unsigned depth_bytes = format_desc->block.bits / 8; 683 struct lp_type zs_type = lp_depth_type(format_desc, z_src_type.length); 684 struct lp_type z_type = zs_type; 685 struct lp_type zs_load_type = zs_type; 686 687 zs_load_type.length = zs_load_type.length / 2; 688 load_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, zs_load_type), 0); 689 690 z_type.width = z_src_type.width; 691 692 lp_build_context_init(&z_bld, gallivm, z_type); 693 694 /* 695 * This is far from ideal, at least for late depth write we should do this 696 * outside the fs loop to avoid all the swizzle stuff. 697 */ 698 if (z_src_type.length == 4) { 699 LLVMValueRef looplsb = LLVMBuildAnd(builder, loop_counter, 700 lp_build_const_int32(gallivm, 1), ""); 701 LLVMValueRef loopmsb = LLVMBuildAnd(builder, loop_counter, 702 lp_build_const_int32(gallivm, 2), ""); 703 LLVMValueRef offset2 = LLVMBuildMul(builder, loopmsb, 704 depth_stride, ""); 705 depth_offset1 = LLVMBuildMul(builder, looplsb, 706 lp_build_const_int32(gallivm, depth_bytes * 2), ""); 707 depth_offset1 = LLVMBuildAdd(builder, depth_offset1, offset2, ""); 708 } 709 else { 710 unsigned i; 711 LLVMValueRef loopx2 = LLVMBuildShl(builder, loop_counter, 712 lp_build_const_int32(gallivm, 1), ""); 713 assert(z_src_type.length == 8); 714 depth_offset1 = LLVMBuildMul(builder, loopx2, depth_stride, ""); 715 /* 716 * We load 2x4 values, and need to swizzle them (order 717 * 0,1,4,5,2,3,6,7) - not so hot with avx unfortunately. 718 */ 719 for (i = 0; i < 8; i++) { 720 shuffles[i] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2); 721 } 722 } 723 724 depth_offset2 = LLVMBuildAdd(builder, depth_offset1, depth_stride, ""); 725 726 zs_dst_ptr1 = LLVMBuildGEP(builder, depth_ptr, &depth_offset1, 1, ""); 727 zs_dst_ptr1 = LLVMBuildBitCast(builder, zs_dst_ptr1, load_ptr_type, ""); 728 zs_dst_ptr2 = LLVMBuildGEP(builder, depth_ptr, &depth_offset2, 1, ""); 729 zs_dst_ptr2 = LLVMBuildBitCast(builder, zs_dst_ptr2, load_ptr_type, ""); 730 731 if (format_desc->block.bits > 32) { 732 s_value = LLVMBuildBitCast(builder, s_value, z_bld.vec_type, ""); 733 } 734 735 if (mask) { 736 mask_value = lp_build_mask_value(mask); 737 z_value = lp_build_select(&z_bld, mask_value, z_value, z_fb); 738 if (format_desc->block.bits > 32) { 739 s_fb = LLVMBuildBitCast(builder, s_fb, z_bld.vec_type, ""); 740 s_value = lp_build_select(&z_bld, mask_value, s_value, s_fb); 741 } 742 } 743 744 if (zs_type.width < z_src_type.width) { 745 /* Truncate ZS values (e.g., when writing to Z16_UNORM) */ 746 z_value = LLVMBuildTrunc(builder, z_value, 747 lp_build_int_vec_type(gallivm, zs_type), ""); 748 } 749 750 if (format_desc->block.bits <= 32) { 751 if (z_src_type.length == 4) { 752 zs_dst1 = lp_build_extract_range(gallivm, z_value, 0, 2); 753 zs_dst2 = lp_build_extract_range(gallivm, z_value, 2, 2); 754 } 755 else { 756 assert(z_src_type.length == 8); 757 zs_dst1 = LLVMBuildShuffleVector(builder, z_value, z_value, 758 LLVMConstVector(&shuffles[0], 759 zs_load_type.length), ""); 760 zs_dst2 = LLVMBuildShuffleVector(builder, z_value, z_value, 761 LLVMConstVector(&shuffles[4], 762 zs_load_type.length), ""); 763 } 764 } 765 else { 766 if (z_src_type.length == 4) { 767 zs_dst1 = lp_build_interleave2(gallivm, z_type, 768 z_value, s_value, 0); 769 zs_dst2 = lp_build_interleave2(gallivm, z_type, 770 z_value, s_value, 1); 771 } 772 else { 773 unsigned i; 774 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH / 2]; 775 assert(z_src_type.length == 8); 776 for (i = 0; i < 8; i++) { 777 shuffles[i*2] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2); 778 shuffles[i*2+1] = lp_build_const_int32(gallivm, (i&1) + (i&2) * 2 + (i&4) / 2 + 779 z_src_type.length); 780 } 781 zs_dst1 = LLVMBuildShuffleVector(builder, z_value, s_value, 782 LLVMConstVector(&shuffles[0], 783 z_src_type.length), ""); 784 zs_dst2 = LLVMBuildShuffleVector(builder, z_value, s_value, 785 LLVMConstVector(&shuffles[8], 786 z_src_type.length), ""); 787 } 788 zs_dst1 = LLVMBuildBitCast(builder, zs_dst1, 789 lp_build_vec_type(gallivm, zs_load_type), ""); 790 zs_dst2 = LLVMBuildBitCast(builder, zs_dst2, 791 lp_build_vec_type(gallivm, zs_load_type), ""); 792 } 793 794 LLVMBuildStore(builder, zs_dst1, zs_dst_ptr1); 795 if (!is_1d) { 796 LLVMBuildStore(builder, zs_dst2, zs_dst_ptr2); 797 } 798 } 799 800 /** 801 * Generate code for performing depth and/or stencil tests. 802 * We operate on a vector of values (typically n 2x2 quads). 803 * 804 * \param depth the depth test state 805 * \param stencil the front/back stencil state 806 * \param type the data type of the fragment depth/stencil values 807 * \param format_desc description of the depth/stencil surface 808 * \param mask the alive/dead pixel mask for the quad (vector) 809 * \param stencil_refs the front/back stencil ref values (scalar) 810 * \param z_src the incoming depth/stencil values (n 2x2 quad values, float32) 811 * \param zs_dst the depth/stencil values in framebuffer 812 * \param face contains boolean value indicating front/back facing polygon 813 */ 814 void 815 lp_build_depth_stencil_test(struct gallivm_state *gallivm, 816 const struct pipe_depth_state *depth, 817 const struct pipe_stencil_state stencil[2], 818 struct lp_type z_src_type, 819 const struct util_format_description *format_desc, 820 struct lp_build_mask_context *mask, 821 LLVMValueRef stencil_refs[2], 822 LLVMValueRef z_src, 823 LLVMValueRef z_fb, 824 LLVMValueRef s_fb, 825 LLVMValueRef face, 826 LLVMValueRef *z_value, 827 LLVMValueRef *s_value, 828 boolean do_branch) 829 { 830 LLVMBuilderRef builder = gallivm->builder; 831 struct lp_type z_type; 832 struct lp_build_context z_bld; 833 struct lp_build_context s_bld; 834 struct lp_type s_type; 835 unsigned z_shift = 0, z_width = 0, z_mask = 0; 836 LLVMValueRef z_dst = NULL; 837 LLVMValueRef stencil_vals = NULL; 838 LLVMValueRef z_bitmask = NULL, stencil_shift = NULL; 839 LLVMValueRef z_pass = NULL, s_pass_mask = NULL; 840 LLVMValueRef current_mask = lp_build_mask_value(mask); 841 LLVMValueRef front_facing = NULL; 842 boolean have_z, have_s; 843 844 /* 845 * Depths are expected to be between 0 and 1, even if they are stored in 846 * floats. Setting these bits here will ensure that the lp_build_conv() call 847 * below won't try to unnecessarily clamp the incoming values. 848 */ 849 if(z_src_type.floating) { 850 z_src_type.sign = FALSE; 851 z_src_type.norm = TRUE; 852 } 853 else { 854 assert(!z_src_type.sign); 855 assert(z_src_type.norm); 856 } 857 858 /* Pick the type matching the depth-stencil format. */ 859 z_type = lp_depth_type(format_desc, z_src_type.length); 860 861 /* Pick the intermediate type for depth operations. */ 862 z_type.width = z_src_type.width; 863 assert(z_type.length == z_src_type.length); 864 865 /* FIXME: for non-float depth/stencil might generate better code 866 * if we'd always split it up to use 128bit operations. 867 * For stencil we'd almost certainly want to pack to 8xi16 values, 868 * for z just run twice. 869 */ 870 871 /* Sanity checking */ 872 { 873 const unsigned z_swizzle = format_desc->swizzle[0]; 874 const unsigned s_swizzle = format_desc->swizzle[1]; 875 876 assert(z_swizzle != PIPE_SWIZZLE_NONE || 877 s_swizzle != PIPE_SWIZZLE_NONE); 878 879 assert(depth->enabled || stencil[0].enabled); 880 881 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_ZS); 882 assert(format_desc->block.width == 1); 883 assert(format_desc->block.height == 1); 884 885 if (stencil[0].enabled) { 886 assert(s_swizzle < 4); 887 assert(format_desc->channel[s_swizzle].type == UTIL_FORMAT_TYPE_UNSIGNED); 888 assert(format_desc->channel[s_swizzle].pure_integer); 889 assert(!format_desc->channel[s_swizzle].normalized); 890 assert(format_desc->channel[s_swizzle].size == 8); 891 } 892 893 if (depth->enabled) { 894 assert(z_swizzle < 4); 895 if (z_type.floating) { 896 assert(z_swizzle == 0); 897 assert(format_desc->channel[z_swizzle].type == 898 UTIL_FORMAT_TYPE_FLOAT); 899 assert(format_desc->channel[z_swizzle].size == 32); 900 } 901 else { 902 assert(format_desc->channel[z_swizzle].type == 903 UTIL_FORMAT_TYPE_UNSIGNED); 904 assert(format_desc->channel[z_swizzle].normalized); 905 assert(!z_type.fixed); 906 } 907 } 908 } 909 910 911 /* Setup build context for Z vals */ 912 lp_build_context_init(&z_bld, gallivm, z_type); 913 914 /* Setup build context for stencil vals */ 915 s_type = lp_int_type(z_type); 916 lp_build_context_init(&s_bld, gallivm, s_type); 917 918 /* Compute and apply the Z/stencil bitmasks and shifts. 919 */ 920 { 921 unsigned s_shift, s_mask; 922 923 z_dst = z_fb; 924 stencil_vals = s_fb; 925 926 have_z = get_z_shift_and_mask(format_desc, &z_shift, &z_width, &z_mask); 927 have_s = get_s_shift_and_mask(format_desc, &s_shift, &s_mask); 928 929 if (have_z) { 930 if (z_mask != 0xffffffff) { 931 z_bitmask = lp_build_const_int_vec(gallivm, z_type, z_mask); 932 } 933 934 /* 935 * Align the framebuffer Z 's LSB to the right. 936 */ 937 if (z_shift) { 938 LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_type, z_shift); 939 z_dst = LLVMBuildLShr(builder, z_dst, shift, "z_dst"); 940 } else if (z_bitmask) { 941 z_dst = LLVMBuildAnd(builder, z_dst, z_bitmask, "z_dst"); 942 } else { 943 lp_build_name(z_dst, "z_dst"); 944 } 945 } 946 947 if (have_s) { 948 if (s_shift) { 949 LLVMValueRef shift = lp_build_const_int_vec(gallivm, s_type, s_shift); 950 stencil_vals = LLVMBuildLShr(builder, stencil_vals, shift, ""); 951 stencil_shift = shift; /* used below */ 952 } 953 954 if (s_mask != 0xffffffff) { 955 LLVMValueRef mask = lp_build_const_int_vec(gallivm, s_type, s_mask); 956 stencil_vals = LLVMBuildAnd(builder, stencil_vals, mask, ""); 957 } 958 959 lp_build_name(stencil_vals, "s_dst"); 960 } 961 } 962 963 if (stencil[0].enabled) { 964 965 if (face) { 966 if (0) { 967 /* 968 * XXX: the scalar expansion below produces atrocious code 969 * (basically producing a 64bit scalar value, then moving the 2 970 * 32bit pieces separately to simd, plus 4 shuffles, which is 971 * seriously lame). But the scalar-simd transitions are always 972 * tricky, so no big surprise there. 973 * This here would be way better, however llvm has some serious 974 * trouble later using it in the select, probably because it will 975 * recognize the expression as constant and move the simd value 976 * away (out of the loop) - and then it will suddenly try 977 * constructing i1 high-bit masks out of it later... 978 * (Try piglit stencil-twoside.) 979 * Note this is NOT due to using SExt/Trunc, it fails exactly the 980 * same even when using native compare/select. 981 * I cannot reproduce this problem when using stand-alone compiler 982 * though, suggesting some problem with optimization passes... 983 * (With stand-alone compilation, the construction of this mask 984 * value, no matter if the easy 3 instruction here or the complex 985 * 16+ one below, never gets separated from where it's used.) 986 * The scalar code still has the same problem, but the generated 987 * code looks a bit better at least for some reason, even if 988 * mostly by luck (the fundamental issue clearly is the same). 989 */ 990 front_facing = lp_build_broadcast(gallivm, s_bld.vec_type, face); 991 /* front_facing = face != 0 ? ~0 : 0 */ 992 front_facing = lp_build_compare(gallivm, s_bld.type, 993 PIPE_FUNC_NOTEQUAL, 994 front_facing, s_bld.zero); 995 } else { 996 LLVMValueRef zero = lp_build_const_int32(gallivm, 0); 997 998 /* front_facing = face != 0 ? ~0 : 0 */ 999 front_facing = LLVMBuildICmp(builder, LLVMIntNE, face, zero, ""); 1000 front_facing = LLVMBuildSExt(builder, front_facing, 1001 LLVMIntTypeInContext(gallivm->context, 1002 s_bld.type.length*s_bld.type.width), 1003 ""); 1004 front_facing = LLVMBuildBitCast(builder, front_facing, 1005 s_bld.int_vec_type, ""); 1006 1007 } 1008 } 1009 1010 s_pass_mask = lp_build_stencil_test(&s_bld, stencil, 1011 stencil_refs, stencil_vals, 1012 front_facing); 1013 1014 /* apply stencil-fail operator */ 1015 { 1016 LLVMValueRef s_fail_mask = lp_build_andnot(&s_bld, current_mask, s_pass_mask); 1017 stencil_vals = lp_build_stencil_op(&s_bld, stencil, S_FAIL_OP, 1018 stencil_refs, stencil_vals, 1019 s_fail_mask, front_facing); 1020 } 1021 } 1022 1023 if (depth->enabled) { 1024 /* 1025 * Convert fragment Z to the desired type, aligning the LSB to the right. 1026 */ 1027 1028 assert(z_type.width == z_src_type.width); 1029 assert(z_type.length == z_src_type.length); 1030 assert(lp_check_value(z_src_type, z_src)); 1031 if (z_src_type.floating) { 1032 /* 1033 * Convert from floating point values 1034 */ 1035 1036 if (!z_type.floating) { 1037 z_src = lp_build_clamped_float_to_unsigned_norm(gallivm, 1038 z_src_type, 1039 z_width, 1040 z_src); 1041 } 1042 } else { 1043 /* 1044 * Convert from unsigned normalized values. 1045 */ 1046 1047 assert(!z_src_type.sign); 1048 assert(!z_src_type.fixed); 1049 assert(z_src_type.norm); 1050 assert(!z_type.floating); 1051 if (z_src_type.width > z_width) { 1052 LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_src_type, 1053 z_src_type.width - z_width); 1054 z_src = LLVMBuildLShr(builder, z_src, shift, ""); 1055 } 1056 } 1057 assert(lp_check_value(z_type, z_src)); 1058 1059 lp_build_name(z_src, "z_src"); 1060 1061 /* compare src Z to dst Z, returning 'pass' mask */ 1062 z_pass = lp_build_cmp(&z_bld, depth->func, z_src, z_dst); 1063 1064 /* mask off bits that failed stencil test */ 1065 if (s_pass_mask) { 1066 current_mask = LLVMBuildAnd(builder, current_mask, s_pass_mask, ""); 1067 } 1068 1069 if (!stencil[0].enabled) { 1070 /* We can potentially skip all remaining operations here, but only 1071 * if stencil is disabled because we still need to update the stencil 1072 * buffer values. Don't need to update Z buffer values. 1073 */ 1074 lp_build_mask_update(mask, z_pass); 1075 1076 if (do_branch) { 1077 lp_build_mask_check(mask); 1078 } 1079 } 1080 1081 if (depth->writemask) { 1082 LLVMValueRef z_pass_mask; 1083 1084 /* mask off bits that failed Z test */ 1085 z_pass_mask = LLVMBuildAnd(builder, current_mask, z_pass, ""); 1086 1087 /* Mix the old and new Z buffer values. 1088 * z_dst[i] = zselectmask[i] ? z_src[i] : z_dst[i] 1089 */ 1090 z_dst = lp_build_select(&z_bld, z_pass_mask, z_src, z_dst); 1091 } 1092 1093 if (stencil[0].enabled) { 1094 /* update stencil buffer values according to z pass/fail result */ 1095 LLVMValueRef z_fail_mask, z_pass_mask; 1096 1097 /* apply Z-fail operator */ 1098 z_fail_mask = lp_build_andnot(&s_bld, current_mask, z_pass); 1099 stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_FAIL_OP, 1100 stencil_refs, stencil_vals, 1101 z_fail_mask, front_facing); 1102 1103 /* apply Z-pass operator */ 1104 z_pass_mask = LLVMBuildAnd(builder, current_mask, z_pass, ""); 1105 stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_PASS_OP, 1106 stencil_refs, stencil_vals, 1107 z_pass_mask, front_facing); 1108 } 1109 } 1110 else { 1111 /* No depth test: apply Z-pass operator to stencil buffer values which 1112 * passed the stencil test. 1113 */ 1114 s_pass_mask = LLVMBuildAnd(builder, current_mask, s_pass_mask, ""); 1115 stencil_vals = lp_build_stencil_op(&s_bld, stencil, Z_PASS_OP, 1116 stencil_refs, stencil_vals, 1117 s_pass_mask, front_facing); 1118 } 1119 1120 /* Put Z and stencil bits in the right place */ 1121 if (have_z && z_shift) { 1122 LLVMValueRef shift = lp_build_const_int_vec(gallivm, z_type, z_shift); 1123 z_dst = LLVMBuildShl(builder, z_dst, shift, ""); 1124 } 1125 if (stencil_vals && stencil_shift) 1126 stencil_vals = LLVMBuildShl(builder, stencil_vals, 1127 stencil_shift, ""); 1128 1129 /* Finally, merge the z/stencil values */ 1130 if (format_desc->block.bits <= 32) { 1131 if (have_z && have_s) 1132 *z_value = LLVMBuildOr(builder, z_dst, stencil_vals, ""); 1133 else if (have_z) 1134 *z_value = z_dst; 1135 else 1136 *z_value = stencil_vals; 1137 *s_value = *z_value; 1138 } 1139 else { 1140 *z_value = z_dst; 1141 *s_value = stencil_vals; 1142 } 1143 1144 if (s_pass_mask) 1145 lp_build_mask_update(mask, s_pass_mask); 1146 1147 if (depth->enabled && stencil[0].enabled) 1148 lp_build_mask_update(mask, z_pass); 1149 } 1150 1151
