1 /************************************************************************** 2 * 3 * Copyright 2010 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 #include "draw_llvm.h" 29 30 #include "draw_context.h" 31 #include "draw_vs.h" 32 33 #include "gallivm/lp_bld_arit.h" 34 #include "gallivm/lp_bld_logic.h" 35 #include "gallivm/lp_bld_const.h" 36 #include "gallivm/lp_bld_swizzle.h" 37 #include "gallivm/lp_bld_struct.h" 38 #include "gallivm/lp_bld_type.h" 39 #include "gallivm/lp_bld_flow.h" 40 #include "gallivm/lp_bld_debug.h" 41 #include "gallivm/lp_bld_tgsi.h" 42 #include "gallivm/lp_bld_printf.h" 43 #include "gallivm/lp_bld_intr.h" 44 #include "gallivm/lp_bld_init.h" 45 #include "gallivm/lp_bld_type.h" 46 #include "gallivm/lp_bld_pack.h" 47 #include "gallivm/lp_bld_format.h" 48 49 #include "tgsi/tgsi_exec.h" 50 #include "tgsi/tgsi_dump.h" 51 52 #include "util/u_math.h" 53 #include "util/u_pointer.h" 54 #include "util/u_string.h" 55 #include "util/u_simple_list.h" 56 57 58 #define DEBUG_STORE 0 59 60 61 static void 62 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var, 63 boolean elts); 64 65 66 /** 67 * Create LLVM type for struct draw_jit_texture 68 */ 69 static LLVMTypeRef 70 create_jit_texture_type(struct gallivm_state *gallivm, const char *struct_name) 71 { 72 LLVMTargetDataRef target = gallivm->target; 73 LLVMTypeRef texture_type; 74 LLVMTypeRef elem_types[DRAW_JIT_TEXTURE_NUM_FIELDS]; 75 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 76 77 elem_types[DRAW_JIT_TEXTURE_WIDTH] = 78 elem_types[DRAW_JIT_TEXTURE_HEIGHT] = 79 elem_types[DRAW_JIT_TEXTURE_DEPTH] = 80 elem_types[DRAW_JIT_TEXTURE_FIRST_LEVEL] = 81 elem_types[DRAW_JIT_TEXTURE_LAST_LEVEL] = int32_type; 82 elem_types[DRAW_JIT_TEXTURE_ROW_STRIDE] = 83 elem_types[DRAW_JIT_TEXTURE_IMG_STRIDE] = 84 LLVMArrayType(int32_type, PIPE_MAX_TEXTURE_LEVELS); 85 elem_types[DRAW_JIT_TEXTURE_DATA] = 86 LLVMArrayType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), 87 PIPE_MAX_TEXTURE_LEVELS); 88 elem_types[DRAW_JIT_TEXTURE_MIN_LOD] = 89 elem_types[DRAW_JIT_TEXTURE_MAX_LOD] = 90 elem_types[DRAW_JIT_TEXTURE_LOD_BIAS] = LLVMFloatTypeInContext(gallivm->context); 91 elem_types[DRAW_JIT_TEXTURE_BORDER_COLOR] = 92 LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 93 94 texture_type = LLVMStructTypeInContext(gallivm->context, elem_types, 95 Elements(elem_types), 0); 96 97 #if HAVE_LLVM < 0x0300 98 LLVMAddTypeName(gallivm->module, struct_name, texture_type); 99 100 /* Make sure the target's struct layout cache doesn't return 101 * stale/invalid data. 102 */ 103 LLVMInvalidateStructLayout(gallivm->target, texture_type); 104 #endif 105 106 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width, 107 target, texture_type, 108 DRAW_JIT_TEXTURE_WIDTH); 109 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height, 110 target, texture_type, 111 DRAW_JIT_TEXTURE_HEIGHT); 112 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, depth, 113 target, texture_type, 114 DRAW_JIT_TEXTURE_DEPTH); 115 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, first_level, 116 target, texture_type, 117 DRAW_JIT_TEXTURE_FIRST_LEVEL); 118 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, last_level, 119 target, texture_type, 120 DRAW_JIT_TEXTURE_LAST_LEVEL); 121 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, row_stride, 122 target, texture_type, 123 DRAW_JIT_TEXTURE_ROW_STRIDE); 124 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, img_stride, 125 target, texture_type, 126 DRAW_JIT_TEXTURE_IMG_STRIDE); 127 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data, 128 target, texture_type, 129 DRAW_JIT_TEXTURE_DATA); 130 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, min_lod, 131 target, texture_type, 132 DRAW_JIT_TEXTURE_MIN_LOD); 133 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, max_lod, 134 target, texture_type, 135 DRAW_JIT_TEXTURE_MAX_LOD); 136 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, lod_bias, 137 target, texture_type, 138 DRAW_JIT_TEXTURE_LOD_BIAS); 139 LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, border_color, 140 target, texture_type, 141 DRAW_JIT_TEXTURE_BORDER_COLOR); 142 143 LP_CHECK_STRUCT_SIZE(struct draw_jit_texture, target, texture_type); 144 145 return texture_type; 146 } 147 148 149 /** 150 * Create LLVM type for struct draw_jit_texture 151 */ 152 static LLVMTypeRef 153 create_jit_context_type(struct gallivm_state *gallivm, 154 LLVMTypeRef texture_type, const char *struct_name) 155 { 156 LLVMTargetDataRef target = gallivm->target; 157 LLVMTypeRef float_type = LLVMFloatTypeInContext(gallivm->context); 158 LLVMTypeRef elem_types[5]; 159 LLVMTypeRef context_type; 160 161 elem_types[0] = LLVMPointerType(float_type, 0); /* vs_constants */ 162 elem_types[1] = LLVMPointerType(float_type, 0); /* gs_constants */ 163 elem_types[2] = LLVMPointerType(LLVMArrayType(LLVMArrayType(float_type, 4), 164 DRAW_TOTAL_CLIP_PLANES), 0); 165 elem_types[3] = LLVMPointerType(float_type, 0); /* viewport */ 166 elem_types[4] = LLVMArrayType(texture_type, 167 PIPE_MAX_SAMPLERS); /* textures */ 168 context_type = LLVMStructTypeInContext(gallivm->context, elem_types, 169 Elements(elem_types), 0); 170 #if HAVE_LLVM < 0x0300 171 LLVMAddTypeName(gallivm->module, struct_name, context_type); 172 173 LLVMInvalidateStructLayout(gallivm->target, context_type); 174 #endif 175 176 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants, 177 target, context_type, 0); 178 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants, 179 target, context_type, 1); 180 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, planes, 181 target, context_type, 2); 182 LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures, 183 target, context_type, 184 DRAW_JIT_CTX_TEXTURES); 185 LP_CHECK_STRUCT_SIZE(struct draw_jit_context, 186 target, context_type); 187 188 return context_type; 189 } 190 191 192 /** 193 * Create LLVM type for struct pipe_vertex_buffer 194 */ 195 static LLVMTypeRef 196 create_jit_vertex_buffer_type(struct gallivm_state *gallivm, const char *struct_name) 197 { 198 LLVMTargetDataRef target = gallivm->target; 199 LLVMTypeRef elem_types[4]; 200 LLVMTypeRef vb_type; 201 202 elem_types[0] = 203 elem_types[1] = LLVMInt32TypeInContext(gallivm->context); 204 elem_types[2] = 205 elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0); /* vs_constants */ 206 207 vb_type = LLVMStructTypeInContext(gallivm->context, elem_types, 208 Elements(elem_types), 0); 209 #if HAVE_LLVM < 0x0300 210 LLVMAddTypeName(gallivm->module, struct_name, vb_type); 211 212 LLVMInvalidateStructLayout(gallivm->target, vb_type); 213 #endif 214 215 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride, 216 target, vb_type, 0); 217 LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset, 218 target, vb_type, 1); 219 220 LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type); 221 222 return vb_type; 223 } 224 225 226 /** 227 * Create LLVM type for struct vertex_header; 228 */ 229 static LLVMTypeRef 230 create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems) 231 { 232 LLVMTargetDataRef target = gallivm->target; 233 LLVMTypeRef elem_types[4]; 234 LLVMTypeRef vertex_header; 235 char struct_name[24]; 236 237 util_snprintf(struct_name, 23, "vertex_header%d", data_elems); 238 239 elem_types[DRAW_JIT_VERTEX_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32); 240 elem_types[DRAW_JIT_VERTEX_CLIP] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 241 elem_types[DRAW_JIT_VERTEX_PRE_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4); 242 elem_types[DRAW_JIT_VERTEX_DATA] = LLVMArrayType(elem_types[1], data_elems); 243 244 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types, 245 Elements(elem_types), 0); 246 #if HAVE_LLVM < 0x0300 247 LLVMAddTypeName(gallivm->module, struct_name, vertex_header); 248 249 LLVMInvalidateStructLayout(gallivm->target, vertex_header); 250 #endif 251 252 /* these are bit-fields and we can't take address of them 253 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask, 254 target, vertex_header, 255 DRAW_JIT_VERTEX_CLIPMASK); 256 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag, 257 target, vertex_header, 258 DRAW_JIT_VERTEX_EDGEFLAG); 259 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad, 260 target, vertex_header, 261 DRAW_JIT_VERTEX_PAD); 262 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id, 263 target, vertex_header, 264 DRAW_JIT_VERTEX_VERTEX_ID); 265 */ 266 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip, 267 target, vertex_header, 268 DRAW_JIT_VERTEX_CLIP); 269 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pre_clip_pos, 270 target, vertex_header, 271 DRAW_JIT_VERTEX_PRE_CLIP_POS); 272 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data, 273 target, vertex_header, 274 DRAW_JIT_VERTEX_DATA); 275 276 assert(LLVMABISizeOfType(target, vertex_header) == 277 offsetof(struct vertex_header, data[data_elems])); 278 279 return vertex_header; 280 } 281 282 283 /** 284 * Create LLVM types for various structures. 285 */ 286 static void 287 create_jit_types(struct draw_llvm_variant *variant) 288 { 289 struct gallivm_state *gallivm = variant->gallivm; 290 LLVMTypeRef texture_type, context_type, buffer_type, vb_type; 291 292 texture_type = create_jit_texture_type(gallivm, "texture"); 293 294 context_type = create_jit_context_type(gallivm, texture_type, "draw_jit_context"); 295 variant->context_ptr_type = LLVMPointerType(context_type, 0); 296 297 buffer_type = LLVMPointerType(LLVMIntTypeInContext(gallivm->context, 8), 0); 298 variant->buffer_ptr_type = LLVMPointerType(buffer_type, 0); 299 300 vb_type = create_jit_vertex_buffer_type(gallivm, "pipe_vertex_buffer"); 301 variant->vb_ptr_type = LLVMPointerType(vb_type, 0); 302 } 303 304 305 static LLVMTypeRef 306 get_context_ptr_type(struct draw_llvm_variant *variant) 307 { 308 if (!variant->context_ptr_type) 309 create_jit_types(variant); 310 return variant->context_ptr_type; 311 } 312 313 314 static LLVMTypeRef 315 get_buffer_ptr_type(struct draw_llvm_variant *variant) 316 { 317 if (!variant->buffer_ptr_type) 318 create_jit_types(variant); 319 return variant->buffer_ptr_type; 320 } 321 322 323 static LLVMTypeRef 324 get_vb_ptr_type(struct draw_llvm_variant *variant) 325 { 326 if (!variant->vb_ptr_type) 327 create_jit_types(variant); 328 return variant->vb_ptr_type; 329 } 330 331 static LLVMTypeRef 332 get_vertex_header_ptr_type(struct draw_llvm_variant *variant) 333 { 334 if (!variant->vertex_header_ptr_type) 335 create_jit_types(variant); 336 return variant->vertex_header_ptr_type; 337 } 338 339 340 /** 341 * Create per-context LLVM info. 342 */ 343 struct draw_llvm * 344 draw_llvm_create(struct draw_context *draw) 345 { 346 struct draw_llvm *llvm; 347 348 llvm = CALLOC_STRUCT( draw_llvm ); 349 if (!llvm) 350 return NULL; 351 352 lp_build_init(); 353 354 llvm->draw = draw; 355 356 llvm->nr_variants = 0; 357 make_empty_list(&llvm->vs_variants_list); 358 359 return llvm; 360 } 361 362 363 /** 364 * Free per-context LLVM info. 365 */ 366 void 367 draw_llvm_destroy(struct draw_llvm *llvm) 368 { 369 /* XXX free other draw_llvm data? */ 370 FREE(llvm); 371 } 372 373 374 /** 375 * Create LLVM-generated code for a vertex shader. 376 */ 377 struct draw_llvm_variant * 378 draw_llvm_create_variant(struct draw_llvm *llvm, 379 unsigned num_inputs, 380 const struct draw_llvm_variant_key *key) 381 { 382 struct draw_llvm_variant *variant; 383 struct llvm_vertex_shader *shader = 384 llvm_vertex_shader(llvm->draw->vs.vertex_shader); 385 LLVMTypeRef vertex_header; 386 387 variant = MALLOC(sizeof *variant + 388 shader->variant_key_size - 389 sizeof variant->key); 390 if (variant == NULL) 391 return NULL; 392 393 variant->llvm = llvm; 394 395 variant->gallivm = gallivm_create(); 396 397 create_jit_types(variant); 398 399 memcpy(&variant->key, key, shader->variant_key_size); 400 401 vertex_header = create_jit_vertex_header(variant->gallivm, num_inputs); 402 403 variant->vertex_header_ptr_type = LLVMPointerType(vertex_header, 0); 404 405 draw_llvm_generate(llvm, variant, FALSE); /* linear */ 406 draw_llvm_generate(llvm, variant, TRUE); /* elts */ 407 408 gallivm_compile_module(variant->gallivm); 409 410 variant->jit_func = (draw_jit_vert_func) 411 gallivm_jit_function(variant->gallivm, variant->function); 412 413 variant->jit_func_elts = (draw_jit_vert_func_elts) 414 gallivm_jit_function(variant->gallivm, variant->function_elts); 415 416 variant->shader = shader; 417 variant->list_item_global.base = variant; 418 variant->list_item_local.base = variant; 419 /*variant->no = */shader->variants_created++; 420 variant->list_item_global.base = variant; 421 422 return variant; 423 } 424 425 426 static void 427 generate_vs(struct draw_llvm_variant *variant, 428 LLVMBuilderRef builder, 429 struct lp_type vs_type, 430 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 431 const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS], 432 const struct lp_bld_tgsi_system_values *system_values, 433 LLVMValueRef context_ptr, 434 struct lp_build_sampler_soa *draw_sampler, 435 boolean clamp_vertex_color) 436 { 437 struct draw_llvm *llvm = variant->llvm; 438 const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens; 439 LLVMValueRef consts_ptr = draw_jit_context_vs_constants(variant->gallivm, context_ptr); 440 struct lp_build_sampler_soa *sampler = 0; 441 442 if (gallivm_debug & GALLIVM_DEBUG_IR) { 443 tgsi_dump(tokens, 0); 444 } 445 446 if (llvm->draw->num_sampler_views && llvm->draw->num_samplers) 447 sampler = draw_sampler; 448 449 lp_build_tgsi_soa(variant->gallivm, 450 tokens, 451 vs_type, 452 NULL /*struct lp_build_mask_context *mask*/, 453 consts_ptr, 454 system_values, 455 NULL /*pos*/, 456 inputs, 457 outputs, 458 sampler, 459 &llvm->draw->vs.vertex_shader->info); 460 461 { 462 LLVMValueRef out; 463 unsigned chan, attrib; 464 struct lp_build_context bld; 465 struct tgsi_shader_info* info = &llvm->draw->vs.vertex_shader->info; 466 lp_build_context_init(&bld, variant->gallivm, vs_type); 467 468 for (attrib = 0; attrib < info->num_outputs; ++attrib) { 469 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 470 if (outputs[attrib][chan]) { 471 switch (info->output_semantic_name[attrib]) { 472 case TGSI_SEMANTIC_COLOR: 473 case TGSI_SEMANTIC_BCOLOR: 474 if (clamp_vertex_color) { 475 out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 476 out = lp_build_clamp(&bld, out, bld.zero, bld.one); 477 LLVMBuildStore(builder, out, outputs[attrib][chan]); 478 } 479 break; 480 case TGSI_SEMANTIC_FOG: 481 if (chan == 1 || chan == 2) 482 LLVMBuildStore(builder, bld.zero, outputs[attrib][chan]); 483 else if (chan == 3) 484 LLVMBuildStore(builder, bld.one, outputs[attrib][chan]); 485 break; 486 } 487 } 488 } 489 } 490 } 491 } 492 493 494 static void 495 generate_fetch(struct gallivm_state *gallivm, 496 LLVMValueRef vbuffers_ptr, 497 LLVMValueRef *res, 498 struct pipe_vertex_element *velem, 499 LLVMValueRef vbuf, 500 LLVMValueRef index, 501 LLVMValueRef instance_id) 502 { 503 const struct util_format_description *format_desc = util_format_description(velem->src_format); 504 LLVMValueRef zero = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context)); 505 LLVMBuilderRef builder = gallivm->builder; 506 LLVMValueRef indices = 507 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 508 velem->vertex_buffer_index, 0); 509 LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, 510 &indices, 1, ""); 511 LLVMValueRef vb_stride = draw_jit_vbuffer_stride(gallivm, vbuf); 512 LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vbuf); 513 LLVMValueRef stride; 514 515 if (velem->instance_divisor) { 516 /* array index = instance_id / instance_divisor */ 517 index = LLVMBuildUDiv(builder, instance_id, 518 lp_build_const_int32(gallivm, velem->instance_divisor), 519 "instance_divisor"); 520 } 521 522 stride = LLVMBuildMul(builder, vb_stride, index, ""); 523 524 vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer"); 525 526 stride = LLVMBuildAdd(builder, stride, 527 vb_buffer_offset, 528 ""); 529 stride = LLVMBuildAdd(builder, stride, 530 lp_build_const_int32(gallivm, velem->src_offset), 531 ""); 532 533 /* lp_build_printf(gallivm, "vbuf index = %d, stride is %d\n", indices, stride);*/ 534 vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, ""); 535 536 *res = lp_build_fetch_rgba_aos(gallivm, 537 format_desc, 538 lp_float32_vec4_type(), 539 vbuffer_ptr, 540 zero, zero, zero); 541 } 542 543 static void 544 convert_to_soa(struct gallivm_state *gallivm, 545 LLVMValueRef (*src_aos)[LP_MAX_VECTOR_WIDTH / 32], 546 LLVMValueRef (*dst_soa)[TGSI_NUM_CHANNELS], 547 unsigned num_attribs, const struct lp_type soa_type) 548 { 549 unsigned i, j, k; 550 struct lp_type aos_channel_type = soa_type; 551 552 debug_assert(TGSI_NUM_CHANNELS == 4); 553 debug_assert((soa_type.length % TGSI_NUM_CHANNELS) == 0); 554 555 aos_channel_type.length >>= 1; 556 557 for (i = 0; i < num_attribs; ++i) { 558 LLVMValueRef aos_channels[TGSI_NUM_CHANNELS]; 559 unsigned pixels_per_channel = soa_type.length / TGSI_NUM_CHANNELS; 560 561 for (j = 0; j < TGSI_NUM_CHANNELS; ++j) { 562 LLVMValueRef channel[LP_MAX_VECTOR_LENGTH] = { 0 }; 563 564 assert(pixels_per_channel <= LP_MAX_VECTOR_LENGTH); 565 566 for (k = 0; k < pixels_per_channel; ++k) { 567 channel[k] = src_aos[i][j + TGSI_NUM_CHANNELS * k]; 568 } 569 570 aos_channels[j] = lp_build_concat(gallivm, channel, aos_channel_type, pixels_per_channel); 571 } 572 573 lp_build_transpose_aos(gallivm, soa_type, aos_channels, dst_soa[i]); 574 } 575 } 576 577 578 static void 579 store_aos(struct gallivm_state *gallivm, 580 LLVMValueRef io_ptr, 581 LLVMValueRef index, 582 LLVMValueRef value) 583 { 584 LLVMTypeRef data_ptr_type = LLVMPointerType(lp_build_vec_type(gallivm, lp_float32_vec4_type()), 0); 585 LLVMBuilderRef builder = gallivm->builder; 586 LLVMValueRef data_ptr = draw_jit_header_data(gallivm, io_ptr); 587 LLVMValueRef indices[3]; 588 589 indices[0] = lp_build_const_int32(gallivm, 0); 590 indices[1] = index; 591 indices[2] = lp_build_const_int32(gallivm, 0); 592 593 #if DEBUG_STORE 594 lp_build_printf(gallivm, " ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr); 595 #endif 596 597 data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, ""); 598 data_ptr = LLVMBuildPointerCast(builder, data_ptr, data_ptr_type, ""); 599 600 /* Unaligned store due to the vertex header */ 601 lp_set_store_alignment(LLVMBuildStore(builder, value, data_ptr), sizeof(float)); 602 } 603 604 605 static void 606 store_aos_array(struct gallivm_state *gallivm, 607 struct lp_type soa_type, 608 LLVMValueRef io_ptr, 609 LLVMValueRef* aos, 610 int attrib, 611 int num_outputs, 612 LLVMValueRef clipmask, 613 boolean have_clipdist) 614 { 615 LLVMBuilderRef builder = gallivm->builder; 616 LLVMValueRef attr_index = lp_build_const_int32(gallivm, attrib); 617 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32]; 618 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32]; 619 int vector_length = soa_type.length; 620 int i; 621 622 debug_assert(TGSI_NUM_CHANNELS == 4); 623 624 for (i = 0; i < vector_length; i++) { 625 inds[i] = lp_build_const_int32(gallivm, i); 626 io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, ""); 627 } 628 629 if (attrib == 0) { 630 /* store vertex header for each of the n vertices */ 631 LLVMValueRef val, cliptmp; 632 int vertex_id_pad_edgeflag; 633 634 /* If this assertion fails, it means we need to update the bit twidding 635 * code here. See struct vertex_header in draw_private.h. 636 */ 637 assert(DRAW_TOTAL_CLIP_PLANES==14); 638 /* initialize vertex id:16 = 0xffff, have_clipdist:1 = 0, edgeflag:1 = 1 */ 639 vertex_id_pad_edgeflag = (0xffff << 16) | (1 << DRAW_TOTAL_CLIP_PLANES); 640 if (have_clipdist) 641 vertex_id_pad_edgeflag |= 1 << (DRAW_TOTAL_CLIP_PLANES+1); 642 val = lp_build_const_int_vec(gallivm, lp_int_type(soa_type), vertex_id_pad_edgeflag); 643 /* OR with the clipmask */ 644 cliptmp = LLVMBuildOr(builder, val, clipmask, ""); 645 for (i = 0; i < vector_length; i++) { 646 LLVMValueRef id_ptr = draw_jit_header_id(gallivm, io_ptrs[i]); 647 val = LLVMBuildExtractElement(builder, cliptmp, inds[i], ""); 648 LLVMBuildStore(builder, val, id_ptr); 649 #if DEBUG_STORE 650 lp_build_printf(gallivm, "io = %p, index %d\n, clipmask = %x\n", 651 io_ptrs[i], inds[i], val); 652 #endif 653 } 654 } 655 656 /* store for each of the n vertices */ 657 for (i = 0; i < vector_length; i++) { 658 store_aos(gallivm, io_ptrs[i], attr_index, aos[i]); 659 } 660 } 661 662 663 static void 664 convert_to_aos(struct gallivm_state *gallivm, 665 LLVMValueRef io, 666 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 667 LLVMValueRef clipmask, 668 int num_outputs, 669 struct lp_type soa_type, 670 boolean have_clipdist) 671 { 672 LLVMBuilderRef builder = gallivm->builder; 673 unsigned chan, attrib, i; 674 675 #if DEBUG_STORE 676 lp_build_printf(gallivm, " # storing begin\n"); 677 #endif 678 for (attrib = 0; attrib < num_outputs; ++attrib) { 679 LLVMValueRef soa[TGSI_NUM_CHANNELS]; 680 LLVMValueRef aos[LP_MAX_VECTOR_WIDTH / 32]; 681 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 682 if (outputs[attrib][chan]) { 683 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 684 lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]); 685 #if DEBUG_STORE 686 lp_build_printf(gallivm, "output %d : %d ", 687 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 688 attrib, 0), 689 LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 690 chan, 0)); 691 lp_build_print_value(gallivm, "val = ", out); 692 #endif 693 soa[chan] = out; 694 } 695 else { 696 soa[chan] = 0; 697 } 698 } 699 700 701 if (soa_type.length == TGSI_NUM_CHANNELS) { 702 lp_build_transpose_aos(gallivm, soa_type, soa, aos); 703 } else { 704 lp_build_transpose_aos(gallivm, soa_type, soa, soa); 705 706 for (i = 0; i < soa_type.length; ++i) { 707 aos[i] = lp_build_extract_range(gallivm, 708 soa[i % TGSI_NUM_CHANNELS], 709 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS, 710 TGSI_NUM_CHANNELS); 711 } 712 } 713 714 store_aos_array(gallivm, 715 soa_type, 716 io, 717 aos, 718 attrib, 719 num_outputs, 720 clipmask, have_clipdist); 721 } 722 #if DEBUG_STORE 723 lp_build_printf(gallivm, " # storing end\n"); 724 #endif 725 } 726 727 728 /** 729 * Stores original vertex positions in clip coordinates 730 */ 731 static void 732 store_clip(struct gallivm_state *gallivm, 733 const struct lp_type vs_type, 734 LLVMValueRef io_ptr, 735 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 736 boolean pre_clip_pos, int idx) 737 { 738 LLVMBuilderRef builder = gallivm->builder; 739 LLVMValueRef soa[4]; 740 LLVMValueRef aos[LP_MAX_VECTOR_LENGTH]; 741 LLVMValueRef indices[2]; 742 LLVMValueRef io_ptrs[LP_MAX_VECTOR_WIDTH / 32]; 743 LLVMValueRef inds[LP_MAX_VECTOR_WIDTH / 32]; 744 LLVMValueRef clip_ptrs[LP_MAX_VECTOR_WIDTH / 32]; 745 int i, j; 746 747 indices[0] = 748 indices[1] = lp_build_const_int32(gallivm, 0); 749 750 for (i = 0; i < vs_type.length; i++) { 751 inds[i] = lp_build_const_int32(gallivm, i); 752 io_ptrs[i] = LLVMBuildGEP(builder, io_ptr, &inds[i], 1, ""); 753 } 754 755 soa[0] = LLVMBuildLoad(builder, outputs[idx][0], ""); /*x0 x1 .. xn*/ 756 soa[1] = LLVMBuildLoad(builder, outputs[idx][1], ""); /*y0 y1 .. yn*/ 757 soa[2] = LLVMBuildLoad(builder, outputs[idx][2], ""); /*z0 z1 .. zn*/ 758 soa[3] = LLVMBuildLoad(builder, outputs[idx][3], ""); /*w0 w1 .. wn*/ 759 760 if (!pre_clip_pos) { 761 for (i = 0; i < vs_type.length; i++) { 762 clip_ptrs[i] = draw_jit_header_clip(gallivm, io_ptrs[i]); 763 } 764 } else { 765 for (i = 0; i < vs_type.length; i++) { 766 clip_ptrs[i] = draw_jit_header_pre_clip_pos(gallivm, io_ptrs[i]); 767 } 768 } 769 770 lp_build_transpose_aos(gallivm, vs_type, soa, soa); 771 for (i = 0; i < vs_type.length; ++i) { 772 aos[i] = lp_build_extract_range(gallivm, 773 soa[i % TGSI_NUM_CHANNELS], 774 (i / TGSI_NUM_CHANNELS) * TGSI_NUM_CHANNELS, 775 TGSI_NUM_CHANNELS); 776 } 777 778 for (j = 0; j < vs_type.length; j++) { 779 LLVMTypeRef clip_ptr_type = LLVMPointerType(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), 4), 0); 780 LLVMValueRef clip_ptr; 781 782 clip_ptr = LLVMBuildGEP(builder, clip_ptrs[j], indices, 2, "clipo"); 783 clip_ptr = LLVMBuildPointerCast(builder, clip_ptr, clip_ptr_type, ""); 784 785 /* Unaligned store */ 786 lp_set_store_alignment(LLVMBuildStore(builder, aos[j], clip_ptr), sizeof(float)); 787 } 788 } 789 790 791 /** 792 * Transforms the outputs for viewport mapping 793 */ 794 static void 795 generate_viewport(struct draw_llvm_variant *variant, 796 LLVMBuilderRef builder, 797 struct lp_type vs_type, 798 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 799 LLVMValueRef context_ptr) 800 { 801 int i; 802 struct gallivm_state *gallivm = variant->gallivm; 803 struct lp_type f32_type = vs_type; 804 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type); 805 LLVMValueRef out3 = LLVMBuildLoad(builder, outputs[0][3], ""); /*w0 w1 .. wn*/ 806 LLVMValueRef const1 = lp_build_const_vec(gallivm, f32_type, 1.0); /*1.0 1.0 1.0 1.0*/ 807 LLVMValueRef vp_ptr = draw_jit_context_viewport(gallivm, context_ptr); 808 809 /* for 1/w convention*/ 810 out3 = LLVMBuildFDiv(builder, const1, out3, ""); 811 LLVMBuildStore(builder, out3, outputs[0][3]); 812 813 /* Viewport Mapping */ 814 for (i=0; i<3; i++) { 815 LLVMValueRef out = LLVMBuildLoad(builder, outputs[0][i], ""); /*x0 x1 .. xn*/ 816 LLVMValueRef scale; 817 LLVMValueRef trans; 818 LLVMValueRef scale_i; 819 LLVMValueRef trans_i; 820 LLVMValueRef index; 821 822 index = lp_build_const_int32(gallivm, i); 823 scale_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 824 825 index = lp_build_const_int32(gallivm, i+4); 826 trans_i = LLVMBuildGEP(builder, vp_ptr, &index, 1, ""); 827 828 scale = lp_build_broadcast(gallivm, vs_type_llvm, 829 LLVMBuildLoad(builder, scale_i, "scale")); 830 trans = lp_build_broadcast(gallivm, vs_type_llvm, 831 LLVMBuildLoad(builder, trans_i, "trans")); 832 833 /* divide by w */ 834 out = LLVMBuildFMul(builder, out, out3, ""); 835 /* mult by scale */ 836 out = LLVMBuildFMul(builder, out, scale, ""); 837 /* add translation */ 838 out = LLVMBuildFAdd(builder, out, trans, ""); 839 840 /* store transformed outputs */ 841 LLVMBuildStore(builder, out, outputs[0][i]); 842 } 843 844 } 845 846 847 /** 848 * Returns clipmask as nxi32 bitmask for the n vertices 849 */ 850 static LLVMValueRef 851 generate_clipmask(struct draw_llvm *llvm, 852 struct gallivm_state *gallivm, 853 struct lp_type vs_type, 854 LLVMValueRef (*outputs)[TGSI_NUM_CHANNELS], 855 boolean clip_xy, 856 boolean clip_z, 857 boolean clip_user, 858 boolean clip_halfz, 859 unsigned ucp_enable, 860 LLVMValueRef context_ptr, 861 boolean *have_clipdist) 862 { 863 LLVMBuilderRef builder = gallivm->builder; 864 LLVMValueRef mask; /* stores the <nxi32> clipmasks */ 865 LLVMValueRef test, temp; 866 LLVMValueRef zero, shift; 867 LLVMValueRef pos_x, pos_y, pos_z, pos_w; 868 LLVMValueRef cv_x, cv_y, cv_z, cv_w; 869 LLVMValueRef plane1, planes, plane_ptr, sum; 870 struct lp_type f32_type = vs_type; 871 struct lp_type i32_type = lp_int_type(vs_type); 872 const unsigned pos = draw_current_shader_position_output(llvm->draw); 873 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw); 874 int num_written_clipdistance = llvm->draw->vs.vertex_shader->info.num_written_clipdistance; 875 bool have_cd = false; 876 unsigned cd[2]; 877 878 cd[0] = draw_current_shader_clipdistance_output(llvm->draw, 0); 879 cd[1] = draw_current_shader_clipdistance_output(llvm->draw, 1); 880 881 if (cd[0] != pos || cd[1] != pos) 882 have_cd = true; 883 884 mask = lp_build_const_int_vec(gallivm, i32_type, 0); 885 temp = lp_build_const_int_vec(gallivm, i32_type, 0); 886 zero = lp_build_const_vec(gallivm, f32_type, 0); /* 0.0f 0.0f 0.0f 0.0f */ 887 shift = lp_build_const_int_vec(gallivm, i32_type, 1); /* 1 1 1 1 */ 888 889 /* 890 * load clipvertex and position from correct locations. 891 * if they are the same just load them once. 892 */ 893 pos_x = LLVMBuildLoad(builder, outputs[pos][0], ""); /*x0 x1 .. xn */ 894 pos_y = LLVMBuildLoad(builder, outputs[pos][1], ""); /*y0 y1 .. yn */ 895 pos_z = LLVMBuildLoad(builder, outputs[pos][2], ""); /*z0 z1 .. zn */ 896 pos_w = LLVMBuildLoad(builder, outputs[pos][3], ""); /*w0 w1 .. wn */ 897 898 if (clip_user && cv != pos) { 899 cv_x = LLVMBuildLoad(builder, outputs[cv][0], ""); /*x0 x1 .. xn */ 900 cv_y = LLVMBuildLoad(builder, outputs[cv][1], ""); /*y0 y1 .. yn */ 901 cv_z = LLVMBuildLoad(builder, outputs[cv][2], ""); /*z0 z1 .. zn */ 902 cv_w = LLVMBuildLoad(builder, outputs[cv][3], ""); /*w0 w1 .. wn */ 903 } else { 904 cv_x = pos_x; 905 cv_y = pos_y; 906 cv_z = pos_z; 907 cv_w = pos_w; 908 } 909 910 /* Cliptest, for hardwired planes */ 911 if (clip_xy) { 912 /* plane 1 */ 913 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_x , pos_w); 914 temp = shift; 915 test = LLVMBuildAnd(builder, test, temp, ""); 916 mask = test; 917 918 /* plane 2 */ 919 test = LLVMBuildFAdd(builder, pos_x, pos_w, ""); 920 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 921 temp = LLVMBuildShl(builder, temp, shift, ""); 922 test = LLVMBuildAnd(builder, test, temp, ""); 923 mask = LLVMBuildOr(builder, mask, test, ""); 924 925 /* plane 3 */ 926 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_y, pos_w); 927 temp = LLVMBuildShl(builder, temp, shift, ""); 928 test = LLVMBuildAnd(builder, test, temp, ""); 929 mask = LLVMBuildOr(builder, mask, test, ""); 930 931 /* plane 4 */ 932 test = LLVMBuildFAdd(builder, pos_y, pos_w, ""); 933 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 934 temp = LLVMBuildShl(builder, temp, shift, ""); 935 test = LLVMBuildAnd(builder, test, temp, ""); 936 mask = LLVMBuildOr(builder, mask, test, ""); 937 } 938 939 if (clip_z) { 940 temp = lp_build_const_int_vec(gallivm, i32_type, 16); 941 if (clip_halfz) { 942 /* plane 5 */ 943 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, pos_z); 944 test = LLVMBuildAnd(builder, test, temp, ""); 945 mask = LLVMBuildOr(builder, mask, test, ""); 946 } 947 else { 948 /* plane 5 */ 949 test = LLVMBuildFAdd(builder, pos_z, pos_w, ""); 950 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, test); 951 test = LLVMBuildAnd(builder, test, temp, ""); 952 mask = LLVMBuildOr(builder, mask, test, ""); 953 } 954 /* plane 6 */ 955 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, pos_z, pos_w); 956 temp = LLVMBuildShl(builder, temp, shift, ""); 957 test = LLVMBuildAnd(builder, test, temp, ""); 958 mask = LLVMBuildOr(builder, mask, test, ""); 959 } 960 961 if (clip_user) { 962 LLVMValueRef planes_ptr = draw_jit_context_planes(gallivm, context_ptr); 963 LLVMValueRef indices[3]; 964 965 /* userclip planes */ 966 while (ucp_enable) { 967 unsigned plane_idx = ffs(ucp_enable)-1; 968 ucp_enable &= ~(1 << plane_idx); 969 plane_idx += 6; 970 971 if (have_cd && num_written_clipdistance) { 972 LLVMValueRef clipdist; 973 int i; 974 i = plane_idx - 6; 975 976 *have_clipdist = TRUE; 977 if (i < 4) { 978 clipdist = LLVMBuildLoad(builder, outputs[cd[0]][i], ""); 979 } else { 980 clipdist = LLVMBuildLoad(builder, outputs[cd[1]][i-4], ""); 981 } 982 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, clipdist); 983 temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx); 984 test = LLVMBuildAnd(builder, test, temp, ""); 985 mask = LLVMBuildOr(builder, mask, test, ""); 986 } else { 987 LLVMTypeRef vs_type_llvm = lp_build_vec_type(gallivm, vs_type); 988 indices[0] = lp_build_const_int32(gallivm, 0); 989 indices[1] = lp_build_const_int32(gallivm, plane_idx); 990 991 indices[2] = lp_build_const_int32(gallivm, 0); 992 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 993 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_x"); 994 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1); 995 sum = LLVMBuildFMul(builder, planes, cv_x, ""); 996 997 indices[2] = lp_build_const_int32(gallivm, 1); 998 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 999 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_y"); 1000 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1); 1001 test = LLVMBuildFMul(builder, planes, cv_y, ""); 1002 sum = LLVMBuildFAdd(builder, sum, test, ""); 1003 1004 indices[2] = lp_build_const_int32(gallivm, 2); 1005 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1006 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_z"); 1007 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1); 1008 test = LLVMBuildFMul(builder, planes, cv_z, ""); 1009 sum = LLVMBuildFAdd(builder, sum, test, ""); 1010 1011 indices[2] = lp_build_const_int32(gallivm, 3); 1012 plane_ptr = LLVMBuildGEP(builder, planes_ptr, indices, 3, ""); 1013 plane1 = LLVMBuildLoad(builder, plane_ptr, "plane_w"); 1014 planes = lp_build_broadcast(gallivm, vs_type_llvm, plane1); 1015 test = LLVMBuildFMul(builder, planes, cv_w, ""); 1016 sum = LLVMBuildFAdd(builder, sum, test, ""); 1017 1018 test = lp_build_compare(gallivm, f32_type, PIPE_FUNC_GREATER, zero, sum); 1019 temp = lp_build_const_int_vec(gallivm, i32_type, 1 << plane_idx); 1020 test = LLVMBuildAnd(builder, test, temp, ""); 1021 mask = LLVMBuildOr(builder, mask, test, ""); 1022 } 1023 } 1024 } 1025 return mask; 1026 } 1027 1028 1029 /** 1030 * Returns boolean if any clipping has occurred 1031 * Used zero/non-zero i32 value to represent boolean 1032 */ 1033 static LLVMValueRef 1034 clipmask_booli32(struct gallivm_state *gallivm, 1035 const struct lp_type vs_type, 1036 LLVMValueRef clipmask_bool_ptr) 1037 { 1038 LLVMBuilderRef builder = gallivm->builder; 1039 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 1040 LLVMValueRef clipmask_bool = LLVMBuildLoad(builder, clipmask_bool_ptr, ""); 1041 LLVMValueRef ret = LLVMConstNull(int32_type); 1042 LLVMValueRef temp; 1043 int i; 1044 1045 /* 1046 * Can do this with log2(vector length) pack instructions and one extract 1047 * (as we don't actually need a or) with sse2 which would be way better. 1048 */ 1049 for (i=0; i < vs_type.length; i++) { 1050 temp = LLVMBuildExtractElement(builder, clipmask_bool, 1051 lp_build_const_int32(gallivm, i) , ""); 1052 ret = LLVMBuildOr(builder, ret, temp, ""); 1053 } 1054 return ret; 1055 } 1056 1057 1058 static void 1059 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant, 1060 boolean elts) 1061 { 1062 struct gallivm_state *gallivm = variant->gallivm; 1063 LLVMContextRef context = gallivm->context; 1064 LLVMTypeRef int32_type = LLVMInt32TypeInContext(context); 1065 LLVMTypeRef arg_types[8]; 1066 LLVMTypeRef func_type; 1067 LLVMValueRef context_ptr; 1068 LLVMBasicBlockRef block; 1069 LLVMBuilderRef builder; 1070 struct lp_type vs_type; 1071 LLVMValueRef end, start; 1072 LLVMValueRef count, fetch_elts, fetch_count; 1073 LLVMValueRef stride, step, io_itr; 1074 LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr; 1075 LLVMValueRef zero = lp_build_const_int32(gallivm, 0); 1076 LLVMValueRef one = lp_build_const_int32(gallivm, 1); 1077 struct draw_context *draw = llvm->draw; 1078 const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info; 1079 unsigned i, j; 1080 struct lp_build_context bld; 1081 struct lp_build_loop_state lp_loop; 1082 const int vector_length = lp_native_vector_width / 32; 1083 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 1084 LLVMValueRef fetch_max; 1085 struct lp_build_sampler_soa *sampler = 0; 1086 LLVMValueRef ret, clipmask_bool_ptr; 1087 const boolean bypass_viewport = variant->key.bypass_viewport; 1088 const boolean enable_cliptest = variant->key.clip_xy || 1089 variant->key.clip_z || 1090 variant->key.clip_user; 1091 LLVMValueRef variant_func; 1092 const unsigned pos = draw_current_shader_position_output(llvm->draw); 1093 const unsigned cv = draw_current_shader_clipvertex_output(llvm->draw); 1094 boolean have_clipdist = FALSE; 1095 struct lp_bld_tgsi_system_values system_values; 1096 1097 memset(&system_values, 0, sizeof(system_values)); 1098 1099 arg_types[0] = get_context_ptr_type(variant); /* context */ 1100 arg_types[1] = get_vertex_header_ptr_type(variant); /* vertex_header */ 1101 arg_types[2] = get_buffer_ptr_type(variant); /* vbuffers */ 1102 if (elts) 1103 arg_types[3] = LLVMPointerType(int32_type, 0);/* fetch_elts * */ 1104 else 1105 arg_types[3] = int32_type; /* start */ 1106 arg_types[4] = int32_type; /* fetch_count / count */ 1107 arg_types[5] = int32_type; /* stride */ 1108 arg_types[6] = get_vb_ptr_type(variant); /* pipe_vertex_buffer's */ 1109 arg_types[7] = int32_type; /* instance_id */ 1110 1111 func_type = LLVMFunctionType(int32_type, arg_types, Elements(arg_types), 0); 1112 1113 variant_func = LLVMAddFunction(gallivm->module, 1114 elts ? "draw_llvm_shader_elts" : "draw_llvm_shader", 1115 func_type); 1116 1117 if (elts) 1118 variant->function_elts = variant_func; 1119 else 1120 variant->function = variant_func; 1121 1122 LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); 1123 for (i = 0; i < Elements(arg_types); ++i) 1124 if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 1125 LLVMAddAttribute(LLVMGetParam(variant_func, i), 1126 LLVMNoAliasAttribute); 1127 1128 context_ptr = LLVMGetParam(variant_func, 0); 1129 io_ptr = LLVMGetParam(variant_func, 1); 1130 vbuffers_ptr = LLVMGetParam(variant_func, 2); 1131 stride = LLVMGetParam(variant_func, 5); 1132 vb_ptr = LLVMGetParam(variant_func, 6); 1133 system_values.instance_id = LLVMGetParam(variant_func, 7); 1134 1135 lp_build_name(context_ptr, "context"); 1136 lp_build_name(io_ptr, "io"); 1137 lp_build_name(vbuffers_ptr, "vbuffers"); 1138 lp_build_name(stride, "stride"); 1139 lp_build_name(vb_ptr, "vb"); 1140 lp_build_name(system_values.instance_id, "instance_id"); 1141 1142 if (elts) { 1143 fetch_elts = LLVMGetParam(variant_func, 3); 1144 fetch_count = LLVMGetParam(variant_func, 4); 1145 lp_build_name(fetch_elts, "fetch_elts"); 1146 lp_build_name(fetch_count, "fetch_count"); 1147 start = count = NULL; 1148 } 1149 else { 1150 start = LLVMGetParam(variant_func, 3); 1151 count = LLVMGetParam(variant_func, 4); 1152 lp_build_name(start, "start"); 1153 lp_build_name(count, "count"); 1154 fetch_elts = fetch_count = NULL; 1155 } 1156 1157 /* 1158 * Function body 1159 */ 1160 1161 block = LLVMAppendBasicBlockInContext(gallivm->context, variant_func, "entry"); 1162 builder = gallivm->builder; 1163 LLVMPositionBuilderAtEnd(builder, block); 1164 1165 lp_build_context_init(&bld, gallivm, lp_type_int(32)); 1166 1167 memset(&vs_type, 0, sizeof vs_type); 1168 vs_type.floating = TRUE; /* floating point values */ 1169 vs_type.sign = TRUE; /* values are signed */ 1170 vs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 1171 vs_type.width = 32; /* 32-bit float */ 1172 vs_type.length = vector_length; 1173 1174 /* hold temporary "bool" clipmask */ 1175 clipmask_bool_ptr = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, vs_type), ""); 1176 LLVMBuildStore(builder, lp_build_zero(gallivm, lp_int_type(vs_type)), clipmask_bool_ptr); 1177 1178 /* code generated texture sampling */ 1179 sampler = draw_llvm_sampler_soa_create( 1180 draw_llvm_variant_key_samplers(&variant->key), 1181 context_ptr); 1182 1183 if (elts) { 1184 start = zero; 1185 end = fetch_count; 1186 } 1187 else { 1188 end = lp_build_add(&bld, start, count); 1189 } 1190 1191 step = lp_build_const_int32(gallivm, vector_length); 1192 1193 fetch_max = LLVMBuildSub(builder, end, one, "fetch_max"); 1194 1195 lp_build_loop_begin(&lp_loop, gallivm, start); 1196 { 1197 LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS]; 1198 LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][LP_MAX_VECTOR_WIDTH / 32] = { { 0 } }; 1199 LLVMValueRef io; 1200 LLVMValueRef clipmask; /* holds the clipmask value */ 1201 const LLVMValueRef (*ptr_aos)[TGSI_NUM_CHANNELS]; 1202 1203 if (elts) 1204 io_itr = lp_loop.counter; 1205 else 1206 io_itr = LLVMBuildSub(builder, lp_loop.counter, start, ""); 1207 1208 io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, ""); 1209 #if DEBUG_STORE 1210 lp_build_printf(gallivm, " --- io %d = %p, loop counter %d\n", 1211 io_itr, io, lp_loop.counter); 1212 #endif 1213 system_values.vertex_id = lp_build_zero(gallivm, lp_type_uint_vec(32, 32*vector_length)); 1214 for (i = 0; i < vector_length; ++i) { 1215 LLVMValueRef true_index = 1216 LLVMBuildAdd(builder, 1217 lp_loop.counter, 1218 lp_build_const_int32(gallivm, i), ""); 1219 1220 /* make sure we're not out of bounds which can happen 1221 * if fetch_count % 4 != 0, because on the last iteration 1222 * a few of the 4 vertex fetches will be out of bounds */ 1223 true_index = lp_build_min(&bld, true_index, fetch_max); 1224 1225 if (elts) { 1226 LLVMValueRef fetch_ptr; 1227 fetch_ptr = LLVMBuildGEP(builder, fetch_elts, 1228 &true_index, 1, ""); 1229 true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt"); 1230 } 1231 1232 system_values.vertex_id = LLVMBuildInsertElement(gallivm->builder, 1233 system_values.vertex_id, true_index, 1234 lp_build_const_int32(gallivm, i), ""); 1235 for (j = 0; j < draw->pt.nr_vertex_elements; ++j) { 1236 struct pipe_vertex_element *velem = &draw->pt.vertex_element[j]; 1237 LLVMValueRef vb_index = 1238 lp_build_const_int32(gallivm, velem->vertex_buffer_index); 1239 LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, ""); 1240 generate_fetch(gallivm, vbuffers_ptr, 1241 &aos_attribs[j][i], velem, vb, true_index, 1242 system_values.instance_id); 1243 } 1244 } 1245 convert_to_soa(gallivm, aos_attribs, inputs, 1246 draw->pt.nr_vertex_elements, vs_type); 1247 1248 ptr_aos = (const LLVMValueRef (*)[TGSI_NUM_CHANNELS]) inputs; 1249 generate_vs(variant, 1250 builder, 1251 vs_type, 1252 outputs, 1253 ptr_aos, 1254 &system_values, 1255 context_ptr, 1256 sampler, 1257 variant->key.clamp_vertex_color); 1258 1259 /* store original positions in clip before further manipulation */ 1260 store_clip(gallivm, vs_type, io, outputs, 0, cv); 1261 store_clip(gallivm, vs_type, io, outputs, 1, pos); 1262 1263 /* do cliptest */ 1264 if (enable_cliptest) { 1265 LLVMValueRef temp = LLVMBuildLoad(builder, clipmask_bool_ptr, ""); 1266 /* allocate clipmask, assign it integer type */ 1267 clipmask = generate_clipmask(llvm, 1268 gallivm, 1269 vs_type, 1270 outputs, 1271 variant->key.clip_xy, 1272 variant->key.clip_z, 1273 variant->key.clip_user, 1274 variant->key.clip_halfz, 1275 variant->key.ucp_enable, 1276 context_ptr, &have_clipdist); 1277 temp = LLVMBuildOr(builder, clipmask, temp, ""); 1278 /* store temporary clipping boolean value */ 1279 LLVMBuildStore(builder, temp, clipmask_bool_ptr); 1280 } 1281 else { 1282 clipmask = lp_build_const_int_vec(gallivm, lp_int_type(vs_type), 0); 1283 } 1284 1285 /* do viewport mapping */ 1286 if (!bypass_viewport) { 1287 generate_viewport(variant, builder, vs_type, outputs, context_ptr); 1288 } 1289 1290 /* store clipmask in vertex header, 1291 * original positions in clip 1292 * and transformed positions in data 1293 */ 1294 convert_to_aos(gallivm, io, outputs, clipmask, 1295 vs_info->num_outputs, vs_type, 1296 have_clipdist); 1297 } 1298 1299 lp_build_loop_end_cond(&lp_loop, end, step, LLVMIntUGE); 1300 1301 sampler->destroy(sampler); 1302 1303 /* return clipping boolean value for function */ 1304 ret = clipmask_booli32(gallivm, vs_type, clipmask_bool_ptr); 1305 1306 LLVMBuildRet(builder, ret); 1307 1308 gallivm_verify_function(gallivm, variant_func); 1309 } 1310 1311 1312 struct draw_llvm_variant_key * 1313 draw_llvm_make_variant_key(struct draw_llvm *llvm, char *store) 1314 { 1315 unsigned i; 1316 struct draw_llvm_variant_key *key; 1317 struct lp_sampler_static_state *sampler; 1318 1319 key = (struct draw_llvm_variant_key *)store; 1320 1321 key->clamp_vertex_color = llvm->draw->rasterizer->clamp_vertex_color; /**/ 1322 1323 /* Presumably all variants of the shader should have the same 1324 * number of vertex elements - ie the number of shader inputs. 1325 */ 1326 key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements; 1327 1328 /* will have to rig this up properly later */ 1329 key->clip_xy = llvm->draw->clip_xy; 1330 key->clip_z = llvm->draw->clip_z; 1331 key->clip_user = llvm->draw->clip_user; 1332 key->bypass_viewport = llvm->draw->identity_viewport; 1333 key->clip_halfz = !llvm->draw->rasterizer->gl_rasterization_rules; 1334 key->need_edgeflags = (llvm->draw->vs.edgeflag_output ? TRUE : FALSE); 1335 key->ucp_enable = llvm->draw->rasterizer->clip_plane_enable; 1336 key->pad = 0; 1337 1338 /* All variants of this shader will have the same value for 1339 * nr_samplers. Not yet trying to compact away holes in the 1340 * sampler array. 1341 */ 1342 key->nr_samplers = llvm->draw->vs.vertex_shader->info.file_max[TGSI_FILE_SAMPLER] + 1; 1343 1344 sampler = draw_llvm_variant_key_samplers(key); 1345 1346 memcpy(key->vertex_element, 1347 llvm->draw->pt.vertex_element, 1348 sizeof(struct pipe_vertex_element) * key->nr_vertex_elements); 1349 1350 memset(sampler, 0, key->nr_samplers * sizeof *sampler); 1351 1352 for (i = 0 ; i < key->nr_samplers; i++) { 1353 lp_sampler_static_state(&sampler[i], 1354 llvm->draw->sampler_views[PIPE_SHADER_VERTEX][i], 1355 llvm->draw->samplers[PIPE_SHADER_VERTEX][i]); 1356 } 1357 1358 return key; 1359 } 1360 1361 1362 void 1363 draw_llvm_set_mapped_texture(struct draw_context *draw, 1364 unsigned sampler_idx, 1365 uint32_t width, uint32_t height, uint32_t depth, 1366 uint32_t first_level, uint32_t last_level, 1367 uint32_t row_stride[PIPE_MAX_TEXTURE_LEVELS], 1368 uint32_t img_stride[PIPE_MAX_TEXTURE_LEVELS], 1369 const void *data[PIPE_MAX_TEXTURE_LEVELS]) 1370 { 1371 unsigned j; 1372 struct draw_jit_texture *jit_tex; 1373 1374 assert(sampler_idx < Elements(draw->llvm->jit_context.textures)); 1375 1376 jit_tex = &draw->llvm->jit_context.textures[sampler_idx]; 1377 1378 jit_tex->width = width; 1379 jit_tex->height = height; 1380 jit_tex->depth = depth; 1381 jit_tex->first_level = first_level; 1382 jit_tex->last_level = last_level; 1383 1384 for (j = first_level; j <= last_level; j++) { 1385 jit_tex->data[j] = data[j]; 1386 jit_tex->row_stride[j] = row_stride[j]; 1387 jit_tex->img_stride[j] = img_stride[j]; 1388 } 1389 } 1390 1391 1392 void 1393 draw_llvm_set_sampler_state(struct draw_context *draw) 1394 { 1395 unsigned i; 1396 1397 for (i = 0; i < draw->num_samplers[PIPE_SHADER_VERTEX]; i++) { 1398 struct draw_jit_texture *jit_tex = &draw->llvm->jit_context.textures[i]; 1399 1400 if (draw->samplers[i]) { 1401 const struct pipe_sampler_state *s 1402 = draw->samplers[PIPE_SHADER_VERTEX][i]; 1403 jit_tex->min_lod = s->min_lod; 1404 jit_tex->max_lod = s->max_lod; 1405 jit_tex->lod_bias = s->lod_bias; 1406 COPY_4V(jit_tex->border_color, s->border_color.f); 1407 } 1408 } 1409 } 1410 1411 1412 void 1413 draw_llvm_destroy_variant(struct draw_llvm_variant *variant) 1414 { 1415 struct draw_llvm *llvm = variant->llvm; 1416 1417 if (variant->function_elts) { 1418 gallivm_free_function(variant->gallivm, 1419 variant->function_elts, variant->jit_func_elts); 1420 } 1421 1422 if (variant->function) { 1423 gallivm_free_function(variant->gallivm, 1424 variant->function, variant->jit_func); 1425 } 1426 1427 gallivm_destroy(variant->gallivm); 1428 1429 remove_from_list(&variant->list_item_local); 1430 variant->shader->variants_cached--; 1431 remove_from_list(&variant->list_item_global); 1432 llvm->nr_variants--; 1433 FREE(variant); 1434 } 1435
