1 /************************************************************************** 2 * 3 * Copyright 2015 VMware, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 28 #include "lp_bld_format.h" 29 #include "lp_bld_type.h" 30 #include "lp_bld_struct.h" 31 #include "lp_bld_const.h" 32 #include "lp_bld_flow.h" 33 #include "lp_bld_swizzle.h" 34 35 #include "util/u_math.h" 36 37 38 /** 39 * @file 40 * Complex block-compression based formats are handled here by using a cache, 41 * so re-decoding of every pixel is not required. 42 * Especially for bilinear filtering, texel reuse is very high hence even 43 * a small cache helps. 44 * The elements in the cache are the decoded blocks - currently things 45 * are restricted to formats which are 4x4 block based, and the decoded 46 * texels must fit into 4x8 bits. 47 * The cache is direct mapped so hitrates aren't all that great and cache 48 * thrashing could happen. 49 * 50 * @author Roland Scheidegger <sroland (at) vmware.com> 51 */ 52 53 54 #if LP_BUILD_FORMAT_CACHE_DEBUG 55 static void 56 update_cache_access(struct gallivm_state *gallivm, 57 LLVMValueRef ptr, 58 unsigned count, 59 unsigned index) 60 { 61 LLVMBuilderRef builder = gallivm->builder; 62 LLVMValueRef member_ptr, cache_access; 63 64 assert(index == LP_BUILD_FORMAT_CACHE_MEMBER_ACCESS_TOTAL || 65 index == LP_BUILD_FORMAT_CACHE_MEMBER_ACCESS_MISS); 66 67 member_ptr = lp_build_struct_get_ptr(gallivm, ptr, index, ""); 68 cache_access = LLVMBuildLoad(builder, member_ptr, "cache_access"); 69 cache_access = LLVMBuildAdd(builder, cache_access, 70 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 71 count, 0), ""); 72 LLVMBuildStore(builder, cache_access, member_ptr); 73 } 74 #endif 75 76 77 static void 78 store_cached_block(struct gallivm_state *gallivm, 79 LLVMValueRef *col, 80 LLVMValueRef tag_value, 81 LLVMValueRef hash_index, 82 LLVMValueRef cache) 83 { 84 LLVMBuilderRef builder = gallivm->builder; 85 LLVMValueRef ptr, indices[3]; 86 LLVMTypeRef type_ptr4x32; 87 unsigned count; 88 89 type_ptr4x32 = LLVMPointerType(LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4), 0); 90 indices[0] = lp_build_const_int32(gallivm, 0); 91 indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS); 92 indices[2] = hash_index; 93 ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), ""); 94 LLVMBuildStore(builder, tag_value, ptr); 95 96 indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_DATA); 97 hash_index = LLVMBuildMul(builder, hash_index, 98 lp_build_const_int32(gallivm, 16), ""); 99 for (count = 0; count < 4; count++) { 100 indices[2] = hash_index; 101 ptr = LLVMBuildGEP(builder, cache, indices, ARRAY_SIZE(indices), ""); 102 ptr = LLVMBuildBitCast(builder, ptr, type_ptr4x32, ""); 103 LLVMBuildStore(builder, col[count], ptr); 104 hash_index = LLVMBuildAdd(builder, hash_index, 105 lp_build_const_int32(gallivm, 4), ""); 106 } 107 } 108 109 110 static LLVMValueRef 111 lookup_cached_pixel(struct gallivm_state *gallivm, 112 LLVMValueRef ptr, 113 LLVMValueRef index) 114 { 115 LLVMBuilderRef builder = gallivm->builder; 116 LLVMValueRef member_ptr, indices[3]; 117 118 indices[0] = lp_build_const_int32(gallivm, 0); 119 indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_DATA); 120 indices[2] = index; 121 member_ptr = LLVMBuildGEP(builder, ptr, indices, ARRAY_SIZE(indices), ""); 122 return LLVMBuildLoad(builder, member_ptr, "cache_data"); 123 } 124 125 126 static LLVMValueRef 127 lookup_tag_data(struct gallivm_state *gallivm, 128 LLVMValueRef ptr, 129 LLVMValueRef index) 130 { 131 LLVMBuilderRef builder = gallivm->builder; 132 LLVMValueRef member_ptr, indices[3]; 133 134 indices[0] = lp_build_const_int32(gallivm, 0); 135 indices[1] = lp_build_const_int32(gallivm, LP_BUILD_FORMAT_CACHE_MEMBER_TAGS); 136 indices[2] = index; 137 member_ptr = LLVMBuildGEP(builder, ptr, indices, ARRAY_SIZE(indices), ""); 138 return LLVMBuildLoad(builder, member_ptr, "tag_data"); 139 } 140 141 142 static void 143 update_cached_block(struct gallivm_state *gallivm, 144 const struct util_format_description *format_desc, 145 LLVMValueRef ptr_addr, 146 LLVMValueRef hash_index, 147 LLVMValueRef cache) 148 149 { 150 LLVMBuilderRef builder = gallivm->builder; 151 LLVMTypeRef i8t = LLVMInt8TypeInContext(gallivm->context); 152 LLVMTypeRef pi8t = LLVMPointerType(i8t, 0); 153 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); 154 LLVMTypeRef i32x4 = LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), 4); 155 LLVMValueRef function; 156 LLVMValueRef tag_value, tmp_ptr; 157 LLVMValueRef col[4]; 158 unsigned i, j; 159 160 /* 161 * Use format_desc->fetch_rgba_8unorm() for each pixel in the block. 162 * This doesn't actually make any sense whatsoever, someone would need 163 * to write a function doing this for all pixels in a block (either as 164 * an external c function or with generated code). Don't ask. 165 */ 166 167 { 168 /* 169 * Function to call looks like: 170 * fetch(uint8_t *dst, const uint8_t *src, unsigned i, unsigned j) 171 */ 172 LLVMTypeRef ret_type; 173 LLVMTypeRef arg_types[4]; 174 LLVMTypeRef function_type; 175 176 assert(format_desc->fetch_rgba_8unorm); 177 178 ret_type = LLVMVoidTypeInContext(gallivm->context); 179 arg_types[0] = pi8t; 180 arg_types[1] = pi8t; 181 arg_types[2] = i32t; 182 arg_types[3] = i32t; 183 function_type = LLVMFunctionType(ret_type, arg_types, 184 ARRAY_SIZE(arg_types), 0); 185 186 /* make const pointer for the C fetch_rgba_8unorm function */ 187 function = lp_build_const_int_pointer(gallivm, 188 func_to_pointer((func_pointer) format_desc->fetch_rgba_8unorm)); 189 190 /* cast the callee pointer to the function's type */ 191 function = LLVMBuildBitCast(builder, function, 192 LLVMPointerType(function_type, 0), 193 "cast callee"); 194 } 195 196 tmp_ptr = lp_build_array_alloca(gallivm, i32x4, 197 lp_build_const_int32(gallivm, 16), 198 "tmp_decode_store"); 199 tmp_ptr = LLVMBuildBitCast(builder, tmp_ptr, pi8t, ""); 200 201 /* 202 * Invoke format_desc->fetch_rgba_8unorm() for each pixel. 203 * This is going to be really really slow. 204 * Note: the block store format is actually 205 * x0y0x0y1x0y2x0y3 x1y0x1y1x1y2x1y3 ... 206 */ 207 for (i = 0; i < 4; ++i) { 208 for (j = 0; j < 4; ++j) { 209 LLVMValueRef args[4]; 210 LLVMValueRef dst_offset = lp_build_const_int32(gallivm, (i * 4 + j) * 4); 211 212 /* 213 * Note we actually supply a pointer to the start of the block, 214 * not the start of the texture. 215 */ 216 args[0] = LLVMBuildGEP(gallivm->builder, tmp_ptr, &dst_offset, 1, ""); 217 args[1] = ptr_addr; 218 args[2] = LLVMConstInt(i32t, i, 0); 219 args[3] = LLVMConstInt(i32t, j, 0); 220 LLVMBuildCall(builder, function, args, ARRAY_SIZE(args), ""); 221 } 222 } 223 224 /* Finally store the block - pointless mem copy + update tag. */ 225 tmp_ptr = LLVMBuildBitCast(builder, tmp_ptr, LLVMPointerType(i32x4, 0), ""); 226 for (i = 0; i < 4; ++i) { 227 LLVMValueRef tmp_offset = lp_build_const_int32(gallivm, i); 228 LLVMValueRef ptr = LLVMBuildGEP(gallivm->builder, tmp_ptr, &tmp_offset, 1, ""); 229 col[i] = LLVMBuildLoad(builder, ptr, ""); 230 } 231 232 tag_value = LLVMBuildPtrToInt(gallivm->builder, ptr_addr, 233 LLVMInt64TypeInContext(gallivm->context), ""); 234 store_cached_block(gallivm, col, tag_value, hash_index, cache); 235 } 236 237 238 /* 239 * Do a cached lookup. 240 * 241 * Returns (vectors of) 4x8 rgba aos value 242 */ 243 LLVMValueRef 244 lp_build_fetch_cached_texels(struct gallivm_state *gallivm, 245 const struct util_format_description *format_desc, 246 unsigned n, 247 LLVMValueRef base_ptr, 248 LLVMValueRef offset, 249 LLVMValueRef i, 250 LLVMValueRef j, 251 LLVMValueRef cache) 252 253 { 254 LLVMBuilderRef builder = gallivm->builder; 255 unsigned count, low_bit, log2size; 256 LLVMValueRef color, offset_stored, addr, ptr_addrtrunc, tmp; 257 LLVMValueRef ij_index, hash_index, hash_mask, block_index; 258 LLVMTypeRef i8t = LLVMInt8TypeInContext(gallivm->context); 259 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); 260 LLVMTypeRef i64t = LLVMInt64TypeInContext(gallivm->context); 261 struct lp_type type; 262 struct lp_build_context bld32; 263 memset(&type, 0, sizeof type); 264 type.width = 32; 265 type.length = n; 266 267 assert(format_desc->block.width == 4); 268 assert(format_desc->block.height == 4); 269 270 lp_build_context_init(&bld32, gallivm, type); 271 272 /* 273 * compute hash - we use direct mapped cache, the hash function could 274 * be better but it needs to be simple 275 * per-element: 276 * compare offset with offset stored at tag (hash) 277 * if not equal decode/store block, update tag 278 * extract color from cache 279 * assemble result vector 280 */ 281 282 /* TODO: not ideal with 32bit pointers... */ 283 284 low_bit = util_logbase2(format_desc->block.bits / 8); 285 log2size = util_logbase2(LP_BUILD_FORMAT_CACHE_SIZE); 286 addr = LLVMBuildPtrToInt(builder, base_ptr, i64t, ""); 287 ptr_addrtrunc = LLVMBuildPtrToInt(builder, base_ptr, i32t, ""); 288 ptr_addrtrunc = lp_build_broadcast_scalar(&bld32, ptr_addrtrunc); 289 /* For the hash function, first mask off the unused lowest bits. Then just 290 do some xor with address bits - only use lower 32bits */ 291 ptr_addrtrunc = LLVMBuildAdd(builder, offset, ptr_addrtrunc, ""); 292 ptr_addrtrunc = LLVMBuildLShr(builder, ptr_addrtrunc, 293 lp_build_const_int_vec(gallivm, type, low_bit), ""); 294 /* This only really makes sense for size 64,128,256 */ 295 hash_index = ptr_addrtrunc; 296 ptr_addrtrunc = LLVMBuildLShr(builder, ptr_addrtrunc, 297 lp_build_const_int_vec(gallivm, type, 2*log2size), ""); 298 hash_index = LLVMBuildXor(builder, ptr_addrtrunc, hash_index, ""); 299 tmp = LLVMBuildLShr(builder, hash_index, 300 lp_build_const_int_vec(gallivm, type, log2size), ""); 301 hash_index = LLVMBuildXor(builder, hash_index, tmp, ""); 302 303 hash_mask = lp_build_const_int_vec(gallivm, type, LP_BUILD_FORMAT_CACHE_SIZE - 1); 304 hash_index = LLVMBuildAnd(builder, hash_index, hash_mask, ""); 305 ij_index = LLVMBuildShl(builder, i, lp_build_const_int_vec(gallivm, type, 2), ""); 306 ij_index = LLVMBuildAdd(builder, ij_index, j, ""); 307 block_index = LLVMBuildShl(builder, hash_index, 308 lp_build_const_int_vec(gallivm, type, 4), ""); 309 block_index = LLVMBuildAdd(builder, ij_index, block_index, ""); 310 311 if (n > 1) { 312 color = LLVMGetUndef(LLVMVectorType(i32t, n)); 313 for (count = 0; count < n; count++) { 314 LLVMValueRef index, cond, colorx; 315 LLVMValueRef block_indexx, hash_indexx, addrx, offsetx, ptr_addrx; 316 struct lp_build_if_state if_ctx; 317 318 index = lp_build_const_int32(gallivm, count); 319 offsetx = LLVMBuildExtractElement(builder, offset, index, ""); 320 addrx = LLVMBuildZExt(builder, offsetx, i64t, ""); 321 addrx = LLVMBuildAdd(builder, addrx, addr, ""); 322 block_indexx = LLVMBuildExtractElement(builder, block_index, index, ""); 323 hash_indexx = LLVMBuildLShr(builder, block_indexx, 324 lp_build_const_int32(gallivm, 4), ""); 325 offset_stored = lookup_tag_data(gallivm, cache, hash_indexx); 326 cond = LLVMBuildICmp(builder, LLVMIntNE, offset_stored, addrx, ""); 327 328 lp_build_if(&if_ctx, gallivm, cond); 329 { 330 ptr_addrx = LLVMBuildIntToPtr(builder, addrx, 331 LLVMPointerType(i8t, 0), ""); 332 update_cached_block(gallivm, format_desc, ptr_addrx, hash_indexx, cache); 333 #if LP_BUILD_FORMAT_CACHE_DEBUG 334 update_cache_access(gallivm, cache, 1, 335 LP_BUILD_FORMAT_CACHE_MEMBER_ACCESS_MISS); 336 #endif 337 } 338 lp_build_endif(&if_ctx); 339 340 colorx = lookup_cached_pixel(gallivm, cache, block_indexx); 341 342 color = LLVMBuildInsertElement(builder, color, colorx, 343 lp_build_const_int32(gallivm, count), ""); 344 } 345 } 346 else { 347 LLVMValueRef cond; 348 struct lp_build_if_state if_ctx; 349 350 tmp = LLVMBuildZExt(builder, offset, i64t, ""); 351 addr = LLVMBuildAdd(builder, tmp, addr, ""); 352 offset_stored = lookup_tag_data(gallivm, cache, hash_index); 353 cond = LLVMBuildICmp(builder, LLVMIntNE, offset_stored, addr, ""); 354 355 lp_build_if(&if_ctx, gallivm, cond); 356 { 357 tmp = LLVMBuildIntToPtr(builder, addr, LLVMPointerType(i8t, 0), ""); 358 update_cached_block(gallivm, format_desc, tmp, hash_index, cache); 359 #if LP_BUILD_FORMAT_CACHE_DEBUG 360 update_cache_access(gallivm, cache, 1, 361 LP_BUILD_FORMAT_CACHE_MEMBER_ACCESS_MISS); 362 #endif 363 } 364 lp_build_endif(&if_ctx); 365 366 color = lookup_cached_pixel(gallivm, cache, block_index); 367 } 368 #if LP_BUILD_FORMAT_CACHE_DEBUG 369 update_cache_access(gallivm, cache, n, 370 LP_BUILD_FORMAT_CACHE_MEMBER_ACCESS_TOTAL); 371 #endif 372 return LLVMBuildBitCast(builder, color, LLVMVectorType(i8t, n * 4), ""); 373 } 374 375
