1 /* 2 Copyright (C) Intel Corp. 2006. All Rights Reserved. 3 Intel funded Tungsten Graphics (http://www.tungstengraphics.com) to 4 develop this 3D driver. 5 6 Permission is hereby granted, free of charge, to any person obtaining 7 a copy of this software and associated documentation files (the 8 "Software"), to deal in the Software without restriction, including 9 without limitation the rights to use, copy, modify, merge, publish, 10 distribute, sublicense, and/or sell copies of the Software, and to 11 permit persons to whom the Software is furnished to do so, subject to 12 the following conditions: 13 14 The above copyright notice and this permission notice (including the 15 next paragraph) shall be included in all copies or substantial 16 portions of the Software. 17 18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 25 26 **********************************************************************/ 27 /* 28 * Authors: 29 * Keith Whitwell <keith (at) tungstengraphics.com> 30 */ 31 32 /** @file brw_state_cache.c 33 * 34 * This file implements a simple static state cache for 965. The 35 * consumers can query the hash table of state using a cache_id, 36 * opaque key data, and receive the corresponding state buffer object 37 * of state (plus associated auxiliary data) in return. Objects in 38 * the cache may not have relocations (pointers to other BOs) in them. 39 * 40 * The inner workings are a simple hash table based on a CRC of the 41 * key data. 42 * 43 * Replacement is not implemented. Instead, when the cache gets too 44 * big we throw out all of the cache data and let it get regenerated. 45 */ 46 47 #include "main/imports.h" 48 #include "intel_batchbuffer.h" 49 #include "brw_state.h" 50 51 #define FILE_DEBUG_FLAG DEBUG_STATE 52 53 static GLuint 54 hash_key(struct brw_cache_item *item) 55 { 56 GLuint *ikey = (GLuint *)item->key; 57 GLuint hash = item->cache_id, i; 58 59 assert(item->key_size % 4 == 0); 60 61 /* I'm sure this can be improved on: 62 */ 63 for (i = 0; i < item->key_size/4; i++) { 64 hash ^= ikey[i]; 65 hash = (hash << 5) | (hash >> 27); 66 } 67 68 return hash; 69 } 70 71 static int 72 brw_cache_item_equals(const struct brw_cache_item *a, 73 const struct brw_cache_item *b) 74 { 75 return a->cache_id == b->cache_id && 76 a->hash == b->hash && 77 a->key_size == b->key_size && 78 (memcmp(a->key, b->key, a->key_size) == 0); 79 } 80 81 static struct brw_cache_item * 82 search_cache(struct brw_cache *cache, GLuint hash, 83 struct brw_cache_item *lookup) 84 { 85 struct brw_cache_item *c; 86 87 #if 0 88 int bucketcount = 0; 89 90 for (c = cache->items[hash % cache->size]; c; c = c->next) 91 bucketcount++; 92 93 fprintf(stderr, "bucket %d/%d = %d/%d items\n", hash % cache->size, 94 cache->size, bucketcount, cache->n_items); 95 #endif 96 97 for (c = cache->items[hash % cache->size]; c; c = c->next) { 98 if (brw_cache_item_equals(lookup, c)) 99 return c; 100 } 101 102 return NULL; 103 } 104 105 106 static void 107 rehash(struct brw_cache *cache) 108 { 109 struct brw_cache_item **items; 110 struct brw_cache_item *c, *next; 111 GLuint size, i; 112 113 size = cache->size * 3; 114 items = (struct brw_cache_item**) calloc(1, size * sizeof(*items)); 115 116 for (i = 0; i < cache->size; i++) 117 for (c = cache->items[i]; c; c = next) { 118 next = c->next; 119 c->next = items[c->hash % size]; 120 items[c->hash % size] = c; 121 } 122 123 FREE(cache->items); 124 cache->items = items; 125 cache->size = size; 126 } 127 128 129 /** 130 * Returns the buffer object matching cache_id and key, or NULL. 131 */ 132 bool 133 brw_search_cache(struct brw_cache *cache, 134 enum brw_cache_id cache_id, 135 const void *key, GLuint key_size, 136 uint32_t *inout_offset, void *out_aux) 137 { 138 struct brw_context *brw = cache->brw; 139 struct brw_cache_item *item; 140 struct brw_cache_item lookup; 141 GLuint hash; 142 143 lookup.cache_id = cache_id; 144 lookup.key = key; 145 lookup.key_size = key_size; 146 hash = hash_key(&lookup); 147 lookup.hash = hash; 148 149 item = search_cache(cache, hash, &lookup); 150 151 if (item == NULL) 152 return false; 153 154 *(void **)out_aux = ((char *)item->key + item->key_size); 155 156 if (item->offset != *inout_offset) { 157 brw->state.dirty.cache |= (1 << cache_id); 158 *inout_offset = item->offset; 159 } 160 161 return true; 162 } 163 164 static void 165 brw_cache_new_bo(struct brw_cache *cache, uint32_t new_size) 166 { 167 struct brw_context *brw = cache->brw; 168 struct intel_context *intel = &brw->intel; 169 drm_intel_bo *new_bo; 170 171 new_bo = drm_intel_bo_alloc(intel->bufmgr, "program cache", new_size, 64); 172 173 /* Copy any existing data that needs to be saved. */ 174 if (cache->next_offset != 0) { 175 drm_intel_bo_map(cache->bo, false); 176 drm_intel_bo_subdata(new_bo, 0, cache->next_offset, cache->bo->virtual); 177 drm_intel_bo_unmap(cache->bo); 178 } 179 180 drm_intel_bo_unreference(cache->bo); 181 cache->bo = new_bo; 182 cache->bo_used_by_gpu = false; 183 184 /* Since we have a new BO in place, we need to signal the units 185 * that depend on it (state base address on gen5+, or unit state before). 186 */ 187 brw->state.dirty.brw |= BRW_NEW_PROGRAM_CACHE; 188 } 189 190 /** 191 * Attempts to find an item in the cache with identical data and aux 192 * data to use 193 */ 194 static bool 195 brw_try_upload_using_copy(struct brw_cache *cache, 196 struct brw_cache_item *result_item, 197 const void *data, 198 const void *aux) 199 { 200 int i; 201 struct brw_cache_item *item; 202 203 for (i = 0; i < cache->size; i++) { 204 for (item = cache->items[i]; item; item = item->next) { 205 const void *item_aux = item->key + item->key_size; 206 int ret; 207 208 if (item->cache_id != result_item->cache_id || 209 item->size != result_item->size || 210 item->aux_size != result_item->aux_size) { 211 continue; 212 } 213 214 if (memcmp(item_aux, aux, item->aux_size) != 0) { 215 continue; 216 } 217 218 drm_intel_bo_map(cache->bo, false); 219 ret = memcmp(cache->bo->virtual + item->offset, data, item->size); 220 drm_intel_bo_unmap(cache->bo); 221 if (ret) 222 continue; 223 224 result_item->offset = item->offset; 225 226 return true; 227 } 228 } 229 230 return false; 231 } 232 233 static void 234 brw_upload_item_data(struct brw_cache *cache, 235 struct brw_cache_item *item, 236 const void *data) 237 { 238 /* Allocate space in the cache BO for our new program. */ 239 if (cache->next_offset + item->size > cache->bo->size) { 240 uint32_t new_size = cache->bo->size * 2; 241 242 while (cache->next_offset + item->size > new_size) 243 new_size *= 2; 244 245 brw_cache_new_bo(cache, new_size); 246 } 247 248 /* If we would block on writing to an in-use program BO, just 249 * recreate it. 250 */ 251 if (cache->bo_used_by_gpu) { 252 brw_cache_new_bo(cache, cache->bo->size); 253 } 254 255 item->offset = cache->next_offset; 256 257 /* Programs are always 64-byte aligned, so set up the next one now */ 258 cache->next_offset = ALIGN(item->offset + item->size, 64); 259 } 260 261 void 262 brw_upload_cache(struct brw_cache *cache, 263 enum brw_cache_id cache_id, 264 const void *key, 265 GLuint key_size, 266 const void *data, 267 GLuint data_size, 268 const void *aux, 269 GLuint aux_size, 270 uint32_t *out_offset, 271 void *out_aux) 272 { 273 struct brw_cache_item *item = CALLOC_STRUCT(brw_cache_item); 274 GLuint hash; 275 void *tmp; 276 277 item->cache_id = cache_id; 278 item->size = data_size; 279 item->key = key; 280 item->key_size = key_size; 281 item->aux_size = aux_size; 282 hash = hash_key(item); 283 item->hash = hash; 284 285 /* If we can find a matching prog/prog_data combo in the cache 286 * already, then reuse the existing stuff. This will mean not 287 * flagging CACHE_NEW_* when transitioning between the two 288 * equivalent hash keys. This is notably useful for programs 289 * generating shaders at runtime, where multiple shaders may 290 * compile to the thing in our backend. 291 */ 292 if (!brw_try_upload_using_copy(cache, item, data, aux)) { 293 brw_upload_item_data(cache, item, data); 294 } 295 296 /* Set up the memory containing the key and aux_data */ 297 tmp = malloc(key_size + aux_size); 298 299 memcpy(tmp, key, key_size); 300 memcpy(tmp + key_size, aux, aux_size); 301 302 item->key = tmp; 303 304 if (cache->n_items > cache->size * 1.5) 305 rehash(cache); 306 307 hash %= cache->size; 308 item->next = cache->items[hash]; 309 cache->items[hash] = item; 310 cache->n_items++; 311 312 /* Copy data to the buffer */ 313 drm_intel_bo_subdata(cache->bo, item->offset, data_size, data); 314 315 *out_offset = item->offset; 316 *(void **)out_aux = (void *)((char *)item->key + item->key_size); 317 cache->brw->state.dirty.cache |= 1 << cache_id; 318 } 319 320 void 321 brw_init_caches(struct brw_context *brw) 322 { 323 struct intel_context *intel = &brw->intel; 324 struct brw_cache *cache = &brw->cache; 325 326 cache->brw = brw; 327 328 cache->size = 7; 329 cache->n_items = 0; 330 cache->items = (struct brw_cache_item **) 331 calloc(1, cache->size * sizeof(struct brw_cache_item)); 332 333 cache->bo = drm_intel_bo_alloc(intel->bufmgr, 334 "program cache", 335 4096, 64); 336 } 337 338 static void 339 brw_clear_cache(struct brw_context *brw, struct brw_cache *cache) 340 { 341 struct intel_context *intel = &brw->intel; 342 struct brw_cache_item *c, *next; 343 GLuint i; 344 345 DBG("%s\n", __FUNCTION__); 346 347 for (i = 0; i < cache->size; i++) { 348 for (c = cache->items[i]; c; c = next) { 349 next = c->next; 350 free((void *)c->key); 351 free(c); 352 } 353 cache->items[i] = NULL; 354 } 355 356 cache->n_items = 0; 357 358 /* Start putting programs into the start of the BO again, since 359 * we'll never find the old results. 360 */ 361 cache->next_offset = 0; 362 363 /* We need to make sure that the programs get regenerated, since 364 * any offsets leftover in brw_context will no longer be valid. 365 */ 366 brw->state.dirty.mesa |= ~0; 367 brw->state.dirty.brw |= ~0; 368 brw->state.dirty.cache |= ~0; 369 intel_batchbuffer_flush(intel); 370 } 371 372 void 373 brw_state_cache_check_size(struct brw_context *brw) 374 { 375 /* un-tuned guess. Each object is generally a page, so 2000 of them is 8 MB of 376 * state cache. 377 */ 378 if (brw->cache.n_items > 2000) { 379 perf_debug("Exceeded state cache size limit. Clearing the set " 380 "of compiled programs, which will trigger recompiles\n"); 381 brw_clear_cache(brw, &brw->cache); 382 } 383 } 384 385 386 static void 387 brw_destroy_cache(struct brw_context *brw, struct brw_cache *cache) 388 { 389 390 DBG("%s\n", __FUNCTION__); 391 392 drm_intel_bo_unreference(cache->bo); 393 cache->bo = NULL; 394 brw_clear_cache(brw, cache); 395 free(cache->items); 396 cache->items = NULL; 397 cache->size = 0; 398 } 399 400 401 void 402 brw_destroy_caches(struct brw_context *brw) 403 { 404 brw_destroy_cache(brw, &brw->cache); 405 } 406
