1 /* 2 * Copyright (C) 2016 Rob Clark <robclark (at) freedesktop.org> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: 24 * Rob Clark <robclark (at) freedesktop.org> 25 */ 26 27 #include "util/hash_table.h" 28 #include "util/set.h" 29 #include "util/list.h" 30 #include "util/u_string.h" 31 32 #include "freedreno_batch.h" 33 #include "freedreno_batch_cache.h" 34 #include "freedreno_context.h" 35 #include "freedreno_resource.h" 36 37 /* Overview: 38 * 39 * The batch cache provides lookup for mapping pipe_framebuffer_state 40 * to a batch. 41 * 42 * It does this via hashtable, with key that roughly matches the 43 * pipe_framebuffer_state, as described below. 44 * 45 * Batch Cache hashtable key: 46 * 47 * To serialize the key, and to avoid dealing with holding a reference to 48 * pipe_surface's (which hold a reference to pipe_resource and complicate 49 * the whole refcnting thing), the key is variable length and inline's the 50 * pertinent details of the pipe_surface. 51 * 52 * Batch: 53 * 54 * Each batch needs to hold a reference to each resource it depends on (ie. 55 * anything that needs a mem2gmem). And a weak reference to resources it 56 * renders to. (If both src[n] and dst[n] are not NULL then they are the 57 * same.) 58 * 59 * When a resource is destroyed, we need to remove entries in the batch 60 * cache that reference the resource, to avoid dangling pointer issues. 61 * So each resource holds a hashset of batches which have reference them 62 * in their hashtable key. 63 * 64 * When a batch has weak reference to no more resources (ie. all the 65 * surfaces it rendered to are destroyed) the batch can be destroyed. 66 * Could happen in an app that renders and never uses the result. More 67 * common scenario, I think, will be that some, but not all, of the 68 * surfaces are destroyed before the batch is submitted. 69 * 70 * If (for example), batch writes to zsbuf but that surface is destroyed 71 * before batch is submitted, we can skip gmem2mem (but still need to 72 * alloc gmem space as before. If the batch depended on previous contents 73 * of that surface, it would be holding a reference so the surface would 74 * not have been destroyed. 75 */ 76 77 struct key { 78 uint32_t width, height, layers; 79 uint16_t samples, num_surfs; 80 struct fd_context *ctx; 81 struct { 82 struct pipe_resource *texture; 83 union pipe_surface_desc u; 84 uint16_t pos, format; 85 } surf[0]; 86 }; 87 88 static struct key * 89 key_alloc(unsigned num_surfs) 90 { 91 struct key *key = 92 CALLOC_VARIANT_LENGTH_STRUCT(key, sizeof(key->surf[0]) * num_surfs); 93 return key; 94 } 95 96 static uint32_t 97 key_hash(const void *_key) 98 { 99 const struct key *key = _key; 100 uint32_t hash = _mesa_fnv32_1a_offset_bias; 101 hash = _mesa_fnv32_1a_accumulate_block(hash, key, offsetof(struct key, surf[0])); 102 hash = _mesa_fnv32_1a_accumulate_block(hash, key->surf, sizeof(key->surf[0]) * key->num_surfs); 103 return hash; 104 } 105 106 static bool 107 key_equals(const void *_a, const void *_b) 108 { 109 const struct key *a = _a; 110 const struct key *b = _b; 111 return (memcmp(a, b, offsetof(struct key, surf[0])) == 0) && 112 (memcmp(a->surf, b->surf, sizeof(a->surf[0]) * a->num_surfs) == 0); 113 } 114 115 void 116 fd_bc_init(struct fd_batch_cache *cache) 117 { 118 cache->ht = _mesa_hash_table_create(NULL, key_hash, key_equals); 119 } 120 121 void 122 fd_bc_fini(struct fd_batch_cache *cache) 123 { 124 _mesa_hash_table_destroy(cache->ht, NULL); 125 } 126 127 void 128 fd_bc_flush(struct fd_batch_cache *cache, struct fd_context *ctx) 129 { 130 struct hash_entry *entry; 131 struct fd_batch *last_batch = NULL; 132 133 pipe_mutex_lock(ctx->screen->lock); 134 135 hash_table_foreach(cache->ht, entry) { 136 struct fd_batch *batch = NULL; 137 fd_batch_reference_locked(&batch, (struct fd_batch *)entry->data); 138 if (batch->ctx == ctx) { 139 pipe_mutex_unlock(ctx->screen->lock); 140 fd_batch_reference(&last_batch, batch); 141 fd_batch_flush(batch, false); 142 pipe_mutex_lock(ctx->screen->lock); 143 } 144 fd_batch_reference_locked(&batch, NULL); 145 } 146 147 pipe_mutex_unlock(ctx->screen->lock); 148 149 if (last_batch) { 150 fd_batch_sync(last_batch); 151 fd_batch_reference(&last_batch, NULL); 152 } 153 } 154 155 void 156 fd_bc_invalidate_context(struct fd_context *ctx) 157 { 158 struct fd_batch_cache *cache = &ctx->screen->batch_cache; 159 struct fd_batch *batch; 160 161 pipe_mutex_lock(ctx->screen->lock); 162 163 foreach_batch(batch, cache, cache->batch_mask) { 164 if (batch->ctx == ctx) 165 fd_batch_reference_locked(&batch, NULL); 166 } 167 168 pipe_mutex_unlock(ctx->screen->lock); 169 } 170 171 void 172 fd_bc_invalidate_batch(struct fd_batch *batch, bool destroy) 173 { 174 if (!batch) 175 return; 176 177 struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache; 178 struct key *key = (struct key *)batch->key; 179 180 pipe_mutex_assert_locked(batch->ctx->screen->lock); 181 182 if (destroy) { 183 cache->batches[batch->idx] = NULL; 184 cache->batch_mask &= ~(1 << batch->idx); 185 } 186 187 if (!key) 188 return; 189 190 DBG("%p: key=%p", batch, batch->key); 191 for (unsigned idx = 0; idx < key->num_surfs; idx++) { 192 struct fd_resource *rsc = fd_resource(key->surf[idx].texture); 193 rsc->bc_batch_mask &= ~(1 << batch->idx); 194 } 195 196 struct hash_entry *entry = 197 _mesa_hash_table_search_pre_hashed(cache->ht, batch->hash, key); 198 _mesa_hash_table_remove(cache->ht, entry); 199 200 batch->key = NULL; 201 free(key); 202 } 203 204 void 205 fd_bc_invalidate_resource(struct fd_resource *rsc, bool destroy) 206 { 207 struct fd_screen *screen = fd_screen(rsc->base.b.screen); 208 struct fd_batch *batch; 209 210 pipe_mutex_lock(screen->lock); 211 212 if (destroy) { 213 foreach_batch(batch, &screen->batch_cache, rsc->batch_mask) { 214 struct set_entry *entry = _mesa_set_search(batch->resources, rsc); 215 _mesa_set_remove(batch->resources, entry); 216 } 217 rsc->batch_mask = 0; 218 219 fd_batch_reference_locked(&rsc->write_batch, NULL); 220 } 221 222 foreach_batch(batch, &screen->batch_cache, rsc->bc_batch_mask) 223 fd_bc_invalidate_batch(batch, false); 224 225 rsc->bc_batch_mask = 0; 226 227 pipe_mutex_unlock(screen->lock); 228 } 229 230 struct fd_batch * 231 fd_bc_alloc_batch(struct fd_batch_cache *cache, struct fd_context *ctx) 232 { 233 struct fd_batch *batch; 234 uint32_t idx; 235 236 pipe_mutex_lock(ctx->screen->lock); 237 238 while ((idx = ffs(~cache->batch_mask)) == 0) { 239 #if 0 240 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) { 241 batch = cache->batches[i]; 242 debug_printf("%d: needs_flush=%d, depends:", batch->idx, batch->needs_flush); 243 struct set_entry *entry; 244 set_foreach(batch->dependencies, entry) { 245 struct fd_batch *dep = (struct fd_batch *)entry->key; 246 debug_printf(" %d", dep->idx); 247 } 248 debug_printf("\n"); 249 } 250 #endif 251 /* TODO: is LRU the better policy? Or perhaps the batch that 252 * depends on the fewest other batches? 253 */ 254 struct fd_batch *flush_batch = NULL; 255 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) { 256 if ((cache->batches[i] == ctx->batch) || 257 !cache->batches[i]->needs_flush) 258 continue; 259 if (!flush_batch || (cache->batches[i]->seqno < flush_batch->seqno)) 260 fd_batch_reference_locked(&flush_batch, cache->batches[i]); 261 } 262 263 /* we can drop lock temporarily here, since we hold a ref, 264 * flush_batch won't disappear under us. 265 */ 266 pipe_mutex_unlock(ctx->screen->lock); 267 DBG("%p: too many batches! flush forced!", flush_batch); 268 fd_batch_flush(flush_batch, true); 269 pipe_mutex_lock(ctx->screen->lock); 270 271 /* While the resources get cleaned up automatically, the flush_batch 272 * doesn't get removed from the dependencies of other batches, so 273 * it won't be unref'd and will remain in the table. 274 * 275 * TODO maybe keep a bitmask of batches that depend on me, to make 276 * this easier: 277 */ 278 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) { 279 struct fd_batch *other = cache->batches[i]; 280 if (!other) 281 continue; 282 if (other->dependents_mask & (1 << flush_batch->idx)) { 283 other->dependents_mask &= ~(1 << flush_batch->idx); 284 struct fd_batch *ref = flush_batch; 285 fd_batch_reference_locked(&ref, NULL); 286 } 287 } 288 289 fd_batch_reference_locked(&flush_batch, NULL); 290 } 291 292 idx--; /* bit zero returns 1 for ffs() */ 293 294 batch = fd_batch_create(ctx); 295 if (!batch) 296 goto out; 297 298 batch->seqno = cache->cnt++; 299 batch->idx = idx; 300 cache->batch_mask |= (1 << idx); 301 302 debug_assert(cache->batches[idx] == NULL); 303 cache->batches[idx] = batch; 304 305 out: 306 pipe_mutex_unlock(ctx->screen->lock); 307 308 return batch; 309 } 310 311 static struct fd_batch * 312 batch_from_key(struct fd_batch_cache *cache, struct key *key, 313 struct fd_context *ctx) 314 { 315 struct fd_batch *batch = NULL; 316 uint32_t hash = key_hash(key); 317 struct hash_entry *entry = 318 _mesa_hash_table_search_pre_hashed(cache->ht, hash, key); 319 320 if (entry) { 321 free(key); 322 fd_batch_reference(&batch, (struct fd_batch *)entry->data); 323 return batch; 324 } 325 326 batch = fd_bc_alloc_batch(cache, ctx); 327 #ifdef DEBUG 328 DBG("%p: hash=0x%08x, %ux%u, %u layers, %u samples", batch, hash, 329 key->width, key->height, key->layers, key->samples); 330 for (unsigned idx = 0; idx < key->num_surfs; idx++) { 331 DBG("%p: surf[%u]: %p (%s) (%u,%u / %u,%u,%u)", batch, key->surf[idx].pos, 332 key->surf[idx].texture, util_format_name(key->surf[idx].format), 333 key->surf[idx].u.buf.first_element, key->surf[idx].u.buf.last_element, 334 key->surf[idx].u.tex.first_layer, key->surf[idx].u.tex.last_layer, 335 key->surf[idx].u.tex.level); 336 } 337 #endif 338 if (!batch) 339 return NULL; 340 341 pipe_mutex_lock(ctx->screen->lock); 342 343 _mesa_hash_table_insert_pre_hashed(cache->ht, hash, key, batch); 344 batch->key = key; 345 batch->hash = hash; 346 347 for (unsigned idx = 0; idx < key->num_surfs; idx++) { 348 struct fd_resource *rsc = fd_resource(key->surf[idx].texture); 349 rsc->bc_batch_mask = (1 << batch->idx); 350 } 351 352 pipe_mutex_unlock(ctx->screen->lock); 353 354 return batch; 355 } 356 357 static void 358 key_surf(struct key *key, unsigned idx, unsigned pos, struct pipe_surface *psurf) 359 { 360 key->surf[idx].texture = psurf->texture; 361 key->surf[idx].u = psurf->u; 362 key->surf[idx].pos = pos; 363 key->surf[idx].format = psurf->format; 364 } 365 366 struct fd_batch * 367 fd_batch_from_fb(struct fd_batch_cache *cache, struct fd_context *ctx, 368 const struct pipe_framebuffer_state *pfb) 369 { 370 unsigned idx = 0, n = pfb->nr_cbufs + (pfb->zsbuf ? 1 : 0); 371 struct key *key = key_alloc(n); 372 373 key->width = pfb->width; 374 key->height = pfb->height; 375 key->layers = pfb->layers; 376 key->samples = pfb->samples; 377 key->ctx = ctx; 378 379 if (pfb->zsbuf) 380 key_surf(key, idx++, 0, pfb->zsbuf); 381 382 for (unsigned i = 0; i < pfb->nr_cbufs; i++) 383 if (pfb->cbufs[i]) 384 key_surf(key, idx++, i + 1, pfb->cbufs[i]); 385 386 key->num_surfs = idx; 387 388 return batch_from_key(cache, key, ctx); 389 } 390