1 /* 2 * Copyright (C) 2016 Rob Clark <robclark (at) freedesktop.org> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: 24 * Rob Clark <robclark (at) freedesktop.org> 25 */ 26 27 #include "util/hash_table.h" 28 #include "util/set.h" 29 #include "util/list.h" 30 #include "util/u_string.h" 31 32 #include "freedreno_batch.h" 33 #include "freedreno_batch_cache.h" 34 #include "freedreno_context.h" 35 #include "freedreno_resource.h" 36 37 /* Overview: 38 * 39 * The batch cache provides lookup for mapping pipe_framebuffer_state 40 * to a batch. 41 * 42 * It does this via hashtable, with key that roughly matches the 43 * pipe_framebuffer_state, as described below. 44 * 45 * Batch Cache hashtable key: 46 * 47 * To serialize the key, and to avoid dealing with holding a reference to 48 * pipe_surface's (which hold a reference to pipe_resource and complicate 49 * the whole refcnting thing), the key is variable length and inline's the 50 * pertinent details of the pipe_surface. 51 * 52 * Batch: 53 * 54 * Each batch needs to hold a reference to each resource it depends on (ie. 55 * anything that needs a mem2gmem). And a weak reference to resources it 56 * renders to. (If both src[n] and dst[n] are not NULL then they are the 57 * same.) 58 * 59 * When a resource is destroyed, we need to remove entries in the batch 60 * cache that reference the resource, to avoid dangling pointer issues. 61 * So each resource holds a hashset of batches which have reference them 62 * in their hashtable key. 63 * 64 * When a batch has weak reference to no more resources (ie. all the 65 * surfaces it rendered to are destroyed) the batch can be destroyed. 66 * Could happen in an app that renders and never uses the result. More 67 * common scenario, I think, will be that some, but not all, of the 68 * surfaces are destroyed before the batch is submitted. 69 * 70 * If (for example), batch writes to zsbuf but that surface is destroyed 71 * before batch is submitted, we can skip gmem2mem (but still need to 72 * alloc gmem space as before. If the batch depended on previous contents 73 * of that surface, it would be holding a reference so the surface would 74 * not have been destroyed. 75 */ 76 77 struct key { 78 uint32_t width, height, layers; 79 uint16_t samples, num_surfs; 80 struct fd_context *ctx; 81 struct { 82 struct pipe_resource *texture; 83 union pipe_surface_desc u; 84 uint16_t pos, format; 85 } surf[0]; 86 }; 87 88 static struct key * 89 key_alloc(unsigned num_surfs) 90 { 91 struct key *key = 92 CALLOC_VARIANT_LENGTH_STRUCT(key, sizeof(key->surf[0]) * num_surfs); 93 return key; 94 } 95 96 static uint32_t 97 key_hash(const void *_key) 98 { 99 const struct key *key = _key; 100 uint32_t hash = _mesa_fnv32_1a_offset_bias; 101 hash = _mesa_fnv32_1a_accumulate_block(hash, key, offsetof(struct key, surf[0])); 102 hash = _mesa_fnv32_1a_accumulate_block(hash, key->surf, sizeof(key->surf[0]) * key->num_surfs); 103 return hash; 104 } 105 106 static bool 107 key_equals(const void *_a, const void *_b) 108 { 109 const struct key *a = _a; 110 const struct key *b = _b; 111 return (memcmp(a, b, offsetof(struct key, surf[0])) == 0) && 112 (memcmp(a->surf, b->surf, sizeof(a->surf[0]) * a->num_surfs) == 0); 113 } 114 115 void 116 fd_bc_init(struct fd_batch_cache *cache) 117 { 118 cache->ht = _mesa_hash_table_create(NULL, key_hash, key_equals); 119 } 120 121 void 122 fd_bc_fini(struct fd_batch_cache *cache) 123 { 124 _mesa_hash_table_destroy(cache->ht, NULL); 125 } 126 127 void 128 fd_bc_flush(struct fd_batch_cache *cache, struct fd_context *ctx) 129 { 130 struct hash_entry *entry; 131 struct fd_batch *last_batch = NULL; 132 133 mtx_lock(&ctx->screen->lock); 134 135 hash_table_foreach(cache->ht, entry) { 136 struct fd_batch *batch = NULL; 137 /* hold a reference since we can drop screen->lock: */ 138 fd_batch_reference_locked(&batch, (struct fd_batch *)entry->data); 139 if (batch->ctx == ctx) { 140 mtx_unlock(&ctx->screen->lock); 141 fd_batch_reference(&last_batch, batch); 142 fd_batch_flush(batch, false, false); 143 mtx_lock(&ctx->screen->lock); 144 } 145 fd_batch_reference_locked(&batch, NULL); 146 } 147 148 mtx_unlock(&ctx->screen->lock); 149 150 if (last_batch) { 151 fd_batch_sync(last_batch); 152 fd_batch_reference(&last_batch, NULL); 153 } 154 } 155 156 /* deferred flush doesn't actually flush, but it marks every other 157 * batch associated with the context as dependent on the current 158 * batch. So when the current batch gets flushed, all other batches 159 * that came before also get flushed. 160 */ 161 void 162 fd_bc_flush_deferred(struct fd_batch_cache *cache, struct fd_context *ctx) 163 { 164 struct fd_batch *current_batch = ctx->batch; 165 struct hash_entry *entry; 166 167 mtx_lock(&ctx->screen->lock); 168 169 hash_table_foreach(cache->ht, entry) { 170 struct fd_batch *batch = entry->data; 171 if (batch == current_batch) 172 continue; 173 if (batch->ctx == ctx) 174 fd_batch_add_dep(current_batch, batch); 175 } 176 177 mtx_unlock(&ctx->screen->lock); 178 } 179 180 void 181 fd_bc_invalidate_context(struct fd_context *ctx) 182 { 183 struct fd_batch_cache *cache = &ctx->screen->batch_cache; 184 struct fd_batch *batch; 185 186 mtx_lock(&ctx->screen->lock); 187 188 foreach_batch(batch, cache, cache->batch_mask) { 189 if (batch->ctx == ctx) 190 fd_batch_reference_locked(&batch, NULL); 191 } 192 193 mtx_unlock(&ctx->screen->lock); 194 } 195 196 void 197 fd_bc_invalidate_batch(struct fd_batch *batch, bool destroy) 198 { 199 if (!batch) 200 return; 201 202 struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache; 203 struct key *key = (struct key *)batch->key; 204 205 pipe_mutex_assert_locked(batch->ctx->screen->lock); 206 207 if (destroy) { 208 cache->batches[batch->idx] = NULL; 209 cache->batch_mask &= ~(1 << batch->idx); 210 } 211 212 if (!key) 213 return; 214 215 DBG("%p: key=%p", batch, batch->key); 216 for (unsigned idx = 0; idx < key->num_surfs; idx++) { 217 struct fd_resource *rsc = fd_resource(key->surf[idx].texture); 218 rsc->bc_batch_mask &= ~(1 << batch->idx); 219 } 220 221 struct hash_entry *entry = 222 _mesa_hash_table_search_pre_hashed(cache->ht, batch->hash, key); 223 _mesa_hash_table_remove(cache->ht, entry); 224 225 batch->key = NULL; 226 free(key); 227 } 228 229 void 230 fd_bc_invalidate_resource(struct fd_resource *rsc, bool destroy) 231 { 232 struct fd_screen *screen = fd_screen(rsc->base.screen); 233 struct fd_batch *batch; 234 235 mtx_lock(&screen->lock); 236 237 if (destroy) { 238 foreach_batch(batch, &screen->batch_cache, rsc->batch_mask) { 239 struct set_entry *entry = _mesa_set_search(batch->resources, rsc); 240 _mesa_set_remove(batch->resources, entry); 241 } 242 rsc->batch_mask = 0; 243 244 fd_batch_reference_locked(&rsc->write_batch, NULL); 245 } 246 247 foreach_batch(batch, &screen->batch_cache, rsc->bc_batch_mask) 248 fd_bc_invalidate_batch(batch, false); 249 250 rsc->bc_batch_mask = 0; 251 252 mtx_unlock(&screen->lock); 253 } 254 255 struct fd_batch * 256 fd_bc_alloc_batch(struct fd_batch_cache *cache, struct fd_context *ctx) 257 { 258 struct fd_batch *batch; 259 uint32_t idx; 260 261 mtx_lock(&ctx->screen->lock); 262 263 while ((idx = ffs(~cache->batch_mask)) == 0) { 264 #if 0 265 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) { 266 batch = cache->batches[i]; 267 debug_printf("%d: needs_flush=%d, depends:", batch->idx, batch->needs_flush); 268 struct set_entry *entry; 269 set_foreach(batch->dependencies, entry) { 270 struct fd_batch *dep = (struct fd_batch *)entry->key; 271 debug_printf(" %d", dep->idx); 272 } 273 debug_printf("\n"); 274 } 275 #endif 276 /* TODO: is LRU the better policy? Or perhaps the batch that 277 * depends on the fewest other batches? 278 */ 279 struct fd_batch *flush_batch = NULL; 280 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) { 281 if ((cache->batches[i] == ctx->batch) || 282 !cache->batches[i]->needs_flush) 283 continue; 284 if (!flush_batch || (cache->batches[i]->seqno < flush_batch->seqno)) 285 fd_batch_reference_locked(&flush_batch, cache->batches[i]); 286 } 287 288 /* we can drop lock temporarily here, since we hold a ref, 289 * flush_batch won't disappear under us. 290 */ 291 mtx_unlock(&ctx->screen->lock); 292 DBG("%p: too many batches! flush forced!", flush_batch); 293 fd_batch_flush(flush_batch, true, false); 294 mtx_lock(&ctx->screen->lock); 295 296 /* While the resources get cleaned up automatically, the flush_batch 297 * doesn't get removed from the dependencies of other batches, so 298 * it won't be unref'd and will remain in the table. 299 * 300 * TODO maybe keep a bitmask of batches that depend on me, to make 301 * this easier: 302 */ 303 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) { 304 struct fd_batch *other = cache->batches[i]; 305 if (!other) 306 continue; 307 if (other->dependents_mask & (1 << flush_batch->idx)) { 308 other->dependents_mask &= ~(1 << flush_batch->idx); 309 struct fd_batch *ref = flush_batch; 310 fd_batch_reference_locked(&ref, NULL); 311 } 312 } 313 314 fd_batch_reference_locked(&flush_batch, NULL); 315 } 316 317 idx--; /* bit zero returns 1 for ffs() */ 318 319 batch = fd_batch_create(ctx, false); 320 if (!batch) 321 goto out; 322 323 batch->seqno = cache->cnt++; 324 batch->idx = idx; 325 cache->batch_mask |= (1 << idx); 326 327 debug_assert(cache->batches[idx] == NULL); 328 cache->batches[idx] = batch; 329 330 out: 331 mtx_unlock(&ctx->screen->lock); 332 333 return batch; 334 } 335 336 static struct fd_batch * 337 batch_from_key(struct fd_batch_cache *cache, struct key *key, 338 struct fd_context *ctx) 339 { 340 struct fd_batch *batch = NULL; 341 uint32_t hash = key_hash(key); 342 struct hash_entry *entry = 343 _mesa_hash_table_search_pre_hashed(cache->ht, hash, key); 344 345 if (entry) { 346 free(key); 347 fd_batch_reference(&batch, (struct fd_batch *)entry->data); 348 return batch; 349 } 350 351 batch = fd_bc_alloc_batch(cache, ctx); 352 #ifdef DEBUG 353 DBG("%p: hash=0x%08x, %ux%u, %u layers, %u samples", batch, hash, 354 key->width, key->height, key->layers, key->samples); 355 for (unsigned idx = 0; idx < key->num_surfs; idx++) { 356 DBG("%p: surf[%u]: %p (%s) (%u,%u / %u,%u,%u)", batch, key->surf[idx].pos, 357 key->surf[idx].texture, util_format_name(key->surf[idx].format), 358 key->surf[idx].u.buf.first_element, key->surf[idx].u.buf.last_element, 359 key->surf[idx].u.tex.first_layer, key->surf[idx].u.tex.last_layer, 360 key->surf[idx].u.tex.level); 361 } 362 #endif 363 if (!batch) 364 return NULL; 365 366 mtx_lock(&ctx->screen->lock); 367 368 _mesa_hash_table_insert_pre_hashed(cache->ht, hash, key, batch); 369 batch->key = key; 370 batch->hash = hash; 371 372 for (unsigned idx = 0; idx < key->num_surfs; idx++) { 373 struct fd_resource *rsc = fd_resource(key->surf[idx].texture); 374 rsc->bc_batch_mask = (1 << batch->idx); 375 } 376 377 mtx_unlock(&ctx->screen->lock); 378 379 return batch; 380 } 381 382 static void 383 key_surf(struct key *key, unsigned idx, unsigned pos, struct pipe_surface *psurf) 384 { 385 key->surf[idx].texture = psurf->texture; 386 key->surf[idx].u = psurf->u; 387 key->surf[idx].pos = pos; 388 key->surf[idx].format = psurf->format; 389 } 390 391 struct fd_batch * 392 fd_batch_from_fb(struct fd_batch_cache *cache, struct fd_context *ctx, 393 const struct pipe_framebuffer_state *pfb) 394 { 395 unsigned idx = 0, n = pfb->nr_cbufs + (pfb->zsbuf ? 1 : 0); 396 struct key *key = key_alloc(n); 397 398 key->width = pfb->width; 399 key->height = pfb->height; 400 key->layers = pfb->layers; 401 key->samples = pfb->samples; 402 key->ctx = ctx; 403 404 if (pfb->zsbuf) 405 key_surf(key, idx++, 0, pfb->zsbuf); 406 407 for (unsigned i = 0; i < pfb->nr_cbufs; i++) 408 if (pfb->cbufs[i]) 409 key_surf(key, idx++, i + 1, pfb->cbufs[i]); 410 411 key->num_surfs = idx; 412 413 return batch_from_key(cache, key, ctx); 414 } 415