1 2 #include <inttypes.h> 3 4 #include "util/u_inlines.h" 5 #include "util/u_memory.h" 6 #include "util/list.h" 7 8 #include "nouveau_winsys.h" 9 #include "nouveau_screen.h" 10 #include "nouveau_mm.h" 11 12 /* TODO: Higher orders can waste a lot of space for npot size buffers, should 13 * add an extra cache for such buffer objects. 14 * 15 * HACK: Max order == 21 to accommodate TF2's 1.5 MiB, frequently reallocated 16 * vertex buffer (VM flush (?) decreases performance dramatically). 17 */ 18 19 #define MM_MIN_ORDER 7 /* >= 6 to not violate ARB_map_buffer_alignment */ 20 #define MM_MAX_ORDER 21 21 22 #define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1) 23 24 #define MM_MIN_SIZE (1 << MM_MIN_ORDER) 25 #define MM_MAX_SIZE (1 << MM_MAX_ORDER) 26 27 struct mm_bucket { 28 struct list_head free; 29 struct list_head used; 30 struct list_head full; 31 int num_free; 32 }; 33 34 struct nouveau_mman { 35 struct nouveau_device *dev; 36 struct mm_bucket bucket[MM_NUM_BUCKETS]; 37 uint32_t domain; 38 union nouveau_bo_config config; 39 uint64_t allocated; 40 }; 41 42 struct mm_slab { 43 struct list_head head; 44 struct nouveau_bo *bo; 45 struct nouveau_mman *cache; 46 int order; 47 int count; 48 int free; 49 uint32_t bits[0]; 50 }; 51 52 static int 53 mm_slab_alloc(struct mm_slab *slab) 54 { 55 int i, n, b; 56 57 if (slab->free == 0) 58 return -1; 59 60 for (i = 0; i < (slab->count + 31) / 32; ++i) { 61 b = ffs(slab->bits[i]) - 1; 62 if (b >= 0) { 63 n = i * 32 + b; 64 assert(n < slab->count); 65 slab->free--; 66 slab->bits[i] &= ~(1 << b); 67 return n; 68 } 69 } 70 return -1; 71 } 72 73 static inline void 74 mm_slab_free(struct mm_slab *slab, int i) 75 { 76 assert(i < slab->count); 77 slab->bits[i / 32] |= 1 << (i % 32); 78 slab->free++; 79 assert(slab->free <= slab->count); 80 } 81 82 static inline int 83 mm_get_order(uint32_t size) 84 { 85 int s = __builtin_clz(size) ^ 31; 86 87 if (size > (1 << s)) 88 s += 1; 89 return s; 90 } 91 92 static struct mm_bucket * 93 mm_bucket_by_order(struct nouveau_mman *cache, int order) 94 { 95 if (order > MM_MAX_ORDER) 96 return NULL; 97 return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER]; 98 } 99 100 static struct mm_bucket * 101 mm_bucket_by_size(struct nouveau_mman *cache, unsigned size) 102 { 103 return mm_bucket_by_order(cache, mm_get_order(size)); 104 } 105 106 /* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */ 107 static inline uint32_t 108 mm_default_slab_size(unsigned chunk_order) 109 { 110 static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] = 111 { 112 12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22, 22 113 }; 114 115 assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER); 116 117 return 1 << slab_order[chunk_order - MM_MIN_ORDER]; 118 } 119 120 static int 121 mm_slab_new(struct nouveau_mman *cache, int chunk_order) 122 { 123 struct mm_slab *slab; 124 int words, ret; 125 const uint32_t size = mm_default_slab_size(chunk_order); 126 127 words = ((size >> chunk_order) + 31) / 32; 128 assert(words); 129 130 slab = MALLOC(sizeof(struct mm_slab) + words * 4); 131 if (!slab) 132 return PIPE_ERROR_OUT_OF_MEMORY; 133 134 memset(&slab->bits[0], ~0, words * 4); 135 136 slab->bo = NULL; 137 138 ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config, 139 &slab->bo); 140 if (ret) { 141 FREE(slab); 142 return PIPE_ERROR_OUT_OF_MEMORY; 143 } 144 145 LIST_INITHEAD(&slab->head); 146 147 slab->cache = cache; 148 slab->order = chunk_order; 149 slab->count = slab->free = size >> chunk_order; 150 151 LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free); 152 153 cache->allocated += size; 154 155 if (nouveau_mesa_debug) 156 debug_printf("MM: new slab, total memory = %"PRIu64" KiB\n", 157 cache->allocated / 1024); 158 159 return PIPE_OK; 160 } 161 162 /* @return token to identify slab or NULL if we just allocated a new bo */ 163 struct nouveau_mm_allocation * 164 nouveau_mm_allocate(struct nouveau_mman *cache, 165 uint32_t size, struct nouveau_bo **bo, uint32_t *offset) 166 { 167 struct mm_bucket *bucket; 168 struct mm_slab *slab; 169 struct nouveau_mm_allocation *alloc; 170 int ret; 171 172 bucket = mm_bucket_by_size(cache, size); 173 if (!bucket) { 174 ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config, 175 bo); 176 if (ret) 177 debug_printf("bo_new(%x, %x): %i\n", 178 size, cache->config.nv50.memtype, ret); 179 180 *offset = 0; 181 return NULL; 182 } 183 184 if (!LIST_IS_EMPTY(&bucket->used)) { 185 slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head); 186 } else { 187 if (LIST_IS_EMPTY(&bucket->free)) { 188 mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER)); 189 } 190 slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head); 191 192 LIST_DEL(&slab->head); 193 LIST_ADD(&slab->head, &bucket->used); 194 } 195 196 *offset = mm_slab_alloc(slab) << slab->order; 197 198 alloc = MALLOC_STRUCT(nouveau_mm_allocation); 199 if (!alloc) 200 return NULL; 201 202 nouveau_bo_ref(slab->bo, bo); 203 204 if (slab->free == 0) { 205 LIST_DEL(&slab->head); 206 LIST_ADD(&slab->head, &bucket->full); 207 } 208 209 alloc->next = NULL; 210 alloc->offset = *offset; 211 alloc->priv = (void *)slab; 212 213 return alloc; 214 } 215 216 void 217 nouveau_mm_free(struct nouveau_mm_allocation *alloc) 218 { 219 struct mm_slab *slab = (struct mm_slab *)alloc->priv; 220 struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order); 221 222 mm_slab_free(slab, alloc->offset >> slab->order); 223 224 if (slab->free == slab->count) { 225 LIST_DEL(&slab->head); 226 LIST_ADDTAIL(&slab->head, &bucket->free); 227 } else 228 if (slab->free == 1) { 229 LIST_DEL(&slab->head); 230 LIST_ADDTAIL(&slab->head, &bucket->used); 231 } 232 233 FREE(alloc); 234 } 235 236 void 237 nouveau_mm_free_work(void *data) 238 { 239 nouveau_mm_free(data); 240 } 241 242 struct nouveau_mman * 243 nouveau_mm_create(struct nouveau_device *dev, uint32_t domain, 244 union nouveau_bo_config *config) 245 { 246 struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman); 247 int i; 248 249 if (!cache) 250 return NULL; 251 252 cache->dev = dev; 253 cache->domain = domain; 254 cache->config = *config; 255 cache->allocated = 0; 256 257 for (i = 0; i < MM_NUM_BUCKETS; ++i) { 258 LIST_INITHEAD(&cache->bucket[i].free); 259 LIST_INITHEAD(&cache->bucket[i].used); 260 LIST_INITHEAD(&cache->bucket[i].full); 261 } 262 263 return cache; 264 } 265 266 static inline void 267 nouveau_mm_free_slabs(struct list_head *head) 268 { 269 struct mm_slab *slab, *next; 270 271 LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) { 272 LIST_DEL(&slab->head); 273 nouveau_bo_ref(NULL, &slab->bo); 274 FREE(slab); 275 } 276 } 277 278 void 279 nouveau_mm_destroy(struct nouveau_mman *cache) 280 { 281 int i; 282 283 if (!cache) 284 return; 285 286 for (i = 0; i < MM_NUM_BUCKETS; ++i) { 287 if (!LIST_IS_EMPTY(&cache->bucket[i].used) || 288 !LIST_IS_EMPTY(&cache->bucket[i].full)) 289 debug_printf("WARNING: destroying GPU memory cache " 290 "with some buffers still in use\n"); 291 292 nouveau_mm_free_slabs(&cache->bucket[i].free); 293 nouveau_mm_free_slabs(&cache->bucket[i].used); 294 nouveau_mm_free_slabs(&cache->bucket[i].full); 295 } 296 297 FREE(cache); 298 } 299
