1 /* GLIB sliced memory - fast concurrent memory chunk allocator 2 * Copyright (C) 2005 Tim Janik 3 * 4 * This library is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU Lesser General Public 6 * License as published by the Free Software Foundation; either 7 * version 2 of the License, or (at your option) any later version. 8 * 9 * This library is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * Lesser General Public License for more details. 13 * 14 * You should have received a copy of the GNU Lesser General Public 15 * License along with this library; if not, write to the 16 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 17 * Boston, MA 02111-1307, USA. 18 */ 19 /* MT safe */ 20 21 #include "config.h" 22 23 #if defined HAVE_POSIX_MEMALIGN && defined POSIX_MEMALIGN_WITH_COMPLIANT_ALLOCS 24 # define HAVE_COMPLIANT_POSIX_MEMALIGN 1 25 #endif 26 27 #ifdef HAVE_COMPLIANT_POSIX_MEMALIGN 28 #define _XOPEN_SOURCE 600 /* posix_memalign() */ 29 #endif 30 #include <stdlib.h> /* posix_memalign() */ 31 #include <string.h> 32 #include <errno.h> 33 #include "gmem.h" /* gslice.h */ 34 #include "gthreadprivate.h" 35 #include "glib.h" 36 #include "galias.h" 37 #ifdef HAVE_UNISTD_H 38 #include <unistd.h> /* sysconf() */ 39 #endif 40 #ifdef G_OS_WIN32 41 #include <windows.h> 42 #include <process.h> 43 #endif 44 45 #include <stdio.h> /* fputs/fprintf */ 46 47 48 /* the GSlice allocator is split up into 4 layers, roughly modelled after the slab 49 * allocator and magazine extensions as outlined in: 50 * + [Bonwick94] Jeff Bonwick, The slab allocator: An object-caching kernel 51 * memory allocator. USENIX 1994, http://citeseer.ist.psu.edu/bonwick94slab.html 52 * + [Bonwick01] Bonwick and Jonathan Adams, Magazines and vmem: Extending the 53 * slab allocator to many cpu's and arbitrary resources. 54 * USENIX 2001, http://citeseer.ist.psu.edu/bonwick01magazines.html 55 * the layers are: 56 * - the thread magazines. for each (aligned) chunk size, a magazine (a list) 57 * of recently freed and soon to be allocated chunks is maintained per thread. 58 * this way, most alloc/free requests can be quickly satisfied from per-thread 59 * free lists which only require one g_private_get() call to retrive the 60 * thread handle. 61 * - the magazine cache. allocating and freeing chunks to/from threads only 62 * occours at magazine sizes from a global depot of magazines. the depot 63 * maintaines a 15 second working set of allocated magazines, so full 64 * magazines are not allocated and released too often. 65 * the chunk size dependent magazine sizes automatically adapt (within limits, 66 * see [3]) to lock contention to properly scale performance across a variety 67 * of SMP systems. 68 * - the slab allocator. this allocator allocates slabs (blocks of memory) close 69 * to the system page size or multiples thereof which have to be page aligned. 70 * the blocks are divided into smaller chunks which are used to satisfy 71 * allocations from the upper layers. the space provided by the reminder of 72 * the chunk size division is used for cache colorization (random distribution 73 * of chunk addresses) to improve processor cache utilization. multiple slabs 74 * with the same chunk size are kept in a partially sorted ring to allow O(1) 75 * freeing and allocation of chunks (as long as the allocation of an entirely 76 * new slab can be avoided). 77 * - the page allocator. on most modern systems, posix_memalign(3) or 78 * memalign(3) should be available, so this is used to allocate blocks with 79 * system page size based alignments and sizes or multiples thereof. 80 * if no memalign variant is provided, valloc() is used instead and 81 * block sizes are limited to the system page size (no multiples thereof). 82 * as a fallback, on system without even valloc(), a malloc(3)-based page 83 * allocator with alloc-only behaviour is used. 84 * 85 * NOTES: 86 * [1] some systems memalign(3) implementations may rely on boundary tagging for 87 * the handed out memory chunks. to avoid excessive page-wise fragmentation, 88 * we reserve 2 * sizeof (void*) per block size for the systems memalign(3), 89 * specified in NATIVE_MALLOC_PADDING. 90 * [2] using the slab allocator alone already provides for a fast and efficient 91 * allocator, it doesn't properly scale beyond single-threaded uses though. 92 * also, the slab allocator implements eager free(3)-ing, i.e. does not 93 * provide any form of caching or working set maintenance. so if used alone, 94 * it's vulnerable to trashing for sequences of balanced (alloc, free) pairs 95 * at certain thresholds. 96 * [3] magazine sizes are bound by an implementation specific minimum size and 97 * a chunk size specific maximum to limit magazine storage sizes to roughly 98 * 16KB. 99 * [4] allocating ca. 8 chunks per block/page keeps a good balance between 100 * external and internal fragmentation (<= 12.5%). [Bonwick94] 101 */ 102 103 /* --- macros and constants --- */ 104 #define LARGEALIGNMENT (256) 105 #define P2ALIGNMENT (2 * sizeof (gsize)) /* fits 2 pointers (assumed to be 2 * GLIB_SIZEOF_SIZE_T below) */ 106 #define ALIGN(size, base) ((base) * (gsize) (((size) + (base) - 1) / (base))) 107 #define NATIVE_MALLOC_PADDING P2ALIGNMENT /* per-page padding left for native malloc(3) see [1] */ 108 #define SLAB_INFO_SIZE P2ALIGN (sizeof (SlabInfo) + NATIVE_MALLOC_PADDING) 109 #define MAX_MAGAZINE_SIZE (256) /* see [3] and allocator_get_magazine_threshold() for this */ 110 #define MIN_MAGAZINE_SIZE (4) 111 #define MAX_STAMP_COUNTER (7) /* distributes the load of gettimeofday() */ 112 #define MAX_SLAB_CHUNK_SIZE(al) (((al)->max_page_size - SLAB_INFO_SIZE) / 8) /* we want at last 8 chunks per page, see [4] */ 113 #define MAX_SLAB_INDEX(al) (SLAB_INDEX (al, MAX_SLAB_CHUNK_SIZE (al)) + 1) 114 #define SLAB_INDEX(al, asize) ((asize) / P2ALIGNMENT - 1) /* asize must be P2ALIGNMENT aligned */ 115 #define SLAB_CHUNK_SIZE(al, ix) (((ix) + 1) * P2ALIGNMENT) 116 #define SLAB_BPAGE_SIZE(al,csz) (8 * (csz) + SLAB_INFO_SIZE) 117 118 /* optimized version of ALIGN (size, P2ALIGNMENT) */ 119 #if GLIB_SIZEOF_SIZE_T * 2 == 8 /* P2ALIGNMENT */ 120 #define P2ALIGN(size) (((size) + 0x7) & ~(gsize) 0x7) 121 #elif GLIB_SIZEOF_SIZE_T * 2 == 16 /* P2ALIGNMENT */ 122 #define P2ALIGN(size) (((size) + 0xf) & ~(gsize) 0xf) 123 #else 124 #define P2ALIGN(size) ALIGN (size, P2ALIGNMENT) 125 #endif 126 127 /* special helpers to avoid gmessage.c dependency */ 128 static void mem_error (const char *format, ...) G_GNUC_PRINTF (1,2); 129 #define mem_assert(cond) do { if (G_LIKELY (cond)) ; else mem_error ("assertion failed: %s", #cond); } while (0) 130 131 /* --- structures --- */ 132 typedef struct _ChunkLink ChunkLink; 133 typedef struct _SlabInfo SlabInfo; 134 typedef struct _CachedMagazine CachedMagazine; 135 struct _ChunkLink { 136 ChunkLink *next; 137 ChunkLink *data; 138 }; 139 struct _SlabInfo { 140 ChunkLink *chunks; 141 guint n_allocated; 142 SlabInfo *next, *prev; 143 }; 144 typedef struct { 145 ChunkLink *chunks; 146 gsize count; /* approximative chunks list length */ 147 } Magazine; 148 typedef struct { 149 Magazine *magazine1; /* array of MAX_SLAB_INDEX (allocator) */ 150 Magazine *magazine2; /* array of MAX_SLAB_INDEX (allocator) */ 151 } ThreadMemory; 152 typedef struct { 153 gboolean always_malloc; 154 gboolean bypass_magazines; 155 gboolean debug_blocks; 156 gsize working_set_msecs; 157 guint color_increment; 158 } SliceConfig; 159 typedef struct { 160 /* const after initialization */ 161 gsize min_page_size, max_page_size; 162 SliceConfig config; 163 gsize max_slab_chunk_size_for_magazine_cache; 164 /* magazine cache */ 165 GMutex *magazine_mutex; 166 ChunkLink **magazines; /* array of MAX_SLAB_INDEX (allocator) */ 167 guint *contention_counters; /* array of MAX_SLAB_INDEX (allocator) */ 168 gint mutex_counter; 169 guint stamp_counter; 170 guint last_stamp; 171 /* slab allocator */ 172 GMutex *slab_mutex; 173 SlabInfo **slab_stack; /* array of MAX_SLAB_INDEX (allocator) */ 174 guint color_accu; 175 } Allocator; 176 177 /* --- g-slice prototypes --- */ 178 static gpointer slab_allocator_alloc_chunk (gsize chunk_size); 179 static void slab_allocator_free_chunk (gsize chunk_size, 180 gpointer mem); 181 static void private_thread_memory_cleanup (gpointer data); 182 static gpointer allocator_memalign (gsize alignment, 183 gsize memsize); 184 static void allocator_memfree (gsize memsize, 185 gpointer mem); 186 static inline void magazine_cache_update_stamp (void); 187 static inline gsize allocator_get_magazine_threshold (Allocator *allocator, 188 guint ix); 189 190 /* --- g-slice memory checker --- */ 191 static void smc_notify_alloc (void *pointer, 192 size_t size); 193 static int smc_notify_free (void *pointer, 194 size_t size); 195 196 /* --- variables --- */ 197 static GPrivate *private_thread_memory = NULL; 198 static gsize sys_page_size = 0; 199 static Allocator allocator[1] = { { 0, }, }; 200 static SliceConfig slice_config = { 201 FALSE, /* always_malloc */ 202 FALSE, /* bypass_magazines */ 203 FALSE, /* debug_blocks */ 204 15 * 1000, /* working_set_msecs */ 205 1, /* color increment, alt: 0x7fffffff */ 206 }; 207 static GMutex *smc_tree_mutex = NULL; /* mutex for G_SLICE=debug-blocks */ 208 209 /* --- auxillary funcitons --- */ 210 void 211 g_slice_set_config (GSliceConfig ckey, 212 gint64 value) 213 { 214 g_return_if_fail (sys_page_size == 0); 215 switch (ckey) 216 { 217 case G_SLICE_CONFIG_ALWAYS_MALLOC: 218 slice_config.always_malloc = value != 0; 219 break; 220 case G_SLICE_CONFIG_BYPASS_MAGAZINES: 221 slice_config.bypass_magazines = value != 0; 222 break; 223 case G_SLICE_CONFIG_WORKING_SET_MSECS: 224 slice_config.working_set_msecs = value; 225 break; 226 case G_SLICE_CONFIG_COLOR_INCREMENT: 227 slice_config.color_increment = value; 228 default: ; 229 } 230 } 231 232 gint64 233 g_slice_get_config (GSliceConfig ckey) 234 { 235 switch (ckey) 236 { 237 case G_SLICE_CONFIG_ALWAYS_MALLOC: 238 return slice_config.always_malloc; 239 case G_SLICE_CONFIG_BYPASS_MAGAZINES: 240 return slice_config.bypass_magazines; 241 case G_SLICE_CONFIG_WORKING_SET_MSECS: 242 return slice_config.working_set_msecs; 243 case G_SLICE_CONFIG_CHUNK_SIZES: 244 return MAX_SLAB_INDEX (allocator); 245 case G_SLICE_CONFIG_COLOR_INCREMENT: 246 return slice_config.color_increment; 247 default: 248 return 0; 249 } 250 } 251 252 gint64* 253 g_slice_get_config_state (GSliceConfig ckey, 254 gint64 address, 255 guint *n_values) 256 { 257 guint i = 0; 258 g_return_val_if_fail (n_values != NULL, NULL); 259 *n_values = 0; 260 switch (ckey) 261 { 262 gint64 array[64]; 263 case G_SLICE_CONFIG_CONTENTION_COUNTER: 264 array[i++] = SLAB_CHUNK_SIZE (allocator, address); 265 array[i++] = allocator->contention_counters[address]; 266 array[i++] = allocator_get_magazine_threshold (allocator, address); 267 *n_values = i; 268 return g_memdup (array, sizeof (array[0]) * *n_values); 269 default: 270 return NULL; 271 } 272 } 273 274 static void 275 slice_config_init (SliceConfig *config) 276 { 277 /* don't use g_malloc/g_message here */ 278 gchar buffer[1024]; 279 const gchar *val = _g_getenv_nomalloc ("G_SLICE", buffer); 280 const GDebugKey keys[] = { 281 { "always-malloc", 1 << 0 }, 282 { "debug-blocks", 1 << 1 }, 283 }; 284 gint flags = !val ? 0 : g_parse_debug_string (val, keys, G_N_ELEMENTS (keys)); 285 *config = slice_config; 286 if (flags & (1 << 0)) /* always-malloc */ 287 config->always_malloc = TRUE; 288 if (flags & (1 << 1)) /* debug-blocks */ 289 config->debug_blocks = TRUE; 290 } 291 292 static void 293 g_slice_init_nomessage (void) 294 { 295 /* we may not use g_error() or friends here */ 296 mem_assert (sys_page_size == 0); 297 mem_assert (MIN_MAGAZINE_SIZE >= 4); 298 299 #ifdef G_OS_WIN32 300 { 301 SYSTEM_INFO system_info; 302 GetSystemInfo (&system_info); 303 sys_page_size = system_info.dwPageSize; 304 } 305 #else 306 sys_page_size = sysconf (_SC_PAGESIZE); /* = sysconf (_SC_PAGE_SIZE); = getpagesize(); */ 307 #endif 308 mem_assert (sys_page_size >= 2 * LARGEALIGNMENT); 309 mem_assert ((sys_page_size & (sys_page_size - 1)) == 0); 310 slice_config_init (&allocator->config); 311 allocator->min_page_size = sys_page_size; 312 #if HAVE_COMPLIANT_POSIX_MEMALIGN || HAVE_MEMALIGN 313 /* allow allocation of pages up to 8KB (with 8KB alignment). 314 * this is useful because many medium to large sized structures 315 * fit less than 8 times (see [4]) into 4KB pages. 316 * we allow very small page sizes here, to reduce wastage in 317 * threads if only small allocations are required (this does 318 * bear the risk of incresing allocation times and fragmentation 319 * though). 320 */ 321 allocator->min_page_size = MAX (allocator->min_page_size, 4096); 322 allocator->max_page_size = MAX (allocator->min_page_size, 8192); 323 allocator->min_page_size = MIN (allocator->min_page_size, 128); 324 #else 325 /* we can only align to system page size */ 326 allocator->max_page_size = sys_page_size; 327 #endif 328 allocator->magazine_mutex = NULL; /* _g_slice_thread_init_nomessage() */ 329 allocator->magazines = g_new0 (ChunkLink*, MAX_SLAB_INDEX (allocator)); 330 allocator->contention_counters = g_new0 (guint, MAX_SLAB_INDEX (allocator)); 331 allocator->mutex_counter = 0; 332 allocator->stamp_counter = MAX_STAMP_COUNTER; /* force initial update */ 333 allocator->last_stamp = 0; 334 allocator->slab_mutex = NULL; /* _g_slice_thread_init_nomessage() */ 335 allocator->slab_stack = g_new0 (SlabInfo*, MAX_SLAB_INDEX (allocator)); 336 allocator->color_accu = 0; 337 magazine_cache_update_stamp(); 338 /* values cached for performance reasons */ 339 allocator->max_slab_chunk_size_for_magazine_cache = MAX_SLAB_CHUNK_SIZE (allocator); 340 if (allocator->config.always_malloc || allocator->config.bypass_magazines) 341 allocator->max_slab_chunk_size_for_magazine_cache = 0; /* non-optimized cases */ 342 /* at this point, g_mem_gc_friendly() should be initialized, this 343 * should have been accomplished by the above g_malloc/g_new calls 344 */ 345 } 346 347 static inline guint 348 allocator_categorize (gsize aligned_chunk_size) 349 { 350 /* speed up the likely path */ 351 if (G_LIKELY (aligned_chunk_size && aligned_chunk_size <= allocator->max_slab_chunk_size_for_magazine_cache)) 352 return 1; /* use magazine cache */ 353 354 /* the above will fail (max_slab_chunk_size_for_magazine_cache == 0) if the 355 * allocator is still uninitialized, or if we are not configured to use the 356 * magazine cache. 357 */ 358 if (!sys_page_size) 359 g_slice_init_nomessage (); 360 if (!allocator->config.always_malloc && 361 aligned_chunk_size && 362 aligned_chunk_size <= MAX_SLAB_CHUNK_SIZE (allocator)) 363 { 364 if (allocator->config.bypass_magazines) 365 return 2; /* use slab allocator, see [2] */ 366 return 1; /* use magazine cache */ 367 } 368 return 0; /* use malloc() */ 369 } 370 371 void 372 _g_slice_thread_init_nomessage (void) 373 { 374 /* we may not use g_error() or friends here */ 375 if (!sys_page_size) 376 g_slice_init_nomessage(); 377 else 378 { 379 /* g_slice_init_nomessage() has been called already, probably due 380 * to a g_slice_alloc1() before g_thread_init(). 381 */ 382 } 383 private_thread_memory = g_private_new (private_thread_memory_cleanup); 384 allocator->magazine_mutex = g_mutex_new(); 385 allocator->slab_mutex = g_mutex_new(); 386 if (allocator->config.debug_blocks) 387 smc_tree_mutex = g_mutex_new(); 388 } 389 390 static inline void 391 g_mutex_lock_a (GMutex *mutex, 392 guint *contention_counter) 393 { 394 gboolean contention = FALSE; 395 if (!g_mutex_trylock (mutex)) 396 { 397 g_mutex_lock (mutex); 398 contention = TRUE; 399 } 400 if (contention) 401 { 402 allocator->mutex_counter++; 403 if (allocator->mutex_counter >= 1) /* quickly adapt to contention */ 404 { 405 allocator->mutex_counter = 0; 406 *contention_counter = MIN (*contention_counter + 1, MAX_MAGAZINE_SIZE); 407 } 408 } 409 else /* !contention */ 410 { 411 allocator->mutex_counter--; 412 if (allocator->mutex_counter < -11) /* moderately recover magazine sizes */ 413 { 414 allocator->mutex_counter = 0; 415 *contention_counter = MAX (*contention_counter, 1) - 1; 416 } 417 } 418 } 419 420 static inline ThreadMemory* 421 thread_memory_from_self (void) 422 { 423 ThreadMemory *tmem = g_private_get (private_thread_memory); 424 if (G_UNLIKELY (!tmem)) 425 { 426 static ThreadMemory *single_thread_memory = NULL; /* remember single-thread info for multi-threaded case */ 427 if (single_thread_memory && g_thread_supported ()) 428 { 429 g_mutex_lock (allocator->slab_mutex); 430 if (single_thread_memory) 431 { 432 /* GSlice has been used before g_thread_init(), and now 433 * we are running threaded. to cope with it, use the saved 434 * thread memory structure from when we weren't threaded. 435 */ 436 tmem = single_thread_memory; 437 single_thread_memory = NULL; /* slab_mutex protected when multi-threaded */ 438 } 439 g_mutex_unlock (allocator->slab_mutex); 440 } 441 if (!tmem) 442 { 443 const guint n_magazines = MAX_SLAB_INDEX (allocator); 444 tmem = g_malloc0 (sizeof (ThreadMemory) + sizeof (Magazine) * 2 * n_magazines); 445 tmem->magazine1 = (Magazine*) (tmem + 1); 446 tmem->magazine2 = &tmem->magazine1[n_magazines]; 447 } 448 /* g_private_get/g_private_set works in the single-threaded xor the multi- 449 * threaded case. but not *across* g_thread_init(), after multi-thread 450 * initialization it returns NULL for previously set single-thread data. 451 */ 452 g_private_set (private_thread_memory, tmem); 453 /* save single-thread thread memory structure, in case we need to 454 * pick it up again after multi-thread initialization happened. 455 */ 456 if (!single_thread_memory && !g_thread_supported ()) 457 single_thread_memory = tmem; /* no slab_mutex created yet */ 458 } 459 return tmem; 460 } 461 462 static inline ChunkLink* 463 magazine_chain_pop_head (ChunkLink **magazine_chunks) 464 { 465 /* magazine chains are linked via ChunkLink->next. 466 * each ChunkLink->data of the toplevel chain may point to a subchain, 467 * linked via ChunkLink->next. ChunkLink->data of the subchains just 468 * contains uninitialized junk. 469 */ 470 ChunkLink *chunk = (*magazine_chunks)->data; 471 if (G_UNLIKELY (chunk)) 472 { 473 /* allocating from freed list */ 474 (*magazine_chunks)->data = chunk->next; 475 } 476 else 477 { 478 chunk = *magazine_chunks; 479 *magazine_chunks = chunk->next; 480 } 481 return chunk; 482 } 483 484 #if 0 /* useful for debugging */ 485 static guint 486 magazine_count (ChunkLink *head) 487 { 488 guint count = 0; 489 if (!head) 490 return 0; 491 while (head) 492 { 493 ChunkLink *child = head->data; 494 count += 1; 495 for (child = head->data; child; child = child->next) 496 count += 1; 497 head = head->next; 498 } 499 return count; 500 } 501 #endif 502 503 static inline gsize 504 allocator_get_magazine_threshold (Allocator *allocator, 505 guint ix) 506 { 507 /* the magazine size calculated here has a lower bound of MIN_MAGAZINE_SIZE, 508 * which is required by the implementation. also, for moderately sized chunks 509 * (say >= 64 bytes), magazine sizes shouldn't be much smaller then the number 510 * of chunks available per page/2 to avoid excessive traffic in the magazine 511 * cache for small to medium sized structures. 512 * the upper bound of the magazine size is effectively provided by 513 * MAX_MAGAZINE_SIZE. for larger chunks, this number is scaled down so that 514 * the content of a single magazine doesn't exceed ca. 16KB. 515 */ 516 gsize chunk_size = SLAB_CHUNK_SIZE (allocator, ix); 517 guint threshold = MAX (MIN_MAGAZINE_SIZE, allocator->max_page_size / MAX (5 * chunk_size, 5 * 32)); 518 guint contention_counter = allocator->contention_counters[ix]; 519 if (G_UNLIKELY (contention_counter)) /* single CPU bias */ 520 { 521 /* adapt contention counter thresholds to chunk sizes */ 522 contention_counter = contention_counter * 64 / chunk_size; 523 threshold = MAX (threshold, contention_counter); 524 } 525 return threshold; 526 } 527 528 /* --- magazine cache --- */ 529 static inline void 530 magazine_cache_update_stamp (void) 531 { 532 if (allocator->stamp_counter >= MAX_STAMP_COUNTER) 533 { 534 GTimeVal tv; 535 g_get_current_time (&tv); 536 allocator->last_stamp = tv.tv_sec * 1000 + tv.tv_usec / 1000; /* milli seconds */ 537 allocator->stamp_counter = 0; 538 } 539 else 540 allocator->stamp_counter++; 541 } 542 543 static inline ChunkLink* 544 magazine_chain_prepare_fields (ChunkLink *magazine_chunks) 545 { 546 ChunkLink *chunk1; 547 ChunkLink *chunk2; 548 ChunkLink *chunk3; 549 ChunkLink *chunk4; 550 /* checked upon initialization: mem_assert (MIN_MAGAZINE_SIZE >= 4); */ 551 /* ensure a magazine with at least 4 unused data pointers */ 552 chunk1 = magazine_chain_pop_head (&magazine_chunks); 553 chunk2 = magazine_chain_pop_head (&magazine_chunks); 554 chunk3 = magazine_chain_pop_head (&magazine_chunks); 555 chunk4 = magazine_chain_pop_head (&magazine_chunks); 556 chunk4->next = magazine_chunks; 557 chunk3->next = chunk4; 558 chunk2->next = chunk3; 559 chunk1->next = chunk2; 560 return chunk1; 561 } 562 563 /* access the first 3 fields of a specially prepared magazine chain */ 564 #define magazine_chain_prev(mc) ((mc)->data) 565 #define magazine_chain_stamp(mc) ((mc)->next->data) 566 #define magazine_chain_uint_stamp(mc) GPOINTER_TO_UINT ((mc)->next->data) 567 #define magazine_chain_next(mc) ((mc)->next->next->data) 568 #define magazine_chain_count(mc) ((mc)->next->next->next->data) 569 570 static void 571 magazine_cache_trim (Allocator *allocator, 572 guint ix, 573 guint stamp) 574 { 575 /* g_mutex_lock (allocator->mutex); done by caller */ 576 /* trim magazine cache from tail */ 577 ChunkLink *current = magazine_chain_prev (allocator->magazines[ix]); 578 ChunkLink *trash = NULL; 579 while (ABS (stamp - magazine_chain_uint_stamp (current)) >= allocator->config.working_set_msecs) 580 { 581 /* unlink */ 582 ChunkLink *prev = magazine_chain_prev (current); 583 ChunkLink *next = magazine_chain_next (current); 584 magazine_chain_next (prev) = next; 585 magazine_chain_prev (next) = prev; 586 /* clear special fields, put on trash stack */ 587 magazine_chain_next (current) = NULL; 588 magazine_chain_count (current) = NULL; 589 magazine_chain_stamp (current) = NULL; 590 magazine_chain_prev (current) = trash; 591 trash = current; 592 /* fixup list head if required */ 593 if (current == allocator->magazines[ix]) 594 { 595 allocator->magazines[ix] = NULL; 596 break; 597 } 598 current = prev; 599 } 600 g_mutex_unlock (allocator->magazine_mutex); 601 /* free trash */ 602 if (trash) 603 { 604 const gsize chunk_size = SLAB_CHUNK_SIZE (allocator, ix); 605 g_mutex_lock (allocator->slab_mutex); 606 while (trash) 607 { 608 current = trash; 609 trash = magazine_chain_prev (current); 610 magazine_chain_prev (current) = NULL; /* clear special field */ 611 while (current) 612 { 613 ChunkLink *chunk = magazine_chain_pop_head (¤t); 614 slab_allocator_free_chunk (chunk_size, chunk); 615 } 616 } 617 g_mutex_unlock (allocator->slab_mutex); 618 } 619 } 620 621 static void 622 magazine_cache_push_magazine (guint ix, 623 ChunkLink *magazine_chunks, 624 gsize count) /* must be >= MIN_MAGAZINE_SIZE */ 625 { 626 ChunkLink *current = magazine_chain_prepare_fields (magazine_chunks); 627 ChunkLink *next, *prev; 628 g_mutex_lock (allocator->magazine_mutex); 629 /* add magazine at head */ 630 next = allocator->magazines[ix]; 631 if (next) 632 prev = magazine_chain_prev (next); 633 else 634 next = prev = current; 635 magazine_chain_next (prev) = current; 636 magazine_chain_prev (next) = current; 637 magazine_chain_prev (current) = prev; 638 magazine_chain_next (current) = next; 639 magazine_chain_count (current) = (gpointer) count; 640 /* stamp magazine */ 641 magazine_cache_update_stamp(); 642 magazine_chain_stamp (current) = GUINT_TO_POINTER (allocator->last_stamp); 643 allocator->magazines[ix] = current; 644 /* free old magazines beyond a certain threshold */ 645 magazine_cache_trim (allocator, ix, allocator->last_stamp); 646 /* g_mutex_unlock (allocator->mutex); was done by magazine_cache_trim() */ 647 } 648 649 static ChunkLink* 650 magazine_cache_pop_magazine (guint ix, 651 gsize *countp) 652 { 653 g_mutex_lock_a (allocator->magazine_mutex, &allocator->contention_counters[ix]); 654 if (!allocator->magazines[ix]) 655 { 656 guint magazine_threshold = allocator_get_magazine_threshold (allocator, ix); 657 gsize i, chunk_size = SLAB_CHUNK_SIZE (allocator, ix); 658 ChunkLink *chunk, *head; 659 g_mutex_unlock (allocator->magazine_mutex); 660 g_mutex_lock (allocator->slab_mutex); 661 head = slab_allocator_alloc_chunk (chunk_size); 662 head->data = NULL; 663 chunk = head; 664 for (i = 1; i < magazine_threshold; i++) 665 { 666 chunk->next = slab_allocator_alloc_chunk (chunk_size); 667 chunk = chunk->next; 668 chunk->data = NULL; 669 } 670 chunk->next = NULL; 671 g_mutex_unlock (allocator->slab_mutex); 672 *countp = i; 673 return head; 674 } 675 else 676 { 677 ChunkLink *current = allocator->magazines[ix]; 678 ChunkLink *prev = magazine_chain_prev (current); 679 ChunkLink *next = magazine_chain_next (current); 680 /* unlink */ 681 magazine_chain_next (prev) = next; 682 magazine_chain_prev (next) = prev; 683 allocator->magazines[ix] = next == current ? NULL : next; 684 g_mutex_unlock (allocator->magazine_mutex); 685 /* clear special fields and hand out */ 686 *countp = (gsize) magazine_chain_count (current); 687 magazine_chain_prev (current) = NULL; 688 magazine_chain_next (current) = NULL; 689 magazine_chain_count (current) = NULL; 690 magazine_chain_stamp (current) = NULL; 691 return current; 692 } 693 } 694 695 /* --- thread magazines --- */ 696 static void 697 private_thread_memory_cleanup (gpointer data) 698 { 699 ThreadMemory *tmem = data; 700 const guint n_magazines = MAX_SLAB_INDEX (allocator); 701 guint ix; 702 for (ix = 0; ix < n_magazines; ix++) 703 { 704 Magazine *mags[2]; 705 guint j; 706 mags[0] = &tmem->magazine1[ix]; 707 mags[1] = &tmem->magazine2[ix]; 708 for (j = 0; j < 2; j++) 709 { 710 Magazine *mag = mags[j]; 711 if (mag->count >= MIN_MAGAZINE_SIZE) 712 magazine_cache_push_magazine (ix, mag->chunks, mag->count); 713 else 714 { 715 const gsize chunk_size = SLAB_CHUNK_SIZE (allocator, ix); 716 g_mutex_lock (allocator->slab_mutex); 717 while (mag->chunks) 718 { 719 ChunkLink *chunk = magazine_chain_pop_head (&mag->chunks); 720 slab_allocator_free_chunk (chunk_size, chunk); 721 } 722 g_mutex_unlock (allocator->slab_mutex); 723 } 724 } 725 } 726 g_free (tmem); 727 } 728 729 static void 730 thread_memory_magazine1_reload (ThreadMemory *tmem, 731 guint ix) 732 { 733 Magazine *mag = &tmem->magazine1[ix]; 734 mem_assert (mag->chunks == NULL); /* ensure that we may reset mag->count */ 735 mag->count = 0; 736 mag->chunks = magazine_cache_pop_magazine (ix, &mag->count); 737 } 738 739 static void 740 thread_memory_magazine2_unload (ThreadMemory *tmem, 741 guint ix) 742 { 743 Magazine *mag = &tmem->magazine2[ix]; 744 magazine_cache_push_magazine (ix, mag->chunks, mag->count); 745 mag->chunks = NULL; 746 mag->count = 0; 747 } 748 749 static inline void 750 thread_memory_swap_magazines (ThreadMemory *tmem, 751 guint ix) 752 { 753 Magazine xmag = tmem->magazine1[ix]; 754 tmem->magazine1[ix] = tmem->magazine2[ix]; 755 tmem->magazine2[ix] = xmag; 756 } 757 758 static inline gboolean 759 thread_memory_magazine1_is_empty (ThreadMemory *tmem, 760 guint ix) 761 { 762 return tmem->magazine1[ix].chunks == NULL; 763 } 764 765 static inline gboolean 766 thread_memory_magazine2_is_full (ThreadMemory *tmem, 767 guint ix) 768 { 769 return tmem->magazine2[ix].count >= allocator_get_magazine_threshold (allocator, ix); 770 } 771 772 static inline gpointer 773 thread_memory_magazine1_alloc (ThreadMemory *tmem, 774 guint ix) 775 { 776 Magazine *mag = &tmem->magazine1[ix]; 777 ChunkLink *chunk = magazine_chain_pop_head (&mag->chunks); 778 if (G_LIKELY (mag->count > 0)) 779 mag->count--; 780 return chunk; 781 } 782 783 static inline void 784 thread_memory_magazine2_free (ThreadMemory *tmem, 785 guint ix, 786 gpointer mem) 787 { 788 Magazine *mag = &tmem->magazine2[ix]; 789 ChunkLink *chunk = mem; 790 chunk->data = NULL; 791 chunk->next = mag->chunks; 792 mag->chunks = chunk; 793 mag->count++; 794 } 795 796 /* --- API functions --- */ 797 gpointer 798 g_slice_alloc (gsize mem_size) 799 { 800 gsize chunk_size; 801 gpointer mem; 802 guint acat; 803 chunk_size = P2ALIGN (mem_size); 804 acat = allocator_categorize (chunk_size); 805 if (G_LIKELY (acat == 1)) /* allocate through magazine layer */ 806 { 807 ThreadMemory *tmem = thread_memory_from_self(); 808 guint ix = SLAB_INDEX (allocator, chunk_size); 809 if (G_UNLIKELY (thread_memory_magazine1_is_empty (tmem, ix))) 810 { 811 thread_memory_swap_magazines (tmem, ix); 812 if (G_UNLIKELY (thread_memory_magazine1_is_empty (tmem, ix))) 813 thread_memory_magazine1_reload (tmem, ix); 814 } 815 mem = thread_memory_magazine1_alloc (tmem, ix); 816 } 817 else if (acat == 2) /* allocate through slab allocator */ 818 { 819 g_mutex_lock (allocator->slab_mutex); 820 mem = slab_allocator_alloc_chunk (chunk_size); 821 g_mutex_unlock (allocator->slab_mutex); 822 } 823 else /* delegate to system malloc */ 824 mem = g_malloc (mem_size); 825 if (G_UNLIKELY (allocator->config.debug_blocks)) 826 smc_notify_alloc (mem, mem_size); 827 return mem; 828 } 829 830 gpointer 831 g_slice_alloc0 (gsize mem_size) 832 { 833 gpointer mem = g_slice_alloc (mem_size); 834 if (mem) 835 memset (mem, 0, mem_size); 836 return mem; 837 } 838 839 gpointer 840 g_slice_copy (gsize mem_size, 841 gconstpointer mem_block) 842 { 843 gpointer mem = g_slice_alloc (mem_size); 844 if (mem) 845 memcpy (mem, mem_block, mem_size); 846 return mem; 847 } 848 849 void 850 g_slice_free1 (gsize mem_size, 851 gpointer mem_block) 852 { 853 gsize chunk_size = P2ALIGN (mem_size); 854 guint acat = allocator_categorize (chunk_size); 855 if (G_UNLIKELY (!mem_block)) 856 return; 857 if (G_UNLIKELY (allocator->config.debug_blocks) && 858 !smc_notify_free (mem_block, mem_size)) 859 abort(); 860 if (G_LIKELY (acat == 1)) /* allocate through magazine layer */ 861 { 862 ThreadMemory *tmem = thread_memory_from_self(); 863 guint ix = SLAB_INDEX (allocator, chunk_size); 864 if (G_UNLIKELY (thread_memory_magazine2_is_full (tmem, ix))) 865 { 866 thread_memory_swap_magazines (tmem, ix); 867 if (G_UNLIKELY (thread_memory_magazine2_is_full (tmem, ix))) 868 thread_memory_magazine2_unload (tmem, ix); 869 } 870 if (G_UNLIKELY (g_mem_gc_friendly)) 871 memset (mem_block, 0, chunk_size); 872 thread_memory_magazine2_free (tmem, ix, mem_block); 873 } 874 else if (acat == 2) /* allocate through slab allocator */ 875 { 876 if (G_UNLIKELY (g_mem_gc_friendly)) 877 memset (mem_block, 0, chunk_size); 878 g_mutex_lock (allocator->slab_mutex); 879 slab_allocator_free_chunk (chunk_size, mem_block); 880 g_mutex_unlock (allocator->slab_mutex); 881 } 882 else /* delegate to system malloc */ 883 { 884 if (G_UNLIKELY (g_mem_gc_friendly)) 885 memset (mem_block, 0, mem_size); 886 g_free (mem_block); 887 } 888 } 889 890 void 891 g_slice_free_chain_with_offset (gsize mem_size, 892 gpointer mem_chain, 893 gsize next_offset) 894 { 895 gpointer slice = mem_chain; 896 /* while the thread magazines and the magazine cache are implemented so that 897 * they can easily be extended to allow for free lists containing more free 898 * lists for the first level nodes, which would allow O(1) freeing in this 899 * function, the benefit of such an extension is questionable, because: 900 * - the magazine size counts will become mere lower bounds which confuses 901 * the code adapting to lock contention; 902 * - freeing a single node to the thread magazines is very fast, so this 903 * O(list_length) operation is multiplied by a fairly small factor; 904 * - memory usage histograms on larger applications seem to indicate that 905 * the amount of released multi node lists is negligible in comparison 906 * to single node releases. 907 * - the major performance bottle neck, namely g_private_get() or 908 * g_mutex_lock()/g_mutex_unlock() has already been moved out of the 909 * inner loop for freeing chained slices. 910 */ 911 gsize chunk_size = P2ALIGN (mem_size); 912 guint acat = allocator_categorize (chunk_size); 913 if (G_LIKELY (acat == 1)) /* allocate through magazine layer */ 914 { 915 ThreadMemory *tmem = thread_memory_from_self(); 916 guint ix = SLAB_INDEX (allocator, chunk_size); 917 while (slice) 918 { 919 guint8 *current = slice; 920 slice = *(gpointer*) (current + next_offset); 921 if (G_UNLIKELY (allocator->config.debug_blocks) && 922 !smc_notify_free (current, mem_size)) 923 abort(); 924 if (G_UNLIKELY (thread_memory_magazine2_is_full (tmem, ix))) 925 { 926 thread_memory_swap_magazines (tmem, ix); 927 if (G_UNLIKELY (thread_memory_magazine2_is_full (tmem, ix))) 928 thread_memory_magazine2_unload (tmem, ix); 929 } 930 if (G_UNLIKELY (g_mem_gc_friendly)) 931 memset (current, 0, chunk_size); 932 thread_memory_magazine2_free (tmem, ix, current); 933 } 934 } 935 else if (acat == 2) /* allocate through slab allocator */ 936 { 937 g_mutex_lock (allocator->slab_mutex); 938 while (slice) 939 { 940 guint8 *current = slice; 941 slice = *(gpointer*) (current + next_offset); 942 if (G_UNLIKELY (allocator->config.debug_blocks) && 943 !smc_notify_free (current, mem_size)) 944 abort(); 945 if (G_UNLIKELY (g_mem_gc_friendly)) 946 memset (current, 0, chunk_size); 947 slab_allocator_free_chunk (chunk_size, current); 948 } 949 g_mutex_unlock (allocator->slab_mutex); 950 } 951 else /* delegate to system malloc */ 952 while (slice) 953 { 954 guint8 *current = slice; 955 slice = *(gpointer*) (current + next_offset); 956 if (G_UNLIKELY (allocator->config.debug_blocks) && 957 !smc_notify_free (current, mem_size)) 958 abort(); 959 if (G_UNLIKELY (g_mem_gc_friendly)) 960 memset (current, 0, mem_size); 961 g_free (current); 962 } 963 } 964 965 /* --- single page allocator --- */ 966 static void 967 allocator_slab_stack_push (Allocator *allocator, 968 guint ix, 969 SlabInfo *sinfo) 970 { 971 /* insert slab at slab ring head */ 972 if (!allocator->slab_stack[ix]) 973 { 974 sinfo->next = sinfo; 975 sinfo->prev = sinfo; 976 } 977 else 978 { 979 SlabInfo *next = allocator->slab_stack[ix], *prev = next->prev; 980 next->prev = sinfo; 981 prev->next = sinfo; 982 sinfo->next = next; 983 sinfo->prev = prev; 984 } 985 allocator->slab_stack[ix] = sinfo; 986 } 987 988 static gsize 989 allocator_aligned_page_size (Allocator *allocator, 990 gsize n_bytes) 991 { 992 gsize val = 1 << g_bit_storage (n_bytes - 1); 993 val = MAX (val, allocator->min_page_size); 994 return val; 995 } 996 997 static void 998 allocator_add_slab (Allocator *allocator, 999 guint ix, 1000 gsize chunk_size) 1001 { 1002 ChunkLink *chunk; 1003 SlabInfo *sinfo; 1004 gsize addr, padding, n_chunks, color = 0; 1005 gsize page_size = allocator_aligned_page_size (allocator, SLAB_BPAGE_SIZE (allocator, chunk_size)); 1006 /* allocate 1 page for the chunks and the slab */ 1007 gpointer aligned_memory = allocator_memalign (page_size, page_size - NATIVE_MALLOC_PADDING); 1008 guint8 *mem = aligned_memory; 1009 guint i; 1010 if (!mem) 1011 { 1012 const gchar *syserr = "unknown error"; 1013 #if HAVE_STRERROR 1014 syserr = strerror (errno); 1015 #endif 1016 mem_error ("failed to allocate %u bytes (alignment: %u): %s\n", 1017 (guint) (page_size - NATIVE_MALLOC_PADDING), (guint) page_size, syserr); 1018 } 1019 /* mask page adress */ 1020 addr = ((gsize) mem / page_size) * page_size; 1021 /* assert alignment */ 1022 mem_assert (aligned_memory == (gpointer) addr); 1023 /* basic slab info setup */ 1024 sinfo = (SlabInfo*) (mem + page_size - SLAB_INFO_SIZE); 1025 sinfo->n_allocated = 0; 1026 sinfo->chunks = NULL; 1027 /* figure cache colorization */ 1028 n_chunks = ((guint8*) sinfo - mem) / chunk_size; 1029 padding = ((guint8*) sinfo - mem) - n_chunks * chunk_size; 1030 if (padding) 1031 { 1032 color = (allocator->color_accu * P2ALIGNMENT) % padding; 1033 allocator->color_accu += allocator->config.color_increment; 1034 } 1035 /* add chunks to free list */ 1036 chunk = (ChunkLink*) (mem + color); 1037 sinfo->chunks = chunk; 1038 for (i = 0; i < n_chunks - 1; i++) 1039 { 1040 chunk->next = (ChunkLink*) ((guint8*) chunk + chunk_size); 1041 chunk = chunk->next; 1042 } 1043 chunk->next = NULL; /* last chunk */ 1044 /* add slab to slab ring */ 1045 allocator_slab_stack_push (allocator, ix, sinfo); 1046 } 1047 1048 static gpointer 1049 slab_allocator_alloc_chunk (gsize chunk_size) 1050 { 1051 ChunkLink *chunk; 1052 guint ix = SLAB_INDEX (allocator, chunk_size); 1053 /* ensure non-empty slab */ 1054 if (!allocator->slab_stack[ix] || !allocator->slab_stack[ix]->chunks) 1055 allocator_add_slab (allocator, ix, chunk_size); 1056 /* allocate chunk */ 1057 chunk = allocator->slab_stack[ix]->chunks; 1058 allocator->slab_stack[ix]->chunks = chunk->next; 1059 allocator->slab_stack[ix]->n_allocated++; 1060 /* rotate empty slabs */ 1061 if (!allocator->slab_stack[ix]->chunks) 1062 allocator->slab_stack[ix] = allocator->slab_stack[ix]->next; 1063 return chunk; 1064 } 1065 1066 static void 1067 slab_allocator_free_chunk (gsize chunk_size, 1068 gpointer mem) 1069 { 1070 ChunkLink *chunk; 1071 gboolean was_empty; 1072 guint ix = SLAB_INDEX (allocator, chunk_size); 1073 gsize page_size = allocator_aligned_page_size (allocator, SLAB_BPAGE_SIZE (allocator, chunk_size)); 1074 gsize addr = ((gsize) mem / page_size) * page_size; 1075 /* mask page adress */ 1076 guint8 *page = (guint8*) addr; 1077 SlabInfo *sinfo = (SlabInfo*) (page + page_size - SLAB_INFO_SIZE); 1078 /* assert valid chunk count */ 1079 mem_assert (sinfo->n_allocated > 0); 1080 /* add chunk to free list */ 1081 was_empty = sinfo->chunks == NULL; 1082 chunk = (ChunkLink*) mem; 1083 chunk->next = sinfo->chunks; 1084 sinfo->chunks = chunk; 1085 sinfo->n_allocated--; 1086 /* keep slab ring partially sorted, empty slabs at end */ 1087 if (was_empty) 1088 { 1089 /* unlink slab */ 1090 SlabInfo *next = sinfo->next, *prev = sinfo->prev; 1091 next->prev = prev; 1092 prev->next = next; 1093 if (allocator->slab_stack[ix] == sinfo) 1094 allocator->slab_stack[ix] = next == sinfo ? NULL : next; 1095 /* insert slab at head */ 1096 allocator_slab_stack_push (allocator, ix, sinfo); 1097 } 1098 /* eagerly free complete unused slabs */ 1099 if (!sinfo->n_allocated) 1100 { 1101 /* unlink slab */ 1102 SlabInfo *next = sinfo->next, *prev = sinfo->prev; 1103 next->prev = prev; 1104 prev->next = next; 1105 if (allocator->slab_stack[ix] == sinfo) 1106 allocator->slab_stack[ix] = next == sinfo ? NULL : next; 1107 /* free slab */ 1108 allocator_memfree (page_size, page); 1109 } 1110 } 1111 1112 /* --- memalign implementation --- */ 1113 #ifdef HAVE_MALLOC_H 1114 #include <malloc.h> /* memalign() */ 1115 #endif 1116 1117 /* from config.h: 1118 * define HAVE_POSIX_MEMALIGN 1 // if free(posix_memalign(3)) works, <stdlib.h> 1119 * define HAVE_COMPLIANT_POSIX_MEMALIGN 1 // if free(posix_memalign(3)) works for sizes != 2^n, <stdlib.h> 1120 * define HAVE_MEMALIGN 1 // if free(memalign(3)) works, <malloc.h> 1121 * define HAVE_VALLOC 1 // if free(valloc(3)) works, <stdlib.h> or <malloc.h> 1122 * if none is provided, we implement malloc(3)-based alloc-only page alignment 1123 */ 1124 1125 #if !(HAVE_COMPLIANT_POSIX_MEMALIGN || HAVE_MEMALIGN || HAVE_VALLOC) 1126 static GTrashStack *compat_valloc_trash = NULL; 1127 #endif 1128 1129 static gpointer 1130 allocator_memalign (gsize alignment, 1131 gsize memsize) 1132 { 1133 gpointer aligned_memory = NULL; 1134 gint err = ENOMEM; 1135 #if HAVE_COMPLIANT_POSIX_MEMALIGN 1136 err = posix_memalign (&aligned_memory, alignment, memsize); 1137 #elif HAVE_MEMALIGN 1138 errno = 0; 1139 aligned_memory = memalign (alignment, memsize); 1140 err = errno; 1141 #elif HAVE_VALLOC 1142 errno = 0; 1143 aligned_memory = valloc (memsize); 1144 err = errno; 1145 #else 1146 /* simplistic non-freeing page allocator */ 1147 mem_assert (alignment == sys_page_size); 1148 mem_assert (memsize <= sys_page_size); 1149 if (!compat_valloc_trash) 1150 { 1151 const guint n_pages = 16; 1152 guint8 *mem = malloc (n_pages * sys_page_size); 1153 err = errno; 1154 if (mem) 1155 { 1156 gint i = n_pages; 1157 guint8 *amem = (guint8*) ALIGN ((gsize) mem, sys_page_size); 1158 if (amem != mem) 1159 i--; /* mem wasn't page aligned */ 1160 while (--i >= 0) 1161 g_trash_stack_push (&compat_valloc_trash, amem + i * sys_page_size); 1162 } 1163 } 1164 aligned_memory = g_trash_stack_pop (&compat_valloc_trash); 1165 #endif 1166 if (!aligned_memory) 1167 errno = err; 1168 return aligned_memory; 1169 } 1170 1171 static void 1172 allocator_memfree (gsize memsize, 1173 gpointer mem) 1174 { 1175 #if HAVE_COMPLIANT_POSIX_MEMALIGN || HAVE_MEMALIGN || HAVE_VALLOC 1176 free (mem); 1177 #else 1178 mem_assert (memsize <= sys_page_size); 1179 g_trash_stack_push (&compat_valloc_trash, mem); 1180 #endif 1181 } 1182 1183 static void 1184 mem_error (const char *format, 1185 ...) 1186 { 1187 const char *pname; 1188 va_list args; 1189 /* at least, put out "MEMORY-ERROR", in case we segfault during the rest of the function */ 1190 fputs ("\n***MEMORY-ERROR***: ", stderr); 1191 pname = g_get_prgname(); 1192 fprintf (stderr, "%s[%ld]: GSlice: ", pname ? pname : "", (long)getpid()); 1193 va_start (args, format); 1194 vfprintf (stderr, format, args); 1195 va_end (args); 1196 fputs ("\n", stderr); 1197 abort(); 1198 _exit (1); 1199 } 1200 1201 /* --- g-slice memory checker tree --- */ 1202 typedef size_t SmcKType; /* key type */ 1203 typedef size_t SmcVType; /* value type */ 1204 typedef struct { 1205 SmcKType key; 1206 SmcVType value; 1207 } SmcEntry; 1208 static void smc_tree_insert (SmcKType key, 1209 SmcVType value); 1210 static gboolean smc_tree_lookup (SmcKType key, 1211 SmcVType *value_p); 1212 static gboolean smc_tree_remove (SmcKType key); 1213 1214 1215 /* --- g-slice memory checker implementation --- */ 1216 static void 1217 smc_notify_alloc (void *pointer, 1218 size_t size) 1219 { 1220 size_t adress = (size_t) pointer; 1221 if (pointer) 1222 smc_tree_insert (adress, size); 1223 } 1224 1225 #if 0 1226 static void 1227 smc_notify_ignore (void *pointer) 1228 { 1229 size_t adress = (size_t) pointer; 1230 if (pointer) 1231 smc_tree_remove (adress); 1232 } 1233 #endif 1234 1235 static int 1236 smc_notify_free (void *pointer, 1237 size_t size) 1238 { 1239 size_t adress = (size_t) pointer; 1240 SmcVType real_size; 1241 gboolean found_one; 1242 1243 if (!pointer) 1244 return 1; /* ignore */ 1245 found_one = smc_tree_lookup (adress, &real_size); 1246 if (!found_one) 1247 { 1248 fprintf (stderr, "GSlice: MemChecker: attempt to release non-allocated block: %p size=%" G_GSIZE_FORMAT "\n", pointer, size); 1249 return 0; 1250 } 1251 if (real_size != size && (real_size || size)) 1252 { 1253 fprintf (stderr, "GSlice: MemChecker: attempt to release block with invalid size: %p size=%" G_GSIZE_FORMAT " invalid-size=%" G_GSIZE_FORMAT "\n", pointer, real_size, size); 1254 return 0; 1255 } 1256 if (!smc_tree_remove (adress)) 1257 { 1258 fprintf (stderr, "GSlice: MemChecker: attempt to release non-allocated block: %p size=%" G_GSIZE_FORMAT "\n", pointer, size); 1259 return 0; 1260 } 1261 return 1; /* all fine */ 1262 } 1263 1264 /* --- g-slice memory checker tree implementation --- */ 1265 #define SMC_TRUNK_COUNT (4093 /* 16381 */) /* prime, to distribute trunk collisions (big, allocated just once) */ 1266 #define SMC_BRANCH_COUNT (511) /* prime, to distribute branch collisions */ 1267 #define SMC_TRUNK_EXTENT (SMC_BRANCH_COUNT * 2039) /* key adress space per trunk, should distribute uniformly across BRANCH_COUNT */ 1268 #define SMC_TRUNK_HASH(k) ((k / SMC_TRUNK_EXTENT) % SMC_TRUNK_COUNT) /* generate new trunk hash per megabyte (roughly) */ 1269 #define SMC_BRANCH_HASH(k) (k % SMC_BRANCH_COUNT) 1270 1271 typedef struct { 1272 SmcEntry *entries; 1273 unsigned int n_entries; 1274 } SmcBranch; 1275 1276 static SmcBranch **smc_tree_root = NULL; 1277 1278 static void 1279 smc_tree_abort (int errval) 1280 { 1281 const char *syserr = "unknown error"; 1282 #if HAVE_STRERROR 1283 syserr = strerror (errval); 1284 #endif 1285 mem_error ("MemChecker: failure in debugging tree: %s", syserr); 1286 } 1287 1288 static inline SmcEntry* 1289 smc_tree_branch_grow_L (SmcBranch *branch, 1290 unsigned int index) 1291 { 1292 unsigned int old_size = branch->n_entries * sizeof (branch->entries[0]); 1293 unsigned int new_size = old_size + sizeof (branch->entries[0]); 1294 SmcEntry *entry; 1295 mem_assert (index <= branch->n_entries); 1296 branch->entries = (SmcEntry*) realloc (branch->entries, new_size); 1297 if (!branch->entries) 1298 smc_tree_abort (errno); 1299 entry = branch->entries + index; 1300 g_memmove (entry + 1, entry, (branch->n_entries - index) * sizeof (entry[0])); 1301 branch->n_entries += 1; 1302 return entry; 1303 } 1304 1305 static inline SmcEntry* 1306 smc_tree_branch_lookup_nearest_L (SmcBranch *branch, 1307 SmcKType key) 1308 { 1309 unsigned int n_nodes = branch->n_entries, offs = 0; 1310 SmcEntry *check = branch->entries; 1311 int cmp = 0; 1312 while (offs < n_nodes) 1313 { 1314 unsigned int i = (offs + n_nodes) >> 1; 1315 check = branch->entries + i; 1316 cmp = key < check->key ? -1 : key != check->key; 1317 if (cmp == 0) 1318 return check; /* return exact match */ 1319 else if (cmp < 0) 1320 n_nodes = i; 1321 else /* (cmp > 0) */ 1322 offs = i + 1; 1323 } 1324 /* check points at last mismatch, cmp > 0 indicates greater key */ 1325 return cmp > 0 ? check + 1 : check; /* return insertion position for inexact match */ 1326 } 1327 1328 static void 1329 smc_tree_insert (SmcKType key, 1330 SmcVType value) 1331 { 1332 unsigned int ix0, ix1; 1333 SmcEntry *entry; 1334 1335 g_mutex_lock (smc_tree_mutex); 1336 ix0 = SMC_TRUNK_HASH (key); 1337 ix1 = SMC_BRANCH_HASH (key); 1338 if (!smc_tree_root) 1339 { 1340 smc_tree_root = calloc (SMC_TRUNK_COUNT, sizeof (smc_tree_root[0])); 1341 if (!smc_tree_root) 1342 smc_tree_abort (errno); 1343 } 1344 if (!smc_tree_root[ix0]) 1345 { 1346 smc_tree_root[ix0] = calloc (SMC_BRANCH_COUNT, sizeof (smc_tree_root[0][0])); 1347 if (!smc_tree_root[ix0]) 1348 smc_tree_abort (errno); 1349 } 1350 entry = smc_tree_branch_lookup_nearest_L (&smc_tree_root[ix0][ix1], key); 1351 if (!entry || /* need create */ 1352 entry >= smc_tree_root[ix0][ix1].entries + smc_tree_root[ix0][ix1].n_entries || /* need append */ 1353 entry->key != key) /* need insert */ 1354 entry = smc_tree_branch_grow_L (&smc_tree_root[ix0][ix1], entry - smc_tree_root[ix0][ix1].entries); 1355 entry->key = key; 1356 entry->value = value; 1357 g_mutex_unlock (smc_tree_mutex); 1358 } 1359 1360 static gboolean 1361 smc_tree_lookup (SmcKType key, 1362 SmcVType *value_p) 1363 { 1364 SmcEntry *entry = NULL; 1365 unsigned int ix0 = SMC_TRUNK_HASH (key), ix1 = SMC_BRANCH_HASH (key); 1366 gboolean found_one = FALSE; 1367 *value_p = 0; 1368 g_mutex_lock (smc_tree_mutex); 1369 if (smc_tree_root && smc_tree_root[ix0]) 1370 { 1371 entry = smc_tree_branch_lookup_nearest_L (&smc_tree_root[ix0][ix1], key); 1372 if (entry && 1373 entry < smc_tree_root[ix0][ix1].entries + smc_tree_root[ix0][ix1].n_entries && 1374 entry->key == key) 1375 { 1376 found_one = TRUE; 1377 *value_p = entry->value; 1378 } 1379 } 1380 g_mutex_unlock (smc_tree_mutex); 1381 return found_one; 1382 } 1383 1384 static gboolean 1385 smc_tree_remove (SmcKType key) 1386 { 1387 unsigned int ix0 = SMC_TRUNK_HASH (key), ix1 = SMC_BRANCH_HASH (key); 1388 gboolean found_one = FALSE; 1389 g_mutex_lock (smc_tree_mutex); 1390 if (smc_tree_root && smc_tree_root[ix0]) 1391 { 1392 SmcEntry *entry = smc_tree_branch_lookup_nearest_L (&smc_tree_root[ix0][ix1], key); 1393 if (entry && 1394 entry < smc_tree_root[ix0][ix1].entries + smc_tree_root[ix0][ix1].n_entries && 1395 entry->key == key) 1396 { 1397 unsigned int i = entry - smc_tree_root[ix0][ix1].entries; 1398 smc_tree_root[ix0][ix1].n_entries -= 1; 1399 g_memmove (entry, entry + 1, (smc_tree_root[ix0][ix1].n_entries - i) * sizeof (entry[0])); 1400 if (!smc_tree_root[ix0][ix1].n_entries) 1401 { 1402 /* avoid useless pressure on the memory system */ 1403 free (smc_tree_root[ix0][ix1].entries); 1404 smc_tree_root[ix0][ix1].entries = NULL; 1405 } 1406 found_one = TRUE; 1407 } 1408 } 1409 g_mutex_unlock (smc_tree_mutex); 1410 return found_one; 1411 } 1412 1413 #ifdef G_ENABLE_DEBUG 1414 void 1415 g_slice_debug_tree_statistics (void) 1416 { 1417 g_mutex_lock (smc_tree_mutex); 1418 if (smc_tree_root) 1419 { 1420 unsigned int i, j, t = 0, o = 0, b = 0, su = 0, ex = 0, en = 4294967295u; 1421 double tf, bf; 1422 for (i = 0; i < SMC_TRUNK_COUNT; i++) 1423 if (smc_tree_root[i]) 1424 { 1425 t++; 1426 for (j = 0; j < SMC_BRANCH_COUNT; j++) 1427 if (smc_tree_root[i][j].n_entries) 1428 { 1429 b++; 1430 su += smc_tree_root[i][j].n_entries; 1431 en = MIN (en, smc_tree_root[i][j].n_entries); 1432 ex = MAX (ex, smc_tree_root[i][j].n_entries); 1433 } 1434 else if (smc_tree_root[i][j].entries) 1435 o++; /* formerly used, now empty */ 1436 } 1437 en = b ? en : 0; 1438 tf = MAX (t, 1.0); /* max(1) to be a valid divisor */ 1439 bf = MAX (b, 1.0); /* max(1) to be a valid divisor */ 1440 fprintf (stderr, "GSlice: MemChecker: %u trunks, %u branches, %u old branches\n", t, b, o); 1441 fprintf (stderr, "GSlice: MemChecker: %f branches per trunk, %.2f%% utilization\n", 1442 b / tf, 1443 100.0 - (SMC_BRANCH_COUNT - b / tf) / (0.01 * SMC_BRANCH_COUNT)); 1444 fprintf (stderr, "GSlice: MemChecker: %f entries per branch, %u minimum, %u maximum\n", 1445 su / bf, en, ex); 1446 } 1447 else 1448 fprintf (stderr, "GSlice: MemChecker: root=NULL\n"); 1449 g_mutex_unlock (smc_tree_mutex); 1450 1451 /* sample statistics (beast + GSLice + 24h scripted core & GUI activity): 1452 * PID %CPU %MEM VSZ RSS COMMAND 1453 * 8887 30.3 45.8 456068 414856 beast-0.7.1 empty.bse 1454 * $ cat /proc/8887/statm # total-program-size resident-set-size shared-pages text/code data/stack library dirty-pages 1455 * 114017 103714 2354 344 0 108676 0 1456 * $ cat /proc/8887/status 1457 * Name: beast-0.7.1 1458 * VmSize: 456068 kB 1459 * VmLck: 0 kB 1460 * VmRSS: 414856 kB 1461 * VmData: 434620 kB 1462 * VmStk: 84 kB 1463 * VmExe: 1376 kB 1464 * VmLib: 13036 kB 1465 * VmPTE: 456 kB 1466 * Threads: 3 1467 * (gdb) print g_slice_debug_tree_statistics () 1468 * GSlice: MemChecker: 422 trunks, 213068 branches, 0 old branches 1469 * GSlice: MemChecker: 504.900474 branches per trunk, 98.81% utilization 1470 * GSlice: MemChecker: 4.965039 entries per branch, 1 minimum, 37 maximum 1471 */ 1472 } 1473 #endif /* G_ENABLE_DEBUG */ 1474 1475 #define __G_SLICE_C__ 1476 #include "galiasdef.c" 1477
