1 #define JEMALLOC_CTL_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /******************************************************************************/ 5 /* Data. */ 6 7 /* 8 * ctl_mtx protects the following: 9 * - ctl_stats.* 10 */ 11 static malloc_mutex_t ctl_mtx; 12 static bool ctl_initialized; 13 static uint64_t ctl_epoch; 14 static ctl_stats_t ctl_stats; 15 16 /******************************************************************************/ 17 /* Helpers for named and indexed nodes. */ 18 19 JEMALLOC_INLINE_C const ctl_named_node_t * 20 ctl_named_node(const ctl_node_t *node) 21 { 22 23 return ((node->named) ? (const ctl_named_node_t *)node : NULL); 24 } 25 26 JEMALLOC_INLINE_C const ctl_named_node_t * 27 ctl_named_children(const ctl_named_node_t *node, int index) 28 { 29 const ctl_named_node_t *children = ctl_named_node(node->children); 30 31 return (children ? &children[index] : NULL); 32 } 33 34 JEMALLOC_INLINE_C const ctl_indexed_node_t * 35 ctl_indexed_node(const ctl_node_t *node) 36 { 37 38 return (!node->named ? (const ctl_indexed_node_t *)node : NULL); 39 } 40 41 /******************************************************************************/ 42 /* Function prototypes for non-inline static functions. */ 43 44 #define CTL_PROTO(n) \ 45 static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ 46 size_t *oldlenp, void *newp, size_t newlen); 47 48 #define INDEX_PROTO(n) \ 49 static const ctl_named_node_t *n##_index(const size_t *mib, \ 50 size_t miblen, size_t i); 51 52 static bool ctl_arena_init(ctl_arena_stats_t *astats); 53 static void ctl_arena_clear(ctl_arena_stats_t *astats); 54 static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, 55 arena_t *arena); 56 static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, 57 ctl_arena_stats_t *astats); 58 static void ctl_arena_refresh(arena_t *arena, unsigned i); 59 static bool ctl_grow(void); 60 static void ctl_refresh(void); 61 static bool ctl_init(void); 62 static int ctl_lookup(const char *name, ctl_node_t const **nodesp, 63 size_t *mibp, size_t *depthp); 64 65 CTL_PROTO(version) 66 CTL_PROTO(epoch) 67 CTL_PROTO(thread_tcache_enabled) 68 CTL_PROTO(thread_tcache_flush) 69 CTL_PROTO(thread_prof_name) 70 CTL_PROTO(thread_prof_active) 71 CTL_PROTO(thread_arena) 72 CTL_PROTO(thread_allocated) 73 CTL_PROTO(thread_allocatedp) 74 CTL_PROTO(thread_deallocated) 75 CTL_PROTO(thread_deallocatedp) 76 CTL_PROTO(config_debug) 77 CTL_PROTO(config_fill) 78 CTL_PROTO(config_lazy_lock) 79 CTL_PROTO(config_munmap) 80 CTL_PROTO(config_prof) 81 CTL_PROTO(config_prof_libgcc) 82 CTL_PROTO(config_prof_libunwind) 83 CTL_PROTO(config_stats) 84 CTL_PROTO(config_tcache) 85 CTL_PROTO(config_tls) 86 CTL_PROTO(config_utrace) 87 CTL_PROTO(config_valgrind) 88 CTL_PROTO(config_xmalloc) 89 CTL_PROTO(opt_abort) 90 CTL_PROTO(opt_dss) 91 CTL_PROTO(opt_lg_chunk) 92 CTL_PROTO(opt_narenas) 93 CTL_PROTO(opt_lg_dirty_mult) 94 CTL_PROTO(opt_stats_print) 95 CTL_PROTO(opt_junk) 96 CTL_PROTO(opt_zero) 97 CTL_PROTO(opt_quarantine) 98 CTL_PROTO(opt_redzone) 99 CTL_PROTO(opt_utrace) 100 CTL_PROTO(opt_xmalloc) 101 CTL_PROTO(opt_tcache) 102 CTL_PROTO(opt_lg_tcache_max) 103 CTL_PROTO(opt_prof) 104 CTL_PROTO(opt_prof_prefix) 105 CTL_PROTO(opt_prof_active) 106 CTL_PROTO(opt_prof_thread_active_init) 107 CTL_PROTO(opt_lg_prof_sample) 108 CTL_PROTO(opt_lg_prof_interval) 109 CTL_PROTO(opt_prof_gdump) 110 CTL_PROTO(opt_prof_final) 111 CTL_PROTO(opt_prof_leak) 112 CTL_PROTO(opt_prof_accum) 113 CTL_PROTO(tcache_create) 114 CTL_PROTO(tcache_flush) 115 CTL_PROTO(tcache_destroy) 116 CTL_PROTO(arena_i_purge) 117 static void arena_purge(unsigned arena_ind); 118 CTL_PROTO(arena_i_dss) 119 CTL_PROTO(arena_i_lg_dirty_mult) 120 CTL_PROTO(arena_i_chunk_alloc) 121 CTL_PROTO(arena_i_chunk_dalloc) 122 CTL_PROTO(arena_i_chunk_purge) 123 INDEX_PROTO(arena_i) 124 CTL_PROTO(arenas_bin_i_size) 125 CTL_PROTO(arenas_bin_i_nregs) 126 CTL_PROTO(arenas_bin_i_run_size) 127 INDEX_PROTO(arenas_bin_i) 128 CTL_PROTO(arenas_lrun_i_size) 129 INDEX_PROTO(arenas_lrun_i) 130 CTL_PROTO(arenas_hchunk_i_size) 131 INDEX_PROTO(arenas_hchunk_i) 132 CTL_PROTO(arenas_narenas) 133 CTL_PROTO(arenas_initialized) 134 CTL_PROTO(arenas_lg_dirty_mult) 135 CTL_PROTO(arenas_quantum) 136 CTL_PROTO(arenas_page) 137 CTL_PROTO(arenas_tcache_max) 138 CTL_PROTO(arenas_nbins) 139 CTL_PROTO(arenas_nhbins) 140 CTL_PROTO(arenas_nlruns) 141 CTL_PROTO(arenas_nhchunks) 142 CTL_PROTO(arenas_extend) 143 CTL_PROTO(prof_thread_active_init) 144 CTL_PROTO(prof_active) 145 CTL_PROTO(prof_dump) 146 CTL_PROTO(prof_gdump) 147 CTL_PROTO(prof_reset) 148 CTL_PROTO(prof_interval) 149 CTL_PROTO(lg_prof_sample) 150 CTL_PROTO(stats_arenas_i_small_allocated) 151 CTL_PROTO(stats_arenas_i_small_nmalloc) 152 CTL_PROTO(stats_arenas_i_small_ndalloc) 153 CTL_PROTO(stats_arenas_i_small_nrequests) 154 CTL_PROTO(stats_arenas_i_large_allocated) 155 CTL_PROTO(stats_arenas_i_large_nmalloc) 156 CTL_PROTO(stats_arenas_i_large_ndalloc) 157 CTL_PROTO(stats_arenas_i_large_nrequests) 158 CTL_PROTO(stats_arenas_i_huge_allocated) 159 CTL_PROTO(stats_arenas_i_huge_nmalloc) 160 CTL_PROTO(stats_arenas_i_huge_ndalloc) 161 CTL_PROTO(stats_arenas_i_huge_nrequests) 162 CTL_PROTO(stats_arenas_i_bins_j_nmalloc) 163 CTL_PROTO(stats_arenas_i_bins_j_ndalloc) 164 CTL_PROTO(stats_arenas_i_bins_j_nrequests) 165 CTL_PROTO(stats_arenas_i_bins_j_curregs) 166 CTL_PROTO(stats_arenas_i_bins_j_nfills) 167 CTL_PROTO(stats_arenas_i_bins_j_nflushes) 168 CTL_PROTO(stats_arenas_i_bins_j_nruns) 169 CTL_PROTO(stats_arenas_i_bins_j_nreruns) 170 CTL_PROTO(stats_arenas_i_bins_j_curruns) 171 INDEX_PROTO(stats_arenas_i_bins_j) 172 CTL_PROTO(stats_arenas_i_lruns_j_nmalloc) 173 CTL_PROTO(stats_arenas_i_lruns_j_ndalloc) 174 CTL_PROTO(stats_arenas_i_lruns_j_nrequests) 175 CTL_PROTO(stats_arenas_i_lruns_j_curruns) 176 INDEX_PROTO(stats_arenas_i_lruns_j) 177 CTL_PROTO(stats_arenas_i_hchunks_j_nmalloc) 178 CTL_PROTO(stats_arenas_i_hchunks_j_ndalloc) 179 CTL_PROTO(stats_arenas_i_hchunks_j_nrequests) 180 CTL_PROTO(stats_arenas_i_hchunks_j_curhchunks) 181 INDEX_PROTO(stats_arenas_i_hchunks_j) 182 CTL_PROTO(stats_arenas_i_nthreads) 183 CTL_PROTO(stats_arenas_i_dss) 184 CTL_PROTO(stats_arenas_i_lg_dirty_mult) 185 CTL_PROTO(stats_arenas_i_pactive) 186 CTL_PROTO(stats_arenas_i_pdirty) 187 CTL_PROTO(stats_arenas_i_mapped) 188 CTL_PROTO(stats_arenas_i_npurge) 189 CTL_PROTO(stats_arenas_i_nmadvise) 190 CTL_PROTO(stats_arenas_i_purged) 191 CTL_PROTO(stats_arenas_i_metadata_mapped) 192 CTL_PROTO(stats_arenas_i_metadata_allocated) 193 INDEX_PROTO(stats_arenas_i) 194 CTL_PROTO(stats_cactive) 195 CTL_PROTO(stats_allocated) 196 CTL_PROTO(stats_active) 197 CTL_PROTO(stats_metadata) 198 CTL_PROTO(stats_resident) 199 CTL_PROTO(stats_mapped) 200 201 /******************************************************************************/ 202 /* mallctl tree. */ 203 204 /* Maximum tree depth. */ 205 #define CTL_MAX_DEPTH 6 206 207 #define NAME(n) {true}, n 208 #define CHILD(t, c) \ 209 sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ 210 (ctl_node_t *)c##_node, \ 211 NULL 212 #define CTL(c) 0, NULL, c##_ctl 213 214 /* 215 * Only handles internal indexed nodes, since there are currently no external 216 * ones. 217 */ 218 #define INDEX(i) {false}, i##_index 219 220 static const ctl_named_node_t thread_tcache_node[] = { 221 {NAME("enabled"), CTL(thread_tcache_enabled)}, 222 {NAME("flush"), CTL(thread_tcache_flush)} 223 }; 224 225 static const ctl_named_node_t thread_prof_node[] = { 226 {NAME("name"), CTL(thread_prof_name)}, 227 {NAME("active"), CTL(thread_prof_active)} 228 }; 229 230 static const ctl_named_node_t thread_node[] = { 231 {NAME("arena"), CTL(thread_arena)}, 232 {NAME("allocated"), CTL(thread_allocated)}, 233 {NAME("allocatedp"), CTL(thread_allocatedp)}, 234 {NAME("deallocated"), CTL(thread_deallocated)}, 235 {NAME("deallocatedp"), CTL(thread_deallocatedp)}, 236 {NAME("tcache"), CHILD(named, thread_tcache)}, 237 {NAME("prof"), CHILD(named, thread_prof)} 238 }; 239 240 static const ctl_named_node_t config_node[] = { 241 {NAME("debug"), CTL(config_debug)}, 242 {NAME("fill"), CTL(config_fill)}, 243 {NAME("lazy_lock"), CTL(config_lazy_lock)}, 244 {NAME("munmap"), CTL(config_munmap)}, 245 {NAME("prof"), CTL(config_prof)}, 246 {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, 247 {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, 248 {NAME("stats"), CTL(config_stats)}, 249 {NAME("tcache"), CTL(config_tcache)}, 250 {NAME("tls"), CTL(config_tls)}, 251 {NAME("utrace"), CTL(config_utrace)}, 252 {NAME("valgrind"), CTL(config_valgrind)}, 253 {NAME("xmalloc"), CTL(config_xmalloc)} 254 }; 255 256 static const ctl_named_node_t opt_node[] = { 257 {NAME("abort"), CTL(opt_abort)}, 258 {NAME("dss"), CTL(opt_dss)}, 259 {NAME("lg_chunk"), CTL(opt_lg_chunk)}, 260 {NAME("narenas"), CTL(opt_narenas)}, 261 {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, 262 {NAME("stats_print"), CTL(opt_stats_print)}, 263 {NAME("junk"), CTL(opt_junk)}, 264 {NAME("zero"), CTL(opt_zero)}, 265 {NAME("quarantine"), CTL(opt_quarantine)}, 266 {NAME("redzone"), CTL(opt_redzone)}, 267 {NAME("utrace"), CTL(opt_utrace)}, 268 {NAME("xmalloc"), CTL(opt_xmalloc)}, 269 {NAME("tcache"), CTL(opt_tcache)}, 270 {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, 271 {NAME("prof"), CTL(opt_prof)}, 272 {NAME("prof_prefix"), CTL(opt_prof_prefix)}, 273 {NAME("prof_active"), CTL(opt_prof_active)}, 274 {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, 275 {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, 276 {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, 277 {NAME("prof_gdump"), CTL(opt_prof_gdump)}, 278 {NAME("prof_final"), CTL(opt_prof_final)}, 279 {NAME("prof_leak"), CTL(opt_prof_leak)}, 280 {NAME("prof_accum"), CTL(opt_prof_accum)} 281 }; 282 283 static const ctl_named_node_t tcache_node[] = { 284 {NAME("create"), CTL(tcache_create)}, 285 {NAME("flush"), CTL(tcache_flush)}, 286 {NAME("destroy"), CTL(tcache_destroy)} 287 }; 288 289 static const ctl_named_node_t chunk_node[] = { 290 {NAME("alloc"), CTL(arena_i_chunk_alloc)}, 291 {NAME("dalloc"), CTL(arena_i_chunk_dalloc)}, 292 {NAME("purge"), CTL(arena_i_chunk_purge)} 293 }; 294 295 static const ctl_named_node_t arena_i_node[] = { 296 {NAME("purge"), CTL(arena_i_purge)}, 297 {NAME("dss"), CTL(arena_i_dss)}, 298 {NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)}, 299 {NAME("chunk"), CHILD(named, chunk)}, 300 }; 301 static const ctl_named_node_t super_arena_i_node[] = { 302 {NAME(""), CHILD(named, arena_i)} 303 }; 304 305 static const ctl_indexed_node_t arena_node[] = { 306 {INDEX(arena_i)} 307 }; 308 309 static const ctl_named_node_t arenas_bin_i_node[] = { 310 {NAME("size"), CTL(arenas_bin_i_size)}, 311 {NAME("nregs"), CTL(arenas_bin_i_nregs)}, 312 {NAME("run_size"), CTL(arenas_bin_i_run_size)} 313 }; 314 static const ctl_named_node_t super_arenas_bin_i_node[] = { 315 {NAME(""), CHILD(named, arenas_bin_i)} 316 }; 317 318 static const ctl_indexed_node_t arenas_bin_node[] = { 319 {INDEX(arenas_bin_i)} 320 }; 321 322 static const ctl_named_node_t arenas_lrun_i_node[] = { 323 {NAME("size"), CTL(arenas_lrun_i_size)} 324 }; 325 static const ctl_named_node_t super_arenas_lrun_i_node[] = { 326 {NAME(""), CHILD(named, arenas_lrun_i)} 327 }; 328 329 static const ctl_indexed_node_t arenas_lrun_node[] = { 330 {INDEX(arenas_lrun_i)} 331 }; 332 333 static const ctl_named_node_t arenas_hchunk_i_node[] = { 334 {NAME("size"), CTL(arenas_hchunk_i_size)} 335 }; 336 static const ctl_named_node_t super_arenas_hchunk_i_node[] = { 337 {NAME(""), CHILD(named, arenas_hchunk_i)} 338 }; 339 340 static const ctl_indexed_node_t arenas_hchunk_node[] = { 341 {INDEX(arenas_hchunk_i)} 342 }; 343 344 static const ctl_named_node_t arenas_node[] = { 345 {NAME("narenas"), CTL(arenas_narenas)}, 346 {NAME("initialized"), CTL(arenas_initialized)}, 347 {NAME("lg_dirty_mult"), CTL(arenas_lg_dirty_mult)}, 348 {NAME("quantum"), CTL(arenas_quantum)}, 349 {NAME("page"), CTL(arenas_page)}, 350 {NAME("tcache_max"), CTL(arenas_tcache_max)}, 351 {NAME("nbins"), CTL(arenas_nbins)}, 352 {NAME("nhbins"), CTL(arenas_nhbins)}, 353 {NAME("bin"), CHILD(indexed, arenas_bin)}, 354 {NAME("nlruns"), CTL(arenas_nlruns)}, 355 {NAME("lrun"), CHILD(indexed, arenas_lrun)}, 356 {NAME("nhchunks"), CTL(arenas_nhchunks)}, 357 {NAME("hchunk"), CHILD(indexed, arenas_hchunk)}, 358 {NAME("extend"), CTL(arenas_extend)} 359 }; 360 361 static const ctl_named_node_t prof_node[] = { 362 {NAME("thread_active_init"), CTL(prof_thread_active_init)}, 363 {NAME("active"), CTL(prof_active)}, 364 {NAME("dump"), CTL(prof_dump)}, 365 {NAME("gdump"), CTL(prof_gdump)}, 366 {NAME("reset"), CTL(prof_reset)}, 367 {NAME("interval"), CTL(prof_interval)}, 368 {NAME("lg_sample"), CTL(lg_prof_sample)} 369 }; 370 371 static const ctl_named_node_t stats_arenas_i_metadata_node[] = { 372 {NAME("mapped"), CTL(stats_arenas_i_metadata_mapped)}, 373 {NAME("allocated"), CTL(stats_arenas_i_metadata_allocated)} 374 }; 375 376 static const ctl_named_node_t stats_arenas_i_small_node[] = { 377 {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, 378 {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, 379 {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, 380 {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} 381 }; 382 383 static const ctl_named_node_t stats_arenas_i_large_node[] = { 384 {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, 385 {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, 386 {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, 387 {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} 388 }; 389 390 static const ctl_named_node_t stats_arenas_i_huge_node[] = { 391 {NAME("allocated"), CTL(stats_arenas_i_huge_allocated)}, 392 {NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)}, 393 {NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)}, 394 {NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)} 395 }; 396 397 static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { 398 {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, 399 {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, 400 {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, 401 {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, 402 {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, 403 {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, 404 {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, 405 {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, 406 {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} 407 }; 408 static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { 409 {NAME(""), CHILD(named, stats_arenas_i_bins_j)} 410 }; 411 412 static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { 413 {INDEX(stats_arenas_i_bins_j)} 414 }; 415 416 static const ctl_named_node_t stats_arenas_i_lruns_j_node[] = { 417 {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, 418 {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, 419 {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, 420 {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} 421 }; 422 static const ctl_named_node_t super_stats_arenas_i_lruns_j_node[] = { 423 {NAME(""), CHILD(named, stats_arenas_i_lruns_j)} 424 }; 425 426 static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = { 427 {INDEX(stats_arenas_i_lruns_j)} 428 }; 429 430 static const ctl_named_node_t stats_arenas_i_hchunks_j_node[] = { 431 {NAME("nmalloc"), CTL(stats_arenas_i_hchunks_j_nmalloc)}, 432 {NAME("ndalloc"), CTL(stats_arenas_i_hchunks_j_ndalloc)}, 433 {NAME("nrequests"), CTL(stats_arenas_i_hchunks_j_nrequests)}, 434 {NAME("curhchunks"), CTL(stats_arenas_i_hchunks_j_curhchunks)} 435 }; 436 static const ctl_named_node_t super_stats_arenas_i_hchunks_j_node[] = { 437 {NAME(""), CHILD(named, stats_arenas_i_hchunks_j)} 438 }; 439 440 static const ctl_indexed_node_t stats_arenas_i_hchunks_node[] = { 441 {INDEX(stats_arenas_i_hchunks_j)} 442 }; 443 444 static const ctl_named_node_t stats_arenas_i_node[] = { 445 {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, 446 {NAME("dss"), CTL(stats_arenas_i_dss)}, 447 {NAME("lg_dirty_mult"), CTL(stats_arenas_i_lg_dirty_mult)}, 448 {NAME("pactive"), CTL(stats_arenas_i_pactive)}, 449 {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, 450 {NAME("mapped"), CTL(stats_arenas_i_mapped)}, 451 {NAME("npurge"), CTL(stats_arenas_i_npurge)}, 452 {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, 453 {NAME("purged"), CTL(stats_arenas_i_purged)}, 454 {NAME("metadata"), CHILD(named, stats_arenas_i_metadata)}, 455 {NAME("small"), CHILD(named, stats_arenas_i_small)}, 456 {NAME("large"), CHILD(named, stats_arenas_i_large)}, 457 {NAME("huge"), CHILD(named, stats_arenas_i_huge)}, 458 {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, 459 {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)}, 460 {NAME("hchunks"), CHILD(indexed, stats_arenas_i_hchunks)} 461 }; 462 static const ctl_named_node_t super_stats_arenas_i_node[] = { 463 {NAME(""), CHILD(named, stats_arenas_i)} 464 }; 465 466 static const ctl_indexed_node_t stats_arenas_node[] = { 467 {INDEX(stats_arenas_i)} 468 }; 469 470 static const ctl_named_node_t stats_node[] = { 471 {NAME("cactive"), CTL(stats_cactive)}, 472 {NAME("allocated"), CTL(stats_allocated)}, 473 {NAME("active"), CTL(stats_active)}, 474 {NAME("metadata"), CTL(stats_metadata)}, 475 {NAME("resident"), CTL(stats_resident)}, 476 {NAME("mapped"), CTL(stats_mapped)}, 477 {NAME("arenas"), CHILD(indexed, stats_arenas)} 478 }; 479 480 static const ctl_named_node_t root_node[] = { 481 {NAME("version"), CTL(version)}, 482 {NAME("epoch"), CTL(epoch)}, 483 {NAME("thread"), CHILD(named, thread)}, 484 {NAME("config"), CHILD(named, config)}, 485 {NAME("opt"), CHILD(named, opt)}, 486 {NAME("tcache"), CHILD(named, tcache)}, 487 {NAME("arena"), CHILD(indexed, arena)}, 488 {NAME("arenas"), CHILD(named, arenas)}, 489 {NAME("prof"), CHILD(named, prof)}, 490 {NAME("stats"), CHILD(named, stats)} 491 }; 492 static const ctl_named_node_t super_root_node[] = { 493 {NAME(""), CHILD(named, root)} 494 }; 495 496 #undef NAME 497 #undef CHILD 498 #undef CTL 499 #undef INDEX 500 501 /******************************************************************************/ 502 503 static bool 504 ctl_arena_init(ctl_arena_stats_t *astats) 505 { 506 507 if (astats->lstats == NULL) { 508 astats->lstats = (malloc_large_stats_t *)a0malloc(nlclasses * 509 sizeof(malloc_large_stats_t)); 510 if (astats->lstats == NULL) 511 return (true); 512 } 513 514 if (astats->hstats == NULL) { 515 astats->hstats = (malloc_huge_stats_t *)a0malloc(nhclasses * 516 sizeof(malloc_huge_stats_t)); 517 if (astats->hstats == NULL) 518 return (true); 519 } 520 521 return (false); 522 } 523 524 static void 525 ctl_arena_clear(ctl_arena_stats_t *astats) 526 { 527 528 astats->dss = dss_prec_names[dss_prec_limit]; 529 astats->lg_dirty_mult = -1; 530 astats->pactive = 0; 531 astats->pdirty = 0; 532 if (config_stats) { 533 memset(&astats->astats, 0, sizeof(arena_stats_t)); 534 astats->allocated_small = 0; 535 astats->nmalloc_small = 0; 536 astats->ndalloc_small = 0; 537 astats->nrequests_small = 0; 538 memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t)); 539 memset(astats->lstats, 0, nlclasses * 540 sizeof(malloc_large_stats_t)); 541 memset(astats->hstats, 0, nhclasses * 542 sizeof(malloc_huge_stats_t)); 543 } 544 } 545 546 static void 547 ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) 548 { 549 unsigned i; 550 551 arena_stats_merge(arena, &cstats->dss, &cstats->lg_dirty_mult, 552 &cstats->pactive, &cstats->pdirty, &cstats->astats, cstats->bstats, 553 cstats->lstats, cstats->hstats); 554 555 for (i = 0; i < NBINS; i++) { 556 cstats->allocated_small += cstats->bstats[i].curregs * 557 index2size(i); 558 cstats->nmalloc_small += cstats->bstats[i].nmalloc; 559 cstats->ndalloc_small += cstats->bstats[i].ndalloc; 560 cstats->nrequests_small += cstats->bstats[i].nrequests; 561 } 562 } 563 564 static void 565 ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) 566 { 567 unsigned i; 568 569 sstats->pactive += astats->pactive; 570 sstats->pdirty += astats->pdirty; 571 572 sstats->astats.mapped += astats->astats.mapped; 573 sstats->astats.npurge += astats->astats.npurge; 574 sstats->astats.nmadvise += astats->astats.nmadvise; 575 sstats->astats.purged += astats->astats.purged; 576 577 sstats->astats.metadata_mapped += astats->astats.metadata_mapped; 578 sstats->astats.metadata_allocated += astats->astats.metadata_allocated; 579 580 sstats->allocated_small += astats->allocated_small; 581 sstats->nmalloc_small += astats->nmalloc_small; 582 sstats->ndalloc_small += astats->ndalloc_small; 583 sstats->nrequests_small += astats->nrequests_small; 584 585 sstats->astats.allocated_large += astats->astats.allocated_large; 586 sstats->astats.nmalloc_large += astats->astats.nmalloc_large; 587 sstats->astats.ndalloc_large += astats->astats.ndalloc_large; 588 sstats->astats.nrequests_large += astats->astats.nrequests_large; 589 590 sstats->astats.allocated_huge += astats->astats.allocated_huge; 591 sstats->astats.nmalloc_huge += astats->astats.nmalloc_huge; 592 sstats->astats.ndalloc_huge += astats->astats.ndalloc_huge; 593 594 for (i = 0; i < NBINS; i++) { 595 sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; 596 sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; 597 sstats->bstats[i].nrequests += astats->bstats[i].nrequests; 598 sstats->bstats[i].curregs += astats->bstats[i].curregs; 599 if (config_tcache) { 600 sstats->bstats[i].nfills += astats->bstats[i].nfills; 601 sstats->bstats[i].nflushes += 602 astats->bstats[i].nflushes; 603 } 604 sstats->bstats[i].nruns += astats->bstats[i].nruns; 605 sstats->bstats[i].reruns += astats->bstats[i].reruns; 606 sstats->bstats[i].curruns += astats->bstats[i].curruns; 607 } 608 609 for (i = 0; i < nlclasses; i++) { 610 sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; 611 sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; 612 sstats->lstats[i].nrequests += astats->lstats[i].nrequests; 613 sstats->lstats[i].curruns += astats->lstats[i].curruns; 614 } 615 616 for (i = 0; i < nhclasses; i++) { 617 sstats->hstats[i].nmalloc += astats->hstats[i].nmalloc; 618 sstats->hstats[i].ndalloc += astats->hstats[i].ndalloc; 619 sstats->hstats[i].curhchunks += astats->hstats[i].curhchunks; 620 } 621 } 622 623 static void 624 ctl_arena_refresh(arena_t *arena, unsigned i) 625 { 626 ctl_arena_stats_t *astats = &ctl_stats.arenas[i]; 627 ctl_arena_stats_t *sstats = &ctl_stats.arenas[ctl_stats.narenas]; 628 629 ctl_arena_clear(astats); 630 631 sstats->nthreads += astats->nthreads; 632 if (config_stats) { 633 ctl_arena_stats_amerge(astats, arena); 634 /* Merge into sum stats as well. */ 635 ctl_arena_stats_smerge(sstats, astats); 636 } else { 637 astats->pactive += arena->nactive; 638 astats->pdirty += arena->ndirty; 639 /* Merge into sum stats as well. */ 640 sstats->pactive += arena->nactive; 641 sstats->pdirty += arena->ndirty; 642 } 643 } 644 645 static bool 646 ctl_grow(void) 647 { 648 ctl_arena_stats_t *astats; 649 650 /* Initialize new arena. */ 651 if (arena_init(ctl_stats.narenas) == NULL) 652 return (true); 653 654 /* Allocate extended arena stats. */ 655 astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) * 656 sizeof(ctl_arena_stats_t)); 657 if (astats == NULL) 658 return (true); 659 660 /* Initialize the new astats element. */ 661 memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * 662 sizeof(ctl_arena_stats_t)); 663 memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); 664 if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { 665 a0dalloc(astats); 666 return (true); 667 } 668 /* Swap merged stats to their new location. */ 669 { 670 ctl_arena_stats_t tstats; 671 memcpy(&tstats, &astats[ctl_stats.narenas], 672 sizeof(ctl_arena_stats_t)); 673 memcpy(&astats[ctl_stats.narenas], 674 &astats[ctl_stats.narenas + 1], sizeof(ctl_arena_stats_t)); 675 memcpy(&astats[ctl_stats.narenas + 1], &tstats, 676 sizeof(ctl_arena_stats_t)); 677 } 678 a0dalloc(ctl_stats.arenas); 679 ctl_stats.arenas = astats; 680 ctl_stats.narenas++; 681 682 return (false); 683 } 684 685 static void 686 ctl_refresh(void) 687 { 688 tsd_t *tsd; 689 unsigned i; 690 bool refreshed; 691 VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); 692 693 /* 694 * Clear sum stats, since they will be merged into by 695 * ctl_arena_refresh(). 696 */ 697 ctl_stats.arenas[ctl_stats.narenas].nthreads = 0; 698 ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]); 699 700 tsd = tsd_fetch(); 701 for (i = 0, refreshed = false; i < ctl_stats.narenas; i++) { 702 tarenas[i] = arena_get(tsd, i, false, false); 703 if (tarenas[i] == NULL && !refreshed) { 704 tarenas[i] = arena_get(tsd, i, false, true); 705 refreshed = true; 706 } 707 } 708 709 for (i = 0; i < ctl_stats.narenas; i++) { 710 if (tarenas[i] != NULL) 711 ctl_stats.arenas[i].nthreads = arena_nbound(i); 712 else 713 ctl_stats.arenas[i].nthreads = 0; 714 } 715 716 for (i = 0; i < ctl_stats.narenas; i++) { 717 bool initialized = (tarenas[i] != NULL); 718 719 ctl_stats.arenas[i].initialized = initialized; 720 if (initialized) 721 ctl_arena_refresh(tarenas[i], i); 722 } 723 724 if (config_stats) { 725 size_t base_allocated, base_resident, base_mapped; 726 base_stats_get(&base_allocated, &base_resident, &base_mapped); 727 ctl_stats.allocated = 728 ctl_stats.arenas[ctl_stats.narenas].allocated_small + 729 ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large + 730 ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; 731 ctl_stats.active = 732 (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE); 733 ctl_stats.metadata = base_allocated + 734 ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + 735 ctl_stats.arenas[ctl_stats.narenas].astats 736 .metadata_allocated; 737 ctl_stats.resident = base_resident + 738 ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + 739 ((ctl_stats.arenas[ctl_stats.narenas].pactive + 740 ctl_stats.arenas[ctl_stats.narenas].pdirty) << LG_PAGE); 741 ctl_stats.mapped = base_mapped + 742 ctl_stats.arenas[ctl_stats.narenas].astats.mapped; 743 } 744 745 ctl_epoch++; 746 } 747 748 static bool 749 ctl_init(void) 750 { 751 bool ret; 752 753 malloc_mutex_lock(&ctl_mtx); 754 if (!ctl_initialized) { 755 /* 756 * Allocate space for one extra arena stats element, which 757 * contains summed stats across all arenas. 758 */ 759 ctl_stats.narenas = narenas_total_get(); 760 ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc( 761 (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); 762 if (ctl_stats.arenas == NULL) { 763 ret = true; 764 goto label_return; 765 } 766 memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) * 767 sizeof(ctl_arena_stats_t)); 768 769 /* 770 * Initialize all stats structures, regardless of whether they 771 * ever get used. Lazy initialization would allow errors to 772 * cause inconsistent state to be viewable by the application. 773 */ 774 if (config_stats) { 775 unsigned i; 776 for (i = 0; i <= ctl_stats.narenas; i++) { 777 if (ctl_arena_init(&ctl_stats.arenas[i])) { 778 unsigned j; 779 for (j = 0; j < i; j++) { 780 a0dalloc( 781 ctl_stats.arenas[j].lstats); 782 a0dalloc( 783 ctl_stats.arenas[j].hstats); 784 } 785 a0dalloc(ctl_stats.arenas); 786 ctl_stats.arenas = NULL; 787 ret = true; 788 goto label_return; 789 } 790 } 791 } 792 ctl_stats.arenas[ctl_stats.narenas].initialized = true; 793 794 ctl_epoch = 0; 795 ctl_refresh(); 796 ctl_initialized = true; 797 } 798 799 ret = false; 800 label_return: 801 malloc_mutex_unlock(&ctl_mtx); 802 return (ret); 803 } 804 805 static int 806 ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, 807 size_t *depthp) 808 { 809 int ret; 810 const char *elm, *tdot, *dot; 811 size_t elen, i, j; 812 const ctl_named_node_t *node; 813 814 elm = name; 815 /* Equivalent to strchrnul(). */ 816 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); 817 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 818 if (elen == 0) { 819 ret = ENOENT; 820 goto label_return; 821 } 822 node = super_root_node; 823 for (i = 0; i < *depthp; i++) { 824 assert(node); 825 assert(node->nchildren > 0); 826 if (ctl_named_node(node->children) != NULL) { 827 const ctl_named_node_t *pnode = node; 828 829 /* Children are named. */ 830 for (j = 0; j < node->nchildren; j++) { 831 const ctl_named_node_t *child = 832 ctl_named_children(node, j); 833 if (strlen(child->name) == elen && 834 strncmp(elm, child->name, elen) == 0) { 835 node = child; 836 if (nodesp != NULL) 837 nodesp[i] = 838 (const ctl_node_t *)node; 839 mibp[i] = j; 840 break; 841 } 842 } 843 if (node == pnode) { 844 ret = ENOENT; 845 goto label_return; 846 } 847 } else { 848 uintmax_t index; 849 const ctl_indexed_node_t *inode; 850 851 /* Children are indexed. */ 852 index = malloc_strtoumax(elm, NULL, 10); 853 if (index == UINTMAX_MAX || index > SIZE_T_MAX) { 854 ret = ENOENT; 855 goto label_return; 856 } 857 858 inode = ctl_indexed_node(node->children); 859 node = inode->index(mibp, *depthp, (size_t)index); 860 if (node == NULL) { 861 ret = ENOENT; 862 goto label_return; 863 } 864 865 if (nodesp != NULL) 866 nodesp[i] = (const ctl_node_t *)node; 867 mibp[i] = (size_t)index; 868 } 869 870 if (node->ctl != NULL) { 871 /* Terminal node. */ 872 if (*dot != '\0') { 873 /* 874 * The name contains more elements than are 875 * in this path through the tree. 876 */ 877 ret = ENOENT; 878 goto label_return; 879 } 880 /* Complete lookup successful. */ 881 *depthp = i + 1; 882 break; 883 } 884 885 /* Update elm. */ 886 if (*dot == '\0') { 887 /* No more elements. */ 888 ret = ENOENT; 889 goto label_return; 890 } 891 elm = &dot[1]; 892 dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : 893 strchr(elm, '\0'); 894 elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); 895 } 896 897 ret = 0; 898 label_return: 899 return (ret); 900 } 901 902 int 903 ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, 904 size_t newlen) 905 { 906 int ret; 907 size_t depth; 908 ctl_node_t const *nodes[CTL_MAX_DEPTH]; 909 size_t mib[CTL_MAX_DEPTH]; 910 const ctl_named_node_t *node; 911 912 if (!ctl_initialized && ctl_init()) { 913 ret = EAGAIN; 914 goto label_return; 915 } 916 917 depth = CTL_MAX_DEPTH; 918 ret = ctl_lookup(name, nodes, mib, &depth); 919 if (ret != 0) 920 goto label_return; 921 922 node = ctl_named_node(nodes[depth-1]); 923 if (node != NULL && node->ctl) 924 ret = node->ctl(mib, depth, oldp, oldlenp, newp, newlen); 925 else { 926 /* The name refers to a partial path through the ctl tree. */ 927 ret = ENOENT; 928 } 929 930 label_return: 931 return(ret); 932 } 933 934 int 935 ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp) 936 { 937 int ret; 938 939 if (!ctl_initialized && ctl_init()) { 940 ret = EAGAIN; 941 goto label_return; 942 } 943 944 ret = ctl_lookup(name, NULL, mibp, miblenp); 945 label_return: 946 return(ret); 947 } 948 949 int 950 ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 951 void *newp, size_t newlen) 952 { 953 int ret; 954 const ctl_named_node_t *node; 955 size_t i; 956 957 if (!ctl_initialized && ctl_init()) { 958 ret = EAGAIN; 959 goto label_return; 960 } 961 962 /* Iterate down the tree. */ 963 node = super_root_node; 964 for (i = 0; i < miblen; i++) { 965 assert(node); 966 assert(node->nchildren > 0); 967 if (ctl_named_node(node->children) != NULL) { 968 /* Children are named. */ 969 if (node->nchildren <= mib[i]) { 970 ret = ENOENT; 971 goto label_return; 972 } 973 node = ctl_named_children(node, mib[i]); 974 } else { 975 const ctl_indexed_node_t *inode; 976 977 /* Indexed element. */ 978 inode = ctl_indexed_node(node->children); 979 node = inode->index(mib, miblen, mib[i]); 980 if (node == NULL) { 981 ret = ENOENT; 982 goto label_return; 983 } 984 } 985 } 986 987 /* Call the ctl function. */ 988 if (node && node->ctl) 989 ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); 990 else { 991 /* Partial MIB. */ 992 ret = ENOENT; 993 } 994 995 label_return: 996 return(ret); 997 } 998 999 bool 1000 ctl_boot(void) 1001 { 1002 1003 if (malloc_mutex_init(&ctl_mtx)) 1004 return (true); 1005 1006 ctl_initialized = false; 1007 1008 return (false); 1009 } 1010 1011 void 1012 ctl_prefork(void) 1013 { 1014 1015 malloc_mutex_prefork(&ctl_mtx); 1016 } 1017 1018 void 1019 ctl_postfork_parent(void) 1020 { 1021 1022 malloc_mutex_postfork_parent(&ctl_mtx); 1023 } 1024 1025 void 1026 ctl_postfork_child(void) 1027 { 1028 1029 malloc_mutex_postfork_child(&ctl_mtx); 1030 } 1031 1032 /******************************************************************************/ 1033 /* *_ctl() functions. */ 1034 1035 #define READONLY() do { \ 1036 if (newp != NULL || newlen != 0) { \ 1037 ret = EPERM; \ 1038 goto label_return; \ 1039 } \ 1040 } while (0) 1041 1042 #define WRITEONLY() do { \ 1043 if (oldp != NULL || oldlenp != NULL) { \ 1044 ret = EPERM; \ 1045 goto label_return; \ 1046 } \ 1047 } while (0) 1048 1049 #define READ_XOR_WRITE() do { \ 1050 if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ 1051 newlen != 0)) { \ 1052 ret = EPERM; \ 1053 goto label_return; \ 1054 } \ 1055 } while (0) 1056 1057 #define READ(v, t) do { \ 1058 if (oldp != NULL && oldlenp != NULL) { \ 1059 if (*oldlenp != sizeof(t)) { \ 1060 size_t copylen = (sizeof(t) <= *oldlenp) \ 1061 ? sizeof(t) : *oldlenp; \ 1062 memcpy(oldp, (void *)&(v), copylen); \ 1063 ret = EINVAL; \ 1064 goto label_return; \ 1065 } else \ 1066 *(t *)oldp = (v); \ 1067 } \ 1068 } while (0) 1069 1070 #define WRITE(v, t) do { \ 1071 if (newp != NULL) { \ 1072 if (newlen != sizeof(t)) { \ 1073 ret = EINVAL; \ 1074 goto label_return; \ 1075 } \ 1076 (v) = *(t *)newp; \ 1077 } \ 1078 } while (0) 1079 1080 /* 1081 * There's a lot of code duplication in the following macros due to limitations 1082 * in how nested cpp macros are expanded. 1083 */ 1084 #define CTL_RO_CLGEN(c, l, n, v, t) \ 1085 static int \ 1086 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1087 void *newp, size_t newlen) \ 1088 { \ 1089 int ret; \ 1090 t oldval; \ 1091 \ 1092 if (!(c)) \ 1093 return (ENOENT); \ 1094 if (l) \ 1095 malloc_mutex_lock(&ctl_mtx); \ 1096 READONLY(); \ 1097 oldval = (v); \ 1098 READ(oldval, t); \ 1099 \ 1100 ret = 0; \ 1101 label_return: \ 1102 if (l) \ 1103 malloc_mutex_unlock(&ctl_mtx); \ 1104 return (ret); \ 1105 } 1106 1107 #define CTL_RO_CGEN(c, n, v, t) \ 1108 static int \ 1109 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1110 void *newp, size_t newlen) \ 1111 { \ 1112 int ret; \ 1113 t oldval; \ 1114 \ 1115 if (!(c)) \ 1116 return (ENOENT); \ 1117 malloc_mutex_lock(&ctl_mtx); \ 1118 READONLY(); \ 1119 oldval = (v); \ 1120 READ(oldval, t); \ 1121 \ 1122 ret = 0; \ 1123 label_return: \ 1124 malloc_mutex_unlock(&ctl_mtx); \ 1125 return (ret); \ 1126 } 1127 1128 #define CTL_RO_GEN(n, v, t) \ 1129 static int \ 1130 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1131 void *newp, size_t newlen) \ 1132 { \ 1133 int ret; \ 1134 t oldval; \ 1135 \ 1136 malloc_mutex_lock(&ctl_mtx); \ 1137 READONLY(); \ 1138 oldval = (v); \ 1139 READ(oldval, t); \ 1140 \ 1141 ret = 0; \ 1142 label_return: \ 1143 malloc_mutex_unlock(&ctl_mtx); \ 1144 return (ret); \ 1145 } 1146 1147 /* 1148 * ctl_mtx is not acquired, under the assumption that no pertinent data will 1149 * mutate during the call. 1150 */ 1151 #define CTL_RO_NL_CGEN(c, n, v, t) \ 1152 static int \ 1153 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1154 void *newp, size_t newlen) \ 1155 { \ 1156 int ret; \ 1157 t oldval; \ 1158 \ 1159 if (!(c)) \ 1160 return (ENOENT); \ 1161 READONLY(); \ 1162 oldval = (v); \ 1163 READ(oldval, t); \ 1164 \ 1165 ret = 0; \ 1166 label_return: \ 1167 return (ret); \ 1168 } 1169 1170 #define CTL_RO_NL_GEN(n, v, t) \ 1171 static int \ 1172 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1173 void *newp, size_t newlen) \ 1174 { \ 1175 int ret; \ 1176 t oldval; \ 1177 \ 1178 READONLY(); \ 1179 oldval = (v); \ 1180 READ(oldval, t); \ 1181 \ 1182 ret = 0; \ 1183 label_return: \ 1184 return (ret); \ 1185 } 1186 1187 #define CTL_TSD_RO_NL_CGEN(c, n, m, t) \ 1188 static int \ 1189 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1190 void *newp, size_t newlen) \ 1191 { \ 1192 int ret; \ 1193 t oldval; \ 1194 tsd_t *tsd; \ 1195 \ 1196 if (!(c)) \ 1197 return (ENOENT); \ 1198 READONLY(); \ 1199 tsd = tsd_fetch(); \ 1200 oldval = (m(tsd)); \ 1201 READ(oldval, t); \ 1202 \ 1203 ret = 0; \ 1204 label_return: \ 1205 return (ret); \ 1206 } 1207 1208 #define CTL_RO_BOOL_CONFIG_GEN(n) \ 1209 static int \ 1210 n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ 1211 void *newp, size_t newlen) \ 1212 { \ 1213 int ret; \ 1214 bool oldval; \ 1215 \ 1216 READONLY(); \ 1217 oldval = n; \ 1218 READ(oldval, bool); \ 1219 \ 1220 ret = 0; \ 1221 label_return: \ 1222 return (ret); \ 1223 } 1224 1225 /******************************************************************************/ 1226 1227 CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) 1228 1229 static int 1230 epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1231 void *newp, size_t newlen) 1232 { 1233 int ret; 1234 UNUSED uint64_t newval; 1235 1236 malloc_mutex_lock(&ctl_mtx); 1237 WRITE(newval, uint64_t); 1238 if (newp != NULL) 1239 ctl_refresh(); 1240 READ(ctl_epoch, uint64_t); 1241 1242 ret = 0; 1243 label_return: 1244 malloc_mutex_unlock(&ctl_mtx); 1245 return (ret); 1246 } 1247 1248 /******************************************************************************/ 1249 1250 CTL_RO_BOOL_CONFIG_GEN(config_debug) 1251 CTL_RO_BOOL_CONFIG_GEN(config_fill) 1252 CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) 1253 CTL_RO_BOOL_CONFIG_GEN(config_munmap) 1254 CTL_RO_BOOL_CONFIG_GEN(config_prof) 1255 CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) 1256 CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind) 1257 CTL_RO_BOOL_CONFIG_GEN(config_stats) 1258 CTL_RO_BOOL_CONFIG_GEN(config_tcache) 1259 CTL_RO_BOOL_CONFIG_GEN(config_tls) 1260 CTL_RO_BOOL_CONFIG_GEN(config_utrace) 1261 CTL_RO_BOOL_CONFIG_GEN(config_valgrind) 1262 CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) 1263 1264 /******************************************************************************/ 1265 1266 CTL_RO_NL_GEN(opt_abort, opt_abort, bool) 1267 CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) 1268 CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) 1269 CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) 1270 CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) 1271 CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) 1272 CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) 1273 CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) 1274 CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) 1275 CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) 1276 CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) 1277 CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) 1278 CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) 1279 CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) 1280 CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) 1281 CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) 1282 CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) 1283 CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, 1284 opt_prof_thread_active_init, bool) 1285 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) 1286 CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) 1287 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) 1288 CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) 1289 CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) 1290 CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) 1291 1292 /******************************************************************************/ 1293 1294 static int 1295 thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1296 void *newp, size_t newlen) 1297 { 1298 int ret; 1299 tsd_t *tsd; 1300 arena_t *oldarena; 1301 unsigned newind, oldind; 1302 1303 tsd = tsd_fetch(); 1304 oldarena = arena_choose(tsd, NULL); 1305 if (oldarena == NULL) 1306 return (EAGAIN); 1307 1308 malloc_mutex_lock(&ctl_mtx); 1309 newind = oldind = oldarena->ind; 1310 WRITE(newind, unsigned); 1311 READ(oldind, unsigned); 1312 if (newind != oldind) { 1313 arena_t *newarena; 1314 1315 if (newind >= ctl_stats.narenas) { 1316 /* New arena index is out of range. */ 1317 ret = EFAULT; 1318 goto label_return; 1319 } 1320 1321 /* Initialize arena if necessary. */ 1322 newarena = arena_get(tsd, newind, true, true); 1323 if (newarena == NULL) { 1324 ret = EAGAIN; 1325 goto label_return; 1326 } 1327 /* Set new arena/tcache associations. */ 1328 arena_migrate(tsd, oldind, newind); 1329 if (config_tcache) { 1330 tcache_t *tcache = tsd_tcache_get(tsd); 1331 if (tcache != NULL) { 1332 tcache_arena_reassociate(tcache, oldarena, 1333 newarena); 1334 } 1335 } 1336 } 1337 1338 ret = 0; 1339 label_return: 1340 malloc_mutex_unlock(&ctl_mtx); 1341 return (ret); 1342 } 1343 1344 CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get, 1345 uint64_t) 1346 CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get, 1347 uint64_t *) 1348 CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get, 1349 uint64_t) 1350 CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, 1351 tsd_thread_deallocatedp_get, uint64_t *) 1352 1353 static int 1354 thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, 1355 size_t *oldlenp, void *newp, size_t newlen) 1356 { 1357 int ret; 1358 bool oldval; 1359 1360 if (!config_tcache) 1361 return (ENOENT); 1362 1363 oldval = tcache_enabled_get(); 1364 if (newp != NULL) { 1365 if (newlen != sizeof(bool)) { 1366 ret = EINVAL; 1367 goto label_return; 1368 } 1369 tcache_enabled_set(*(bool *)newp); 1370 } 1371 READ(oldval, bool); 1372 1373 ret = 0; 1374 label_return: 1375 return (ret); 1376 } 1377 1378 static int 1379 thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, 1380 size_t *oldlenp, void *newp, size_t newlen) 1381 { 1382 int ret; 1383 1384 if (!config_tcache) 1385 return (ENOENT); 1386 1387 READONLY(); 1388 WRITEONLY(); 1389 1390 tcache_flush(); 1391 1392 ret = 0; 1393 label_return: 1394 return (ret); 1395 } 1396 1397 static int 1398 thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, 1399 size_t *oldlenp, void *newp, size_t newlen) 1400 { 1401 int ret; 1402 1403 if (!config_prof) 1404 return (ENOENT); 1405 1406 READ_XOR_WRITE(); 1407 1408 if (newp != NULL) { 1409 tsd_t *tsd; 1410 1411 if (newlen != sizeof(const char *)) { 1412 ret = EINVAL; 1413 goto label_return; 1414 } 1415 1416 tsd = tsd_fetch(); 1417 1418 if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != 1419 0) 1420 goto label_return; 1421 } else { 1422 const char *oldname = prof_thread_name_get(); 1423 READ(oldname, const char *); 1424 } 1425 1426 ret = 0; 1427 label_return: 1428 return (ret); 1429 } 1430 1431 static int 1432 thread_prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, 1433 size_t *oldlenp, void *newp, size_t newlen) 1434 { 1435 int ret; 1436 bool oldval; 1437 1438 if (!config_prof) 1439 return (ENOENT); 1440 1441 oldval = prof_thread_active_get(); 1442 if (newp != NULL) { 1443 if (newlen != sizeof(bool)) { 1444 ret = EINVAL; 1445 goto label_return; 1446 } 1447 if (prof_thread_active_set(*(bool *)newp)) { 1448 ret = EAGAIN; 1449 goto label_return; 1450 } 1451 } 1452 READ(oldval, bool); 1453 1454 ret = 0; 1455 label_return: 1456 return (ret); 1457 } 1458 1459 /******************************************************************************/ 1460 1461 static int 1462 tcache_create_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1463 void *newp, size_t newlen) 1464 { 1465 int ret; 1466 tsd_t *tsd; 1467 unsigned tcache_ind; 1468 1469 if (!config_tcache) 1470 return (ENOENT); 1471 1472 tsd = tsd_fetch(); 1473 1474 malloc_mutex_lock(&ctl_mtx); 1475 READONLY(); 1476 if (tcaches_create(tsd, &tcache_ind)) { 1477 ret = EFAULT; 1478 goto label_return; 1479 } 1480 READ(tcache_ind, unsigned); 1481 1482 ret = 0; 1483 label_return: 1484 malloc_mutex_unlock(&ctl_mtx); 1485 return (ret); 1486 } 1487 1488 static int 1489 tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1490 void *newp, size_t newlen) 1491 { 1492 int ret; 1493 tsd_t *tsd; 1494 unsigned tcache_ind; 1495 1496 if (!config_tcache) 1497 return (ENOENT); 1498 1499 tsd = tsd_fetch(); 1500 1501 WRITEONLY(); 1502 tcache_ind = UINT_MAX; 1503 WRITE(tcache_ind, unsigned); 1504 if (tcache_ind == UINT_MAX) { 1505 ret = EFAULT; 1506 goto label_return; 1507 } 1508 tcaches_flush(tsd, tcache_ind); 1509 1510 ret = 0; 1511 label_return: 1512 return (ret); 1513 } 1514 1515 static int 1516 tcache_destroy_ctl(const size_t *mib, size_t miblen, void *oldp, 1517 size_t *oldlenp, void *newp, size_t newlen) 1518 { 1519 int ret; 1520 tsd_t *tsd; 1521 unsigned tcache_ind; 1522 1523 if (!config_tcache) 1524 return (ENOENT); 1525 1526 tsd = tsd_fetch(); 1527 1528 WRITEONLY(); 1529 tcache_ind = UINT_MAX; 1530 WRITE(tcache_ind, unsigned); 1531 if (tcache_ind == UINT_MAX) { 1532 ret = EFAULT; 1533 goto label_return; 1534 } 1535 tcaches_destroy(tsd, tcache_ind); 1536 1537 ret = 0; 1538 label_return: 1539 return (ret); 1540 } 1541 1542 /******************************************************************************/ 1543 1544 /* ctl_mutex must be held during execution of this function. */ 1545 static void 1546 arena_purge(unsigned arena_ind) 1547 { 1548 tsd_t *tsd; 1549 unsigned i; 1550 bool refreshed; 1551 VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); 1552 1553 tsd = tsd_fetch(); 1554 for (i = 0, refreshed = false; i < ctl_stats.narenas; i++) { 1555 tarenas[i] = arena_get(tsd, i, false, false); 1556 if (tarenas[i] == NULL && !refreshed) { 1557 tarenas[i] = arena_get(tsd, i, false, true); 1558 refreshed = true; 1559 } 1560 } 1561 1562 if (arena_ind == ctl_stats.narenas) { 1563 unsigned i; 1564 for (i = 0; i < ctl_stats.narenas; i++) { 1565 if (tarenas[i] != NULL) 1566 arena_purge_all(tarenas[i]); 1567 } 1568 } else { 1569 assert(arena_ind < ctl_stats.narenas); 1570 if (tarenas[arena_ind] != NULL) 1571 arena_purge_all(tarenas[arena_ind]); 1572 } 1573 } 1574 1575 static int 1576 arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1577 void *newp, size_t newlen) 1578 { 1579 int ret; 1580 1581 READONLY(); 1582 WRITEONLY(); 1583 malloc_mutex_lock(&ctl_mtx); 1584 arena_purge(mib[1]); 1585 malloc_mutex_unlock(&ctl_mtx); 1586 1587 ret = 0; 1588 label_return: 1589 return (ret); 1590 } 1591 1592 static int 1593 arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1594 void *newp, size_t newlen) 1595 { 1596 int ret; 1597 const char *dss = NULL; 1598 unsigned arena_ind = mib[1]; 1599 dss_prec_t dss_prec_old = dss_prec_limit; 1600 dss_prec_t dss_prec = dss_prec_limit; 1601 1602 malloc_mutex_lock(&ctl_mtx); 1603 WRITE(dss, const char *); 1604 if (dss != NULL) { 1605 int i; 1606 bool match = false; 1607 1608 for (i = 0; i < dss_prec_limit; i++) { 1609 if (strcmp(dss_prec_names[i], dss) == 0) { 1610 dss_prec = i; 1611 match = true; 1612 break; 1613 } 1614 } 1615 1616 if (!match) { 1617 ret = EINVAL; 1618 goto label_return; 1619 } 1620 } 1621 1622 if (arena_ind < ctl_stats.narenas) { 1623 arena_t *arena = arena_get(tsd_fetch(), arena_ind, false, true); 1624 if (arena == NULL || (dss_prec != dss_prec_limit && 1625 arena_dss_prec_set(arena, dss_prec))) { 1626 ret = EFAULT; 1627 goto label_return; 1628 } 1629 dss_prec_old = arena_dss_prec_get(arena); 1630 } else { 1631 if (dss_prec != dss_prec_limit && 1632 chunk_dss_prec_set(dss_prec)) { 1633 ret = EFAULT; 1634 goto label_return; 1635 } 1636 dss_prec_old = chunk_dss_prec_get(); 1637 } 1638 1639 dss = dss_prec_names[dss_prec_old]; 1640 READ(dss, const char *); 1641 1642 ret = 0; 1643 label_return: 1644 malloc_mutex_unlock(&ctl_mtx); 1645 return (ret); 1646 } 1647 1648 static int 1649 arena_i_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp, 1650 size_t *oldlenp, void *newp, size_t newlen) 1651 { 1652 int ret; 1653 unsigned arena_ind = mib[1]; 1654 arena_t *arena; 1655 1656 arena = arena_get(tsd_fetch(), arena_ind, false, true); 1657 if (arena == NULL) { 1658 ret = EFAULT; 1659 goto label_return; 1660 } 1661 1662 if (oldp != NULL && oldlenp != NULL) { 1663 size_t oldval = arena_lg_dirty_mult_get(arena); 1664 READ(oldval, ssize_t); 1665 } 1666 if (newp != NULL) { 1667 if (newlen != sizeof(ssize_t)) { 1668 ret = EINVAL; 1669 goto label_return; 1670 } 1671 if (arena_lg_dirty_mult_set(arena, *(ssize_t *)newp)) { 1672 ret = EFAULT; 1673 goto label_return; 1674 } 1675 } 1676 1677 ret = 0; 1678 label_return: 1679 return (ret); 1680 } 1681 1682 #define CHUNK_FUNC(n) \ 1683 static int \ 1684 arena_i_chunk_##n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ 1685 size_t *oldlenp, void *newp, size_t newlen) \ 1686 { \ 1687 \ 1688 int ret; \ 1689 unsigned arena_ind = mib[1]; \ 1690 arena_t *arena; \ 1691 \ 1692 malloc_mutex_lock(&ctl_mtx); \ 1693 if (arena_ind < narenas_total_get() && (arena = \ 1694 arena_get(tsd_fetch(), arena_ind, false, true)) != NULL) { \ 1695 malloc_mutex_lock(&arena->lock); \ 1696 READ(arena->chunk_##n, chunk_##n##_t *); \ 1697 WRITE(arena->chunk_##n, chunk_##n##_t *); \ 1698 } else { \ 1699 ret = EFAULT; \ 1700 goto label_outer_return; \ 1701 } \ 1702 ret = 0; \ 1703 label_return: \ 1704 malloc_mutex_unlock(&arena->lock); \ 1705 label_outer_return: \ 1706 malloc_mutex_unlock(&ctl_mtx); \ 1707 return (ret); \ 1708 } 1709 CHUNK_FUNC(alloc) 1710 CHUNK_FUNC(dalloc) 1711 CHUNK_FUNC(purge) 1712 #undef CHUNK_FUNC 1713 1714 static const ctl_named_node_t * 1715 arena_i_index(const size_t *mib, size_t miblen, size_t i) 1716 { 1717 const ctl_named_node_t * ret; 1718 1719 malloc_mutex_lock(&ctl_mtx); 1720 if (i > ctl_stats.narenas) { 1721 ret = NULL; 1722 goto label_return; 1723 } 1724 1725 ret = super_arena_i_node; 1726 label_return: 1727 malloc_mutex_unlock(&ctl_mtx); 1728 return (ret); 1729 } 1730 1731 /******************************************************************************/ 1732 1733 static int 1734 arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp, 1735 size_t *oldlenp, void *newp, size_t newlen) 1736 { 1737 int ret; 1738 unsigned narenas; 1739 1740 malloc_mutex_lock(&ctl_mtx); 1741 READONLY(); 1742 if (*oldlenp != sizeof(unsigned)) { 1743 ret = EINVAL; 1744 goto label_return; 1745 } 1746 narenas = ctl_stats.narenas; 1747 READ(narenas, unsigned); 1748 1749 ret = 0; 1750 label_return: 1751 malloc_mutex_unlock(&ctl_mtx); 1752 return (ret); 1753 } 1754 1755 static int 1756 arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, 1757 size_t *oldlenp, void *newp, size_t newlen) 1758 { 1759 int ret; 1760 unsigned nread, i; 1761 1762 malloc_mutex_lock(&ctl_mtx); 1763 READONLY(); 1764 if (*oldlenp != ctl_stats.narenas * sizeof(bool)) { 1765 ret = EINVAL; 1766 nread = (*oldlenp < ctl_stats.narenas * sizeof(bool)) 1767 ? (*oldlenp / sizeof(bool)) : ctl_stats.narenas; 1768 } else { 1769 ret = 0; 1770 nread = ctl_stats.narenas; 1771 } 1772 1773 for (i = 0; i < nread; i++) 1774 ((bool *)oldp)[i] = ctl_stats.arenas[i].initialized; 1775 1776 label_return: 1777 malloc_mutex_unlock(&ctl_mtx); 1778 return (ret); 1779 } 1780 1781 static int 1782 arenas_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp, 1783 size_t *oldlenp, void *newp, size_t newlen) 1784 { 1785 int ret; 1786 1787 if (oldp != NULL && oldlenp != NULL) { 1788 size_t oldval = arena_lg_dirty_mult_default_get(); 1789 READ(oldval, ssize_t); 1790 } 1791 if (newp != NULL) { 1792 if (newlen != sizeof(ssize_t)) { 1793 ret = EINVAL; 1794 goto label_return; 1795 } 1796 if (arena_lg_dirty_mult_default_set(*(ssize_t *)newp)) { 1797 ret = EFAULT; 1798 goto label_return; 1799 } 1800 } 1801 1802 ret = 0; 1803 label_return: 1804 return (ret); 1805 } 1806 1807 CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) 1808 CTL_RO_NL_GEN(arenas_page, PAGE, size_t) 1809 CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) 1810 CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) 1811 CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned) 1812 CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) 1813 CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) 1814 CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) 1815 static const ctl_named_node_t * 1816 arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) 1817 { 1818 1819 if (i > NBINS) 1820 return (NULL); 1821 return (super_arenas_bin_i_node); 1822 } 1823 1824 CTL_RO_NL_GEN(arenas_nlruns, nlclasses, unsigned) 1825 CTL_RO_NL_GEN(arenas_lrun_i_size, index2size(NBINS+mib[2]), size_t) 1826 static const ctl_named_node_t * 1827 arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) 1828 { 1829 1830 if (i > nlclasses) 1831 return (NULL); 1832 return (super_arenas_lrun_i_node); 1833 } 1834 1835 CTL_RO_NL_GEN(arenas_nhchunks, nhclasses, unsigned) 1836 CTL_RO_NL_GEN(arenas_hchunk_i_size, index2size(NBINS+nlclasses+mib[2]), size_t) 1837 static const ctl_named_node_t * 1838 arenas_hchunk_i_index(const size_t *mib, size_t miblen, size_t i) 1839 { 1840 1841 if (i > nhclasses) 1842 return (NULL); 1843 return (super_arenas_hchunk_i_node); 1844 } 1845 1846 static int 1847 arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1848 void *newp, size_t newlen) 1849 { 1850 int ret; 1851 unsigned narenas; 1852 1853 malloc_mutex_lock(&ctl_mtx); 1854 READONLY(); 1855 if (ctl_grow()) { 1856 ret = EAGAIN; 1857 goto label_return; 1858 } 1859 narenas = ctl_stats.narenas - 1; 1860 READ(narenas, unsigned); 1861 1862 ret = 0; 1863 label_return: 1864 malloc_mutex_unlock(&ctl_mtx); 1865 return (ret); 1866 } 1867 1868 /******************************************************************************/ 1869 1870 static int 1871 prof_thread_active_init_ctl(const size_t *mib, size_t miblen, void *oldp, 1872 size_t *oldlenp, void *newp, size_t newlen) 1873 { 1874 int ret; 1875 bool oldval; 1876 1877 if (!config_prof) 1878 return (ENOENT); 1879 1880 if (newp != NULL) { 1881 if (newlen != sizeof(bool)) { 1882 ret = EINVAL; 1883 goto label_return; 1884 } 1885 oldval = prof_thread_active_init_set(*(bool *)newp); 1886 } else 1887 oldval = prof_thread_active_init_get(); 1888 READ(oldval, bool); 1889 1890 ret = 0; 1891 label_return: 1892 return (ret); 1893 } 1894 1895 static int 1896 prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1897 void *newp, size_t newlen) 1898 { 1899 int ret; 1900 bool oldval; 1901 1902 if (!config_prof) 1903 return (ENOENT); 1904 1905 if (newp != NULL) { 1906 if (newlen != sizeof(bool)) { 1907 ret = EINVAL; 1908 goto label_return; 1909 } 1910 oldval = prof_active_set(*(bool *)newp); 1911 } else 1912 oldval = prof_active_get(); 1913 READ(oldval, bool); 1914 1915 ret = 0; 1916 label_return: 1917 return (ret); 1918 } 1919 1920 static int 1921 prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1922 void *newp, size_t newlen) 1923 { 1924 int ret; 1925 const char *filename = NULL; 1926 1927 if (!config_prof) 1928 return (ENOENT); 1929 1930 WRITEONLY(); 1931 WRITE(filename, const char *); 1932 1933 if (prof_mdump(filename)) { 1934 ret = EFAULT; 1935 goto label_return; 1936 } 1937 1938 ret = 0; 1939 label_return: 1940 return (ret); 1941 } 1942 1943 static int 1944 prof_gdump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1945 void *newp, size_t newlen) 1946 { 1947 int ret; 1948 bool oldval; 1949 1950 if (!config_prof) 1951 return (ENOENT); 1952 1953 if (newp != NULL) { 1954 if (newlen != sizeof(bool)) { 1955 ret = EINVAL; 1956 goto label_return; 1957 } 1958 oldval = prof_gdump_set(*(bool *)newp); 1959 } else 1960 oldval = prof_gdump_get(); 1961 READ(oldval, bool); 1962 1963 ret = 0; 1964 label_return: 1965 return (ret); 1966 } 1967 1968 static int 1969 prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, 1970 void *newp, size_t newlen) 1971 { 1972 int ret; 1973 size_t lg_sample = lg_prof_sample; 1974 tsd_t *tsd; 1975 1976 if (!config_prof) 1977 return (ENOENT); 1978 1979 WRITEONLY(); 1980 WRITE(lg_sample, size_t); 1981 if (lg_sample >= (sizeof(uint64_t) << 3)) 1982 lg_sample = (sizeof(uint64_t) << 3) - 1; 1983 1984 tsd = tsd_fetch(); 1985 1986 prof_reset(tsd, lg_sample); 1987 1988 ret = 0; 1989 label_return: 1990 return (ret); 1991 } 1992 1993 CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) 1994 CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) 1995 1996 /******************************************************************************/ 1997 1998 CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) 1999 CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) 2000 CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) 2001 CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t) 2002 CTL_RO_CGEN(config_stats, stats_resident, ctl_stats.resident, size_t) 2003 CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) 2004 2005 CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) 2006 CTL_RO_GEN(stats_arenas_i_lg_dirty_mult, ctl_stats.arenas[mib[2]].lg_dirty_mult, 2007 ssize_t) 2008 CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) 2009 CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) 2010 CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) 2011 CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, 2012 ctl_stats.arenas[mib[2]].astats.mapped, size_t) 2013 CTL_RO_CGEN(config_stats, stats_arenas_i_npurge, 2014 ctl_stats.arenas[mib[2]].astats.npurge, uint64_t) 2015 CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, 2016 ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) 2017 CTL_RO_CGEN(config_stats, stats_arenas_i_purged, 2018 ctl_stats.arenas[mib[2]].astats.purged, uint64_t) 2019 CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_mapped, 2020 ctl_stats.arenas[mib[2]].astats.metadata_mapped, size_t) 2021 CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_allocated, 2022 ctl_stats.arenas[mib[2]].astats.metadata_allocated, size_t) 2023 2024 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, 2025 ctl_stats.arenas[mib[2]].allocated_small, size_t) 2026 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, 2027 ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t) 2028 CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, 2029 ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) 2030 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, 2031 ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) 2032 CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, 2033 ctl_stats.arenas[mib[2]].astats.allocated_large, size_t) 2034 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, 2035 ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) 2036 CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, 2037 ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) 2038 CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, 2039 ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t) 2040 CTL_RO_CGEN(config_stats, stats_arenas_i_huge_allocated, 2041 ctl_stats.arenas[mib[2]].astats.allocated_huge, size_t) 2042 CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nmalloc, 2043 ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) 2044 CTL_RO_CGEN(config_stats, stats_arenas_i_huge_ndalloc, 2045 ctl_stats.arenas[mib[2]].astats.ndalloc_huge, uint64_t) 2046 CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nrequests, 2047 ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) /* Intentional. */ 2048 2049 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, 2050 ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) 2051 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, 2052 ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) 2053 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, 2054 ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) 2055 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, 2056 ctl_stats.arenas[mib[2]].bstats[mib[4]].curregs, size_t) 2057 CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills, 2058 ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) 2059 CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes, 2060 ctl_stats.arenas[mib[2]].bstats[mib[4]].nflushes, uint64_t) 2061 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nruns, 2062 ctl_stats.arenas[mib[2]].bstats[mib[4]].nruns, uint64_t) 2063 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns, 2064 ctl_stats.arenas[mib[2]].bstats[mib[4]].reruns, uint64_t) 2065 CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns, 2066 ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) 2067 2068 static const ctl_named_node_t * 2069 stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) 2070 { 2071 2072 if (j > NBINS) 2073 return (NULL); 2074 return (super_stats_arenas_i_bins_j_node); 2075 } 2076 2077 CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nmalloc, 2078 ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t) 2079 CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_ndalloc, 2080 ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t) 2081 CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nrequests, 2082 ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t) 2083 CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns, 2084 ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) 2085 2086 static const ctl_named_node_t * 2087 stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) 2088 { 2089 2090 if (j > nlclasses) 2091 return (NULL); 2092 return (super_stats_arenas_i_lruns_j_node); 2093 } 2094 2095 CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nmalloc, 2096 ctl_stats.arenas[mib[2]].hstats[mib[4]].nmalloc, uint64_t) 2097 CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_ndalloc, 2098 ctl_stats.arenas[mib[2]].hstats[mib[4]].ndalloc, uint64_t) 2099 CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nrequests, 2100 ctl_stats.arenas[mib[2]].hstats[mib[4]].nmalloc, /* Intentional. */ 2101 uint64_t) 2102 CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_curhchunks, 2103 ctl_stats.arenas[mib[2]].hstats[mib[4]].curhchunks, size_t) 2104 2105 static const ctl_named_node_t * 2106 stats_arenas_i_hchunks_j_index(const size_t *mib, size_t miblen, size_t j) 2107 { 2108 2109 if (j > nhclasses) 2110 return (NULL); 2111 return (super_stats_arenas_i_hchunks_j_node); 2112 } 2113 2114 static const ctl_named_node_t * 2115 stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) 2116 { 2117 const ctl_named_node_t * ret; 2118 2119 malloc_mutex_lock(&ctl_mtx); 2120 if (i > ctl_stats.narenas || !ctl_stats.arenas[i].initialized) { 2121 ret = NULL; 2122 goto label_return; 2123 } 2124 2125 ret = super_stats_arenas_i_node; 2126 label_return: 2127 malloc_mutex_unlock(&ctl_mtx); 2128 return (ret); 2129 } 2130
