Home | History | Annotate | Download | only in src
      1 /*
      2  * Copyright (C) 2016 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 static pthread_mutex_t malloc_disabled_lock = PTHREAD_MUTEX_INITIALIZER;
     18 static bool malloc_disabled_tcache;
     19 
     20 static void je_iterate_chunk(arena_chunk_t *chunk,
     21     void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg);
     22 static void je_iterate_small(arena_run_t *run,
     23     void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg);
     24 
     25 /* je_iterate calls callback for each allocation found in the memory region
     26  * between [base, base+size).  base will be rounded down to by the jemalloc
     27  * chunk size, and base+size will be rounded up to the chunk size.  If no memory
     28  * managed by jemalloc is found in the requested region, je_iterate returns -1
     29  * and sets errno to EINVAL.
     30  *
     31  * je_iterate must be called when no allocations are in progress, either
     32  * when single-threaded (for example just after a fork), or between
     33  * jemalloc_prefork() and jemalloc_postfork_parent().  The callback must
     34  * not attempt to allocate with jemalloc.
     35  */
     36 int je_iterate(uintptr_t base, size_t size,
     37     void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) {
     38 
     39   int error = EINVAL;
     40   uintptr_t ptr = (uintptr_t)CHUNK_ADDR2BASE(base);
     41   uintptr_t end = CHUNK_CEILING(base + size);
     42 
     43   while (ptr < end) {
     44     assert(ptr == (uintptr_t)CHUNK_ADDR2BASE(ptr));
     45     extent_node_t *node;
     46 
     47     node = chunk_lookup((void *)ptr, false);
     48     if (node == NULL) {
     49       ptr += chunksize;
     50       continue;
     51     }
     52 
     53     assert(extent_node_achunk_get(node) ||
     54         (uintptr_t)extent_node_addr_get(node) == ptr);
     55 
     56     error = 0;
     57     if (extent_node_achunk_get(node)) {
     58       /* Chunk */
     59       arena_chunk_t *chunk = (arena_chunk_t *)ptr;
     60       ptr += chunksize;
     61 
     62       if (&chunk->node != node) {
     63           /* Empty retained chunk */
     64           continue;
     65       }
     66 
     67       je_iterate_chunk(chunk, callback, arg);
     68     } else if ((uintptr_t)extent_node_addr_get(node) == ptr) {
     69       /* Huge allocation */
     70       callback(ptr, extent_node_size_get(node), arg);
     71       ptr = CHUNK_CEILING(ptr + extent_node_size_get(node));
     72     }
     73   }
     74 
     75   if (error) {
     76     set_errno(error);
     77     return -1;
     78   }
     79 
     80   return 0;
     81 }
     82 
     83 /* Iterate over a valid jemalloc chunk, calling callback for each large
     84  * allocation run, and calling je_iterate_small for each small allocation run */
     85 static void je_iterate_chunk(arena_chunk_t *chunk,
     86     void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) {
     87   size_t pageind;
     88 
     89   pageind = map_bias;
     90 
     91   while (pageind < chunk_npages) {
     92     size_t mapbits;
     93     size_t size;
     94 
     95     mapbits = arena_mapbits_get(chunk, pageind);
     96     if (!arena_mapbits_allocated_get(chunk, pageind)) {
     97       /* Unallocated run */
     98       size = arena_mapbits_unallocated_size_get(chunk, pageind);
     99     } else if (arena_mapbits_large_get(chunk, pageind)) {
    100       /* Large allocation run */
    101       void *rpages;
    102 
    103       size = arena_mapbits_large_size_get(chunk, pageind);
    104       rpages = arena_miscelm_to_rpages(arena_miscelm_get(chunk, pageind));
    105       callback((uintptr_t)rpages, size, arg);
    106     } else {
    107       /* Run of small allocations */
    108       szind_t binind;
    109       arena_run_t *run;
    110 
    111       assert(arena_mapbits_small_runind_get(chunk, pageind) == pageind);
    112       binind = arena_mapbits_binind_get(chunk, pageind);
    113       run = &arena_miscelm_get(chunk, pageind)->run;
    114       assert(run->binind == binind);
    115       size = arena_bin_info[binind].run_size;
    116 
    117       je_iterate_small(run, callback, arg);
    118     }
    119     assert(size == PAGE_CEILING(size));
    120     assert(size > 0);
    121     pageind += size >> LG_PAGE;
    122   }
    123 
    124 }
    125 
    126 /* Iterate over a valid jemalloc small allocation run, calling callback for each
    127  * active allocation. */
    128 static void je_iterate_small(arena_run_t *run,
    129     void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg) {
    130   szind_t binind;
    131   const arena_bin_info_t *bin_info;
    132   uint32_t regind;
    133   uintptr_t ptr;
    134   void *rpages;
    135 
    136   binind = run->binind;
    137   bin_info = &arena_bin_info[binind];
    138   rpages = arena_miscelm_to_rpages(arena_run_to_miscelm(run));
    139   ptr = (uintptr_t)rpages + bin_info->reg0_offset;
    140 
    141   for (regind = 0; regind < bin_info->nregs; regind++) {
    142     if (bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)) {
    143       callback(ptr, bin_info->reg_size, arg);
    144     }
    145     ptr += bin_info->reg_interval;
    146   }
    147 }
    148 
    149 static void je_malloc_disable_prefork() {
    150   pthread_mutex_lock(&malloc_disabled_lock);
    151 }
    152 
    153 static void je_malloc_disable_postfork_parent() {
    154   pthread_mutex_unlock(&malloc_disabled_lock);
    155 }
    156 
    157 static void je_malloc_disable_postfork_child() {
    158   pthread_mutex_init(&malloc_disabled_lock, NULL);
    159 }
    160 
    161 void je_malloc_disable_init() {
    162   if (pthread_atfork(je_malloc_disable_prefork,
    163       je_malloc_disable_postfork_parent, je_malloc_disable_postfork_child) != 0) {
    164     malloc_write("<jemalloc>: Error in pthread_atfork()\n");
    165     if (opt_abort)
    166       abort();
    167   }
    168 }
    169 
    170 void je_malloc_disable() {
    171   static pthread_once_t once_control = PTHREAD_ONCE_INIT;
    172   pthread_once(&once_control, je_malloc_disable_init);
    173 
    174   pthread_mutex_lock(&malloc_disabled_lock);
    175   bool new_tcache = false;
    176   size_t old_len = sizeof(malloc_disabled_tcache);
    177   je_mallctl("thread.tcache.enabled",
    178       &malloc_disabled_tcache, &old_len,
    179       &new_tcache, sizeof(new_tcache));
    180   jemalloc_prefork();
    181 }
    182 
    183 void je_malloc_enable() {
    184   jemalloc_postfork_parent();
    185   if (malloc_disabled_tcache) {
    186     je_mallctl("thread.tcache.enabled", NULL, NULL,
    187         &malloc_disabled_tcache, sizeof(malloc_disabled_tcache));
    188   }
    189   pthread_mutex_unlock(&malloc_disabled_lock);
    190 }
    191