1 #define JEMALLOC_CHUNK_DSS_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 /******************************************************************************/ 4 /* Data. */ 5 6 const char *dss_prec_names[] = { 7 "disabled", 8 "primary", 9 "secondary", 10 "N/A" 11 }; 12 13 /* 14 * Current dss precedence default, used when creating new arenas. NB: This is 15 * stored as unsigned rather than dss_prec_t because in principle there's no 16 * guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use 17 * atomic operations to synchronize the setting. 18 */ 19 static unsigned dss_prec_default = (unsigned)DSS_PREC_DEFAULT; 20 21 /* Base address of the DSS. */ 22 static void *dss_base; 23 /* Atomic boolean indicating whether the DSS is exhausted. */ 24 static unsigned dss_exhausted; 25 /* Atomic current upper limit on DSS addresses. */ 26 static void *dss_max; 27 28 /******************************************************************************/ 29 30 static void * 31 chunk_dss_sbrk(intptr_t increment) 32 { 33 34 #ifdef JEMALLOC_DSS 35 return (sbrk(increment)); 36 #else 37 not_implemented(); 38 return (NULL); 39 #endif 40 } 41 42 dss_prec_t 43 chunk_dss_prec_get(void) 44 { 45 dss_prec_t ret; 46 47 if (!have_dss) 48 return (dss_prec_disabled); 49 ret = (dss_prec_t)atomic_read_u(&dss_prec_default); 50 return (ret); 51 } 52 53 bool 54 chunk_dss_prec_set(dss_prec_t dss_prec) 55 { 56 57 if (!have_dss) 58 return (dss_prec != dss_prec_disabled); 59 atomic_write_u(&dss_prec_default, (unsigned)dss_prec); 60 return (false); 61 } 62 63 static void * 64 chunk_dss_max_update(void *new_addr) 65 { 66 void *max_cur; 67 spin_t spinner; 68 69 /* 70 * Get the current end of the DSS as max_cur and assure that dss_max is 71 * up to date. 72 */ 73 spin_init(&spinner); 74 while (true) { 75 void *max_prev = atomic_read_p(&dss_max); 76 77 max_cur = chunk_dss_sbrk(0); 78 if ((uintptr_t)max_prev > (uintptr_t)max_cur) { 79 /* 80 * Another thread optimistically updated dss_max. Wait 81 * for it to finish. 82 */ 83 spin_adaptive(&spinner); 84 continue; 85 } 86 if (!atomic_cas_p(&dss_max, max_prev, max_cur)) 87 break; 88 } 89 /* Fixed new_addr can only be supported if it is at the edge of DSS. */ 90 if (new_addr != NULL && max_cur != new_addr) 91 return (NULL); 92 93 return (max_cur); 94 } 95 96 void * 97 chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, 98 size_t alignment, bool *zero, bool *commit) 99 { 100 cassert(have_dss); 101 assert(size > 0 && (size & chunksize_mask) == 0); 102 assert(alignment > 0 && (alignment & chunksize_mask) == 0); 103 104 /* 105 * sbrk() uses a signed increment argument, so take care not to 106 * interpret a huge allocation request as a negative increment. 107 */ 108 if ((intptr_t)size < 0) 109 return (NULL); 110 111 if (!atomic_read_u(&dss_exhausted)) { 112 /* 113 * The loop is necessary to recover from races with other 114 * threads that are using the DSS for something other than 115 * malloc. 116 */ 117 while (true) { 118 void *ret, *cpad, *max_cur, *dss_next, *dss_prev; 119 size_t gap_size, cpad_size; 120 intptr_t incr; 121 122 max_cur = chunk_dss_max_update(new_addr); 123 if (max_cur == NULL) 124 goto label_oom; 125 126 /* 127 * Calculate how much padding is necessary to 128 * chunk-align the end of the DSS. 129 */ 130 gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) & 131 chunksize_mask; 132 /* 133 * Compute how much chunk-aligned pad space (if any) is 134 * necessary to satisfy alignment. This space can be 135 * recycled for later use. 136 */ 137 cpad = (void *)((uintptr_t)dss_max + gap_size); 138 ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, 139 alignment); 140 cpad_size = (uintptr_t)ret - (uintptr_t)cpad; 141 dss_next = (void *)((uintptr_t)ret + size); 142 if ((uintptr_t)ret < (uintptr_t)dss_max || 143 (uintptr_t)dss_next < (uintptr_t)dss_max) 144 goto label_oom; /* Wrap-around. */ 145 incr = gap_size + cpad_size + size; 146 147 /* 148 * Optimistically update dss_max, and roll back below if 149 * sbrk() fails. No other thread will try to extend the 150 * DSS while dss_max is greater than the current DSS 151 * max reported by sbrk(0). 152 */ 153 if (atomic_cas_p(&dss_max, max_cur, dss_next)) 154 continue; 155 156 /* Try to allocate. */ 157 dss_prev = chunk_dss_sbrk(incr); 158 if (dss_prev == max_cur) { 159 /* Success. */ 160 if (cpad_size != 0) { 161 chunk_hooks_t chunk_hooks = 162 CHUNK_HOOKS_INITIALIZER; 163 chunk_dalloc_wrapper(tsdn, arena, 164 &chunk_hooks, cpad, cpad_size, 165 arena_extent_sn_next(arena), false, 166 true); 167 } 168 if (*zero) { 169 JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( 170 ret, size); 171 memset(ret, 0, size); 172 } 173 if (!*commit) 174 *commit = pages_decommit(ret, size); 175 return (ret); 176 } 177 178 /* 179 * Failure, whether due to OOM or a race with a raw 180 * sbrk() call from outside the allocator. Try to roll 181 * back optimistic dss_max update; if rollback fails, 182 * it's due to another caller of this function having 183 * succeeded since this invocation started, in which 184 * case rollback is not necessary. 185 */ 186 atomic_cas_p(&dss_max, dss_next, max_cur); 187 if (dss_prev == (void *)-1) { 188 /* OOM. */ 189 atomic_write_u(&dss_exhausted, (unsigned)true); 190 goto label_oom; 191 } 192 } 193 } 194 label_oom: 195 return (NULL); 196 } 197 198 static bool 199 chunk_in_dss_helper(void *chunk, void *max) 200 { 201 202 return ((uintptr_t)chunk >= (uintptr_t)dss_base && (uintptr_t)chunk < 203 (uintptr_t)max); 204 } 205 206 bool 207 chunk_in_dss(void *chunk) 208 { 209 210 cassert(have_dss); 211 212 return (chunk_in_dss_helper(chunk, atomic_read_p(&dss_max))); 213 } 214 215 bool 216 chunk_dss_mergeable(void *chunk_a, void *chunk_b) 217 { 218 void *max; 219 220 cassert(have_dss); 221 222 max = atomic_read_p(&dss_max); 223 return (chunk_in_dss_helper(chunk_a, max) == 224 chunk_in_dss_helper(chunk_b, max)); 225 } 226 227 void 228 chunk_dss_boot(void) 229 { 230 231 cassert(have_dss); 232 233 dss_base = chunk_dss_sbrk(0); 234 dss_exhausted = (unsigned)(dss_base == (void *)-1); 235 dss_max = dss_base; 236 } 237 238 /******************************************************************************/ 239
