1 #define JEMALLOC_QUARANTINE_C_ 2 #include "jemalloc/internal/jemalloc_internal.h" 3 4 /* 5 * Quarantine pointers close to NULL are used to encode state information that 6 * is used for cleaning up during thread shutdown. 7 */ 8 #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) 9 #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) 10 #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY 11 12 /******************************************************************************/ 13 /* Function prototypes for non-inline static functions. */ 14 15 static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine); 16 static void quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine); 17 static void quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine, 18 size_t upper_bound); 19 20 /******************************************************************************/ 21 22 static quarantine_t * 23 quarantine_init(tsdn_t *tsdn, size_t lg_maxobjs) 24 { 25 quarantine_t *quarantine; 26 size_t size; 27 28 size = offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) * 29 sizeof(quarantine_obj_t)); 30 quarantine = (quarantine_t *)iallocztm(tsdn, size, size2index(size), 31 false, NULL, true, arena_get(TSDN_NULL, 0, true), true); 32 if (quarantine == NULL) 33 return (NULL); 34 quarantine->curbytes = 0; 35 quarantine->curobjs = 0; 36 quarantine->first = 0; 37 quarantine->lg_maxobjs = lg_maxobjs; 38 39 return (quarantine); 40 } 41 42 void 43 quarantine_alloc_hook_work(tsd_t *tsd) 44 { 45 quarantine_t *quarantine; 46 47 if (!tsd_nominal(tsd)) 48 return; 49 50 quarantine = quarantine_init(tsd_tsdn(tsd), LG_MAXOBJS_INIT); 51 /* 52 * Check again whether quarantine has been initialized, because 53 * quarantine_init() may have triggered recursive initialization. 54 */ 55 if (tsd_quarantine_get(tsd) == NULL) 56 tsd_quarantine_set(tsd, quarantine); 57 else 58 idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true); 59 } 60 61 static quarantine_t * 62 quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) 63 { 64 quarantine_t *ret; 65 66 ret = quarantine_init(tsd_tsdn(tsd), quarantine->lg_maxobjs + 1); 67 if (ret == NULL) { 68 quarantine_drain_one(tsd_tsdn(tsd), quarantine); 69 return (quarantine); 70 } 71 72 ret->curbytes = quarantine->curbytes; 73 ret->curobjs = quarantine->curobjs; 74 if (quarantine->first + quarantine->curobjs <= (ZU(1) << 75 quarantine->lg_maxobjs)) { 76 /* objs ring buffer data are contiguous. */ 77 memcpy(ret->objs, &quarantine->objs[quarantine->first], 78 quarantine->curobjs * sizeof(quarantine_obj_t)); 79 } else { 80 /* objs ring buffer data wrap around. */ 81 size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - 82 quarantine->first; 83 size_t ncopy_b = quarantine->curobjs - ncopy_a; 84 85 memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a 86 * sizeof(quarantine_obj_t)); 87 memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * 88 sizeof(quarantine_obj_t)); 89 } 90 idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true); 91 92 tsd_quarantine_set(tsd, ret); 93 return (ret); 94 } 95 96 static void 97 quarantine_drain_one(tsdn_t *tsdn, quarantine_t *quarantine) 98 { 99 quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; 100 assert(obj->usize == isalloc(tsdn, obj->ptr, config_prof)); 101 idalloctm(tsdn, obj->ptr, NULL, false, true); 102 quarantine->curbytes -= obj->usize; 103 quarantine->curobjs--; 104 quarantine->first = (quarantine->first + 1) & ((ZU(1) << 105 quarantine->lg_maxobjs) - 1); 106 } 107 108 static void 109 quarantine_drain(tsdn_t *tsdn, quarantine_t *quarantine, size_t upper_bound) 110 { 111 112 while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) 113 quarantine_drain_one(tsdn, quarantine); 114 } 115 116 void 117 quarantine(tsd_t *tsd, void *ptr) 118 { 119 quarantine_t *quarantine; 120 size_t usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); 121 122 cassert(config_fill); 123 assert(opt_quarantine); 124 125 if ((quarantine = tsd_quarantine_get(tsd)) == NULL) { 126 idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true); 127 return; 128 } 129 /* 130 * Drain one or more objects if the quarantine size limit would be 131 * exceeded by appending ptr. 132 */ 133 if (quarantine->curbytes + usize > opt_quarantine) { 134 size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine 135 - usize : 0; 136 quarantine_drain(tsd_tsdn(tsd), quarantine, upper_bound); 137 } 138 /* Grow the quarantine ring buffer if it's full. */ 139 if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) 140 quarantine = quarantine_grow(tsd, quarantine); 141 /* quarantine_grow() must free a slot if it fails to grow. */ 142 assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); 143 /* Append ptr if its size doesn't exceed the quarantine size. */ 144 if (quarantine->curbytes + usize <= opt_quarantine) { 145 size_t offset = (quarantine->first + quarantine->curobjs) & 146 ((ZU(1) << quarantine->lg_maxobjs) - 1); 147 quarantine_obj_t *obj = &quarantine->objs[offset]; 148 obj->ptr = ptr; 149 obj->usize = usize; 150 quarantine->curbytes += usize; 151 quarantine->curobjs++; 152 if (config_fill && unlikely(opt_junk_free)) { 153 /* 154 * Only do redzone validation if Valgrind isn't in 155 * operation. 156 */ 157 if ((!config_valgrind || likely(!in_valgrind)) 158 && usize <= SMALL_MAXCLASS) 159 arena_quarantine_junk_small(ptr, usize); 160 else 161 memset(ptr, JEMALLOC_FREE_JUNK, usize); 162 } 163 } else { 164 assert(quarantine->curbytes == 0); 165 idalloctm(tsd_tsdn(tsd), ptr, NULL, false, true); 166 } 167 } 168 169 void 170 quarantine_cleanup(tsd_t *tsd) 171 { 172 quarantine_t *quarantine; 173 174 if (!config_fill) 175 return; 176 177 quarantine = tsd_quarantine_get(tsd); 178 if (quarantine != NULL) { 179 quarantine_drain(tsd_tsdn(tsd), quarantine, 0); 180 idalloctm(tsd_tsdn(tsd), quarantine, NULL, true, true); 181 tsd_quarantine_set(tsd, NULL); 182 } 183 } 184
