Home | History | Annotate | Download | only in src 
      1 #define	JEMALLOC_QUARANTINE_C_
      2 #include "jemalloc/internal/jemalloc_internal.h"
      3 
      4 /*
      5  * quarantine pointers close to NULL are used to encode state information that
      6  * is used for cleaning up during thread shutdown.
      7  */
      8 #define	QUARANTINE_STATE_REINCARNATED	((quarantine_t *)(uintptr_t)1)
      9 #define	QUARANTINE_STATE_PURGATORY	((quarantine_t *)(uintptr_t)2)
     10 #define	QUARANTINE_STATE_MAX		QUARANTINE_STATE_PURGATORY
     11 
     12 /******************************************************************************/
     13 /* Data. */
     14 
     15 malloc_tsd_data(, quarantine, quarantine_t *, NULL)
     16 
     17 /******************************************************************************/
     18 /* Function prototypes for non-inline static functions. */
     19 
     20 static quarantine_t	*quarantine_grow(quarantine_t *quarantine);
     21 static void	quarantine_drain_one(quarantine_t *quarantine);
     22 static void	quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
     23 
     24 /******************************************************************************/
     25 
     26 quarantine_t *
     27 quarantine_init(size_t lg_maxobjs)
     28 {
     29 	quarantine_t *quarantine;
     30 
     31 	quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
     32 	    ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
     33 	if (quarantine == NULL)
     34 		return (NULL);
     35 	quarantine->curbytes = 0;
     36 	quarantine->curobjs = 0;
     37 	quarantine->first = 0;
     38 	quarantine->lg_maxobjs = lg_maxobjs;
     39 
     40 	quarantine_tsd_set(&quarantine);
     41 
     42 	return (quarantine);
     43 }
     44 
     45 static quarantine_t *
     46 quarantine_grow(quarantine_t *quarantine)
     47 {
     48 	quarantine_t *ret;
     49 
     50 	ret = quarantine_init(quarantine->lg_maxobjs + 1);
     51 	if (ret == NULL) {
     52 		quarantine_drain_one(quarantine);
     53 		return (quarantine);
     54 	}
     55 
     56 	ret->curbytes = quarantine->curbytes;
     57 	ret->curobjs = quarantine->curobjs;
     58 	if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
     59 	    quarantine->lg_maxobjs)) {
     60 		/* objs ring buffer data are contiguous. */
     61 		memcpy(ret->objs, &quarantine->objs[quarantine->first],
     62 		    quarantine->curobjs * sizeof(quarantine_obj_t));
     63 	} else {
     64 		/* objs ring buffer data wrap around. */
     65 		size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
     66 		    quarantine->first;
     67 		size_t ncopy_b = quarantine->curobjs - ncopy_a;
     68 
     69 		memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
     70 		    * sizeof(quarantine_obj_t));
     71 		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
     72 		    sizeof(quarantine_obj_t));
     73 	}
     74 	idalloc(quarantine);
     75 
     76 	return (ret);
     77 }
     78 
     79 static void
     80 quarantine_drain_one(quarantine_t *quarantine)
     81 {
     82 	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
     83 	assert(obj->usize == isalloc(obj->ptr, config_prof));
     84 	idalloc(obj->ptr);
     85 	quarantine->curbytes -= obj->usize;
     86 	quarantine->curobjs--;
     87 	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
     88 	    quarantine->lg_maxobjs) - 1);
     89 }
     90 
     91 static void
     92 quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
     93 {
     94 
     95 	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
     96 		quarantine_drain_one(quarantine);
     97 }
     98 
     99 void
    100 quarantine(void *ptr)
    101 {
    102 	quarantine_t *quarantine;
    103 	size_t usize = isalloc(ptr, config_prof);
    104 
    105 	cassert(config_fill);
    106 	assert(opt_quarantine);
    107 
    108 	quarantine = *quarantine_tsd_get();
    109 	if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
    110 		if (quarantine == QUARANTINE_STATE_PURGATORY) {
    111 			/*
    112 			 * Make a note that quarantine() was called after
    113 			 * quarantine_cleanup() was called.
    114 			 */
    115 			quarantine = QUARANTINE_STATE_REINCARNATED;
    116 			quarantine_tsd_set(&quarantine);
    117 		}
    118 		idalloc(ptr);
    119 		return;
    120 	}
    121 	/*
    122 	 * Drain one or more objects if the quarantine size limit would be
    123 	 * exceeded by appending ptr.
    124 	 */
    125 	if (quarantine->curbytes + usize > opt_quarantine) {
    126 		size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
    127 		    - usize : 0;
    128 		quarantine_drain(quarantine, upper_bound);
    129 	}
    130 	/* Grow the quarantine ring buffer if it's full. */
    131 	if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
    132 		quarantine = quarantine_grow(quarantine);
    133 	/* quarantine_grow() must free a slot if it fails to grow. */
    134 	assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
    135 	/* Append ptr if its size doesn't exceed the quarantine size. */
    136 	if (quarantine->curbytes + usize <= opt_quarantine) {
    137 		size_t offset = (quarantine->first + quarantine->curobjs) &
    138 		    ((ZU(1) << quarantine->lg_maxobjs) - 1);
    139 		quarantine_obj_t *obj = &quarantine->objs[offset];
    140 		obj->ptr = ptr;
    141 		obj->usize = usize;
    142 		quarantine->curbytes += usize;
    143 		quarantine->curobjs++;
    144 		if (config_fill && opt_junk) {
    145 			/*
    146 			 * Only do redzone validation if Valgrind isn't in
    147 			 * operation.
    148 			 */
    149 			if ((config_valgrind == false || in_valgrind == false)
    150 			    && usize <= SMALL_MAXCLASS)
    151 				arena_quarantine_junk_small(ptr, usize);
    152 			else
    153 				memset(ptr, 0x5a, usize);
    154 		}
    155 	} else {
    156 		assert(quarantine->curbytes == 0);
    157 		idalloc(ptr);
    158 	}
    159 }
    160 
    161 void
    162 quarantine_cleanup(void *arg)
    163 {
    164 	quarantine_t *quarantine = *(quarantine_t **)arg;
    165 
    166 	if (quarantine == QUARANTINE_STATE_REINCARNATED) {
    167 		/*
    168 		 * Another destructor deallocated memory after this destructor
    169 		 * was called.  Reset quarantine to QUARANTINE_STATE_PURGATORY
    170 		 * in order to receive another callback.
    171 		 */
    172 		quarantine = QUARANTINE_STATE_PURGATORY;
    173 		quarantine_tsd_set(&quarantine);
    174 	} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
    175 		/*
    176 		 * The previous time this destructor was called, we set the key
    177 		 * to QUARANTINE_STATE_PURGATORY so that other destructors
    178 		 * wouldn't cause re-creation of the quarantine.  This time, do
    179 		 * nothing, so that the destructor will not be called again.
    180 		 */
    181 	} else if (quarantine != NULL) {
    182 		quarantine_drain(quarantine, 0);
    183 		idalloc(quarantine);
    184 		quarantine = QUARANTINE_STATE_PURGATORY;
    185 		quarantine_tsd_set(&quarantine);
    186 	}
    187 }
    188 
    189 bool
    190 quarantine_boot(void)
    191 {
    192 
    193 	cassert(config_fill);
    194 
    195 	if (quarantine_tsd_boot())
    196 		return (true);
    197 
    198 	return (false);
    199 }
    200