1 /******************************************************************************/ 2 #ifdef JEMALLOC_H_TYPES 3 4 typedef struct witness_s witness_t; 5 typedef unsigned witness_rank_t; 6 typedef ql_head(witness_t) witness_list_t; 7 typedef int witness_comp_t (const witness_t *, const witness_t *); 8 9 /* 10 * Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by 11 * the witness machinery. 12 */ 13 #define WITNESS_RANK_OMIT 0U 14 15 #define WITNESS_RANK_INIT 1U 16 #define WITNESS_RANK_CTL 1U 17 #define WITNESS_RANK_TCACHES 2U 18 #define WITNESS_RANK_ARENAS 3U 19 20 #define WITNESS_RANK_PROF_DUMP 4U 21 #define WITNESS_RANK_PROF_BT2GCTX 5U 22 #define WITNESS_RANK_PROF_TDATAS 6U 23 #define WITNESS_RANK_PROF_TDATA 7U 24 #define WITNESS_RANK_PROF_GCTX 8U 25 26 #define WITNESS_RANK_ARENA 9U 27 #define WITNESS_RANK_ARENA_CHUNKS 10U 28 #define WITNESS_RANK_ARENA_NODE_CACHE 11U 29 30 #define WITNESS_RANK_BASE 12U 31 32 #define WITNESS_RANK_LEAF 0xffffffffU 33 #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF 34 #define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF 35 #define WITNESS_RANK_DSS WITNESS_RANK_LEAF 36 #define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF 37 #define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF 38 #define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF 39 #define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF 40 #define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF 41 42 #define WITNESS_INITIALIZER(rank) {"initializer", rank, NULL, {NULL, NULL}} 43 44 #endif /* JEMALLOC_H_TYPES */ 45 /******************************************************************************/ 46 #ifdef JEMALLOC_H_STRUCTS 47 48 struct witness_s { 49 /* Name, used for printing lock order reversal messages. */ 50 const char *name; 51 52 /* 53 * Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses 54 * must be acquired in order of increasing rank. 55 */ 56 witness_rank_t rank; 57 58 /* 59 * If two witnesses are of equal rank and they have the samp comp 60 * function pointer, it is called as a last attempt to differentiate 61 * between witnesses of equal rank. 62 */ 63 witness_comp_t *comp; 64 65 /* Linkage for thread's currently owned locks. */ 66 ql_elm(witness_t) link; 67 }; 68 69 #endif /* JEMALLOC_H_STRUCTS */ 70 /******************************************************************************/ 71 #ifdef JEMALLOC_H_EXTERNS 72 73 void witness_init(witness_t *witness, const char *name, witness_rank_t rank, 74 witness_comp_t *comp); 75 #ifdef JEMALLOC_JET 76 typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); 77 extern witness_lock_error_t *witness_lock_error; 78 #else 79 void witness_lock_error(const witness_list_t *witnesses, 80 const witness_t *witness); 81 #endif 82 #ifdef JEMALLOC_JET 83 typedef void (witness_owner_error_t)(const witness_t *); 84 extern witness_owner_error_t *witness_owner_error; 85 #else 86 void witness_owner_error(const witness_t *witness); 87 #endif 88 #ifdef JEMALLOC_JET 89 typedef void (witness_not_owner_error_t)(const witness_t *); 90 extern witness_not_owner_error_t *witness_not_owner_error; 91 #else 92 void witness_not_owner_error(const witness_t *witness); 93 #endif 94 #ifdef JEMALLOC_JET 95 typedef void (witness_lockless_error_t)(const witness_list_t *); 96 extern witness_lockless_error_t *witness_lockless_error; 97 #else 98 void witness_lockless_error(const witness_list_t *witnesses); 99 #endif 100 101 void witnesses_cleanup(tsd_t *tsd); 102 void witness_fork_cleanup(tsd_t *tsd); 103 void witness_prefork(tsd_t *tsd); 104 void witness_postfork_parent(tsd_t *tsd); 105 void witness_postfork_child(tsd_t *tsd); 106 107 #endif /* JEMALLOC_H_EXTERNS */ 108 /******************************************************************************/ 109 #ifdef JEMALLOC_H_INLINES 110 111 #ifndef JEMALLOC_ENABLE_INLINE 112 bool witness_owner(tsd_t *tsd, const witness_t *witness); 113 void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); 114 void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); 115 void witness_assert_lockless(tsdn_t *tsdn); 116 void witness_lock(tsdn_t *tsdn, witness_t *witness); 117 void witness_unlock(tsdn_t *tsdn, witness_t *witness); 118 #endif 119 120 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) 121 JEMALLOC_INLINE bool 122 witness_owner(tsd_t *tsd, const witness_t *witness) 123 { 124 witness_list_t *witnesses; 125 witness_t *w; 126 127 witnesses = tsd_witnessesp_get(tsd); 128 ql_foreach(w, witnesses, link) { 129 if (w == witness) 130 return (true); 131 } 132 133 return (false); 134 } 135 136 JEMALLOC_INLINE void 137 witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) 138 { 139 tsd_t *tsd; 140 141 if (!config_debug) 142 return; 143 144 if (tsdn_null(tsdn)) 145 return; 146 tsd = tsdn_tsd(tsdn); 147 if (witness->rank == WITNESS_RANK_OMIT) 148 return; 149 150 if (witness_owner(tsd, witness)) 151 return; 152 witness_owner_error(witness); 153 } 154 155 JEMALLOC_INLINE void 156 witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) 157 { 158 tsd_t *tsd; 159 witness_list_t *witnesses; 160 witness_t *w; 161 162 if (!config_debug) 163 return; 164 165 if (tsdn_null(tsdn)) 166 return; 167 tsd = tsdn_tsd(tsdn); 168 if (witness->rank == WITNESS_RANK_OMIT) 169 return; 170 171 witnesses = tsd_witnessesp_get(tsd); 172 ql_foreach(w, witnesses, link) { 173 if (w == witness) 174 witness_not_owner_error(witness); 175 } 176 } 177 178 JEMALLOC_INLINE void 179 witness_assert_lockless(tsdn_t *tsdn) 180 { 181 tsd_t *tsd; 182 witness_list_t *witnesses; 183 witness_t *w; 184 185 if (!config_debug) 186 return; 187 188 if (tsdn_null(tsdn)) 189 return; 190 tsd = tsdn_tsd(tsdn); 191 192 witnesses = tsd_witnessesp_get(tsd); 193 w = ql_last(witnesses, link); 194 if (w != NULL) 195 witness_lockless_error(witnesses); 196 } 197 198 JEMALLOC_INLINE void 199 witness_lock(tsdn_t *tsdn, witness_t *witness) 200 { 201 tsd_t *tsd; 202 witness_list_t *witnesses; 203 witness_t *w; 204 205 if (!config_debug) 206 return; 207 208 if (tsdn_null(tsdn)) 209 return; 210 tsd = tsdn_tsd(tsdn); 211 if (witness->rank == WITNESS_RANK_OMIT) 212 return; 213 214 witness_assert_not_owner(tsdn, witness); 215 216 witnesses = tsd_witnessesp_get(tsd); 217 w = ql_last(witnesses, link); 218 if (w == NULL) { 219 /* No other locks; do nothing. */ 220 } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) { 221 /* Forking, and relaxed ranking satisfied. */ 222 } else if (w->rank > witness->rank) { 223 /* Not forking, rank order reversal. */ 224 witness_lock_error(witnesses, witness); 225 } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != 226 witness->comp || w->comp(w, witness) > 0)) { 227 /* 228 * Missing/incompatible comparison function, or comparison 229 * function indicates rank order reversal. 230 */ 231 witness_lock_error(witnesses, witness); 232 } 233 234 ql_elm_new(witness, link); 235 ql_tail_insert(witnesses, witness, link); 236 } 237 238 JEMALLOC_INLINE void 239 witness_unlock(tsdn_t *tsdn, witness_t *witness) 240 { 241 tsd_t *tsd; 242 witness_list_t *witnesses; 243 244 if (!config_debug) 245 return; 246 247 if (tsdn_null(tsdn)) 248 return; 249 tsd = tsdn_tsd(tsdn); 250 if (witness->rank == WITNESS_RANK_OMIT) 251 return; 252 253 /* 254 * Check whether owner before removal, rather than relying on 255 * witness_assert_owner() to abort, so that unit tests can test this 256 * function's failure mode without causing undefined behavior. 257 */ 258 if (witness_owner(tsd, witness)) { 259 witnesses = tsd_witnessesp_get(tsd); 260 ql_remove(witnesses, witness, link); 261 } else 262 witness_assert_owner(tsdn, witness); 263 } 264 #endif 265 266 #endif /* JEMALLOC_H_INLINES */ 267 /******************************************************************************/ 268
