1 /* 2 * Implementation of the Global Interpreter Lock (GIL). 3 */ 4 5 #include <stdlib.h> 6 #include <errno.h> 7 8 9 /* First some general settings */ 10 11 #define INTERVAL (_PyRuntime.ceval.gil.interval >= 1 ? _PyRuntime.ceval.gil.interval : 1) 12 13 14 /* 15 Notes about the implementation: 16 17 - The GIL is just a boolean variable (locked) whose access is protected 18 by a mutex (gil_mutex), and whose changes are signalled by a condition 19 variable (gil_cond). gil_mutex is taken for short periods of time, 20 and therefore mostly uncontended. 21 22 - In the GIL-holding thread, the main loop (PyEval_EvalFrameEx) must be 23 able to release the GIL on demand by another thread. A volatile boolean 24 variable (gil_drop_request) is used for that purpose, which is checked 25 at every turn of the eval loop. That variable is set after a wait of 26 `interval` microseconds on `gil_cond` has timed out. 27 28 [Actually, another volatile boolean variable (eval_breaker) is used 29 which ORs several conditions into one. Volatile booleans are 30 sufficient as inter-thread signalling means since Python is run 31 on cache-coherent architectures only.] 32 33 - A thread wanting to take the GIL will first let pass a given amount of 34 time (`interval` microseconds) before setting gil_drop_request. This 35 encourages a defined switching period, but doesn't enforce it since 36 opcodes can take an arbitrary time to execute. 37 38 The `interval` value is available for the user to read and modify 39 using the Python API `sys.{get,set}switchinterval()`. 40 41 - When a thread releases the GIL and gil_drop_request is set, that thread 42 ensures that another GIL-awaiting thread gets scheduled. 43 It does so by waiting on a condition variable (switch_cond) until 44 the value of last_holder is changed to something else than its 45 own thread state pointer, indicating that another thread was able to 46 take the GIL. 47 48 This is meant to prohibit the latency-adverse behaviour on multi-core 49 machines where one thread would speculatively release the GIL, but still 50 run and end up being the first to re-acquire it, making the "timeslices" 51 much longer than expected. 52 (Note: this mechanism is enabled with FORCE_SWITCHING above) 53 */ 54 55 #include "condvar.h" 56 57 #define MUTEX_INIT(mut) \ 58 if (PyMUTEX_INIT(&(mut))) { \ 59 Py_FatalError("PyMUTEX_INIT(" #mut ") failed"); }; 60 #define MUTEX_FINI(mut) \ 61 if (PyMUTEX_FINI(&(mut))) { \ 62 Py_FatalError("PyMUTEX_FINI(" #mut ") failed"); }; 63 #define MUTEX_LOCK(mut) \ 64 if (PyMUTEX_LOCK(&(mut))) { \ 65 Py_FatalError("PyMUTEX_LOCK(" #mut ") failed"); }; 66 #define MUTEX_UNLOCK(mut) \ 67 if (PyMUTEX_UNLOCK(&(mut))) { \ 68 Py_FatalError("PyMUTEX_UNLOCK(" #mut ") failed"); }; 69 70 #define COND_INIT(cond) \ 71 if (PyCOND_INIT(&(cond))) { \ 72 Py_FatalError("PyCOND_INIT(" #cond ") failed"); }; 73 #define COND_FINI(cond) \ 74 if (PyCOND_FINI(&(cond))) { \ 75 Py_FatalError("PyCOND_FINI(" #cond ") failed"); }; 76 #define COND_SIGNAL(cond) \ 77 if (PyCOND_SIGNAL(&(cond))) { \ 78 Py_FatalError("PyCOND_SIGNAL(" #cond ") failed"); }; 79 #define COND_WAIT(cond, mut) \ 80 if (PyCOND_WAIT(&(cond), &(mut))) { \ 81 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); }; 82 #define COND_TIMED_WAIT(cond, mut, microseconds, timeout_result) \ 83 { \ 84 int r = PyCOND_TIMEDWAIT(&(cond), &(mut), (microseconds)); \ 85 if (r < 0) \ 86 Py_FatalError("PyCOND_WAIT(" #cond ") failed"); \ 87 if (r) /* 1 == timeout, 2 == impl. can't say, so assume timeout */ \ 88 timeout_result = 1; \ 89 else \ 90 timeout_result = 0; \ 91 } \ 92 93 94 #define DEFAULT_INTERVAL 5000 95 96 static void _gil_initialize(struct _gil_runtime_state *state) 97 { 98 _Py_atomic_int uninitialized = {-1}; 99 state->locked = uninitialized; 100 state->interval = DEFAULT_INTERVAL; 101 } 102 103 static int gil_created(void) 104 { 105 return (_Py_atomic_load_explicit(&_PyRuntime.ceval.gil.locked, 106 _Py_memory_order_acquire) 107 ) >= 0; 108 } 109 110 static void create_gil(void) 111 { 112 MUTEX_INIT(_PyRuntime.ceval.gil.mutex); 113 #ifdef FORCE_SWITCHING 114 MUTEX_INIT(_PyRuntime.ceval.gil.switch_mutex); 115 #endif 116 COND_INIT(_PyRuntime.ceval.gil.cond); 117 #ifdef FORCE_SWITCHING 118 COND_INIT(_PyRuntime.ceval.gil.switch_cond); 119 #endif 120 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder, 0); 121 _Py_ANNOTATE_RWLOCK_CREATE(&_PyRuntime.ceval.gil.locked); 122 _Py_atomic_store_explicit(&_PyRuntime.ceval.gil.locked, 0, 123 _Py_memory_order_release); 124 } 125 126 static void destroy_gil(void) 127 { 128 /* some pthread-like implementations tie the mutex to the cond 129 * and must have the cond destroyed first. 130 */ 131 COND_FINI(_PyRuntime.ceval.gil.cond); 132 MUTEX_FINI(_PyRuntime.ceval.gil.mutex); 133 #ifdef FORCE_SWITCHING 134 COND_FINI(_PyRuntime.ceval.gil.switch_cond); 135 MUTEX_FINI(_PyRuntime.ceval.gil.switch_mutex); 136 #endif 137 _Py_atomic_store_explicit(&_PyRuntime.ceval.gil.locked, -1, 138 _Py_memory_order_release); 139 _Py_ANNOTATE_RWLOCK_DESTROY(&_PyRuntime.ceval.gil.locked); 140 } 141 142 static void recreate_gil(void) 143 { 144 _Py_ANNOTATE_RWLOCK_DESTROY(&_PyRuntime.ceval.gil.locked); 145 /* XXX should we destroy the old OS resources here? */ 146 create_gil(); 147 } 148 149 static void drop_gil(PyThreadState *tstate) 150 { 151 if (!_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked)) 152 Py_FatalError("drop_gil: GIL is not locked"); 153 /* tstate is allowed to be NULL (early interpreter init) */ 154 if (tstate != NULL) { 155 /* Sub-interpreter support: threads might have been switched 156 under our feet using PyThreadState_Swap(). Fix the GIL last 157 holder variable so that our heuristics work. */ 158 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder, 159 (uintptr_t)tstate); 160 } 161 162 MUTEX_LOCK(_PyRuntime.ceval.gil.mutex); 163 _Py_ANNOTATE_RWLOCK_RELEASED(&_PyRuntime.ceval.gil.locked, /*is_write=*/1); 164 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.locked, 0); 165 COND_SIGNAL(_PyRuntime.ceval.gil.cond); 166 MUTEX_UNLOCK(_PyRuntime.ceval.gil.mutex); 167 168 #ifdef FORCE_SWITCHING 169 if (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil_drop_request) && 170 tstate != NULL) 171 { 172 MUTEX_LOCK(_PyRuntime.ceval.gil.switch_mutex); 173 /* Not switched yet => wait */ 174 if (((PyThreadState*)_Py_atomic_load_relaxed( 175 &_PyRuntime.ceval.gil.last_holder) 176 ) == tstate) 177 { 178 RESET_GIL_DROP_REQUEST(); 179 /* NOTE: if COND_WAIT does not atomically start waiting when 180 releasing the mutex, another thread can run through, take 181 the GIL and drop it again, and reset the condition 182 before we even had a chance to wait for it. */ 183 COND_WAIT(_PyRuntime.ceval.gil.switch_cond, 184 _PyRuntime.ceval.gil.switch_mutex); 185 } 186 MUTEX_UNLOCK(_PyRuntime.ceval.gil.switch_mutex); 187 } 188 #endif 189 } 190 191 static void take_gil(PyThreadState *tstate) 192 { 193 int err; 194 if (tstate == NULL) 195 Py_FatalError("take_gil: NULL tstate"); 196 197 err = errno; 198 MUTEX_LOCK(_PyRuntime.ceval.gil.mutex); 199 200 if (!_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked)) 201 goto _ready; 202 203 while (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked)) { 204 int timed_out = 0; 205 unsigned long saved_switchnum; 206 207 saved_switchnum = _PyRuntime.ceval.gil.switch_number; 208 COND_TIMED_WAIT(_PyRuntime.ceval.gil.cond, _PyRuntime.ceval.gil.mutex, 209 INTERVAL, timed_out); 210 /* If we timed out and no switch occurred in the meantime, it is time 211 to ask the GIL-holding thread to drop it. */ 212 if (timed_out && 213 _Py_atomic_load_relaxed(&_PyRuntime.ceval.gil.locked) && 214 _PyRuntime.ceval.gil.switch_number == saved_switchnum) { 215 SET_GIL_DROP_REQUEST(); 216 } 217 } 218 _ready: 219 #ifdef FORCE_SWITCHING 220 /* This mutex must be taken before modifying 221 _PyRuntime.ceval.gil.last_holder (see drop_gil()). */ 222 MUTEX_LOCK(_PyRuntime.ceval.gil.switch_mutex); 223 #endif 224 /* We now hold the GIL */ 225 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.locked, 1); 226 _Py_ANNOTATE_RWLOCK_ACQUIRED(&_PyRuntime.ceval.gil.locked, /*is_write=*/1); 227 228 if (tstate != (PyThreadState*)_Py_atomic_load_relaxed( 229 &_PyRuntime.ceval.gil.last_holder)) 230 { 231 _Py_atomic_store_relaxed(&_PyRuntime.ceval.gil.last_holder, 232 (uintptr_t)tstate); 233 ++_PyRuntime.ceval.gil.switch_number; 234 } 235 236 #ifdef FORCE_SWITCHING 237 COND_SIGNAL(_PyRuntime.ceval.gil.switch_cond); 238 MUTEX_UNLOCK(_PyRuntime.ceval.gil.switch_mutex); 239 #endif 240 if (_Py_atomic_load_relaxed(&_PyRuntime.ceval.gil_drop_request)) { 241 RESET_GIL_DROP_REQUEST(); 242 } 243 if (tstate->async_exc != NULL) { 244 _PyEval_SignalAsyncExc(); 245 } 246 247 MUTEX_UNLOCK(_PyRuntime.ceval.gil.mutex); 248 errno = err; 249 } 250 251 void _PyEval_SetSwitchInterval(unsigned long microseconds) 252 { 253 _PyRuntime.ceval.gil.interval = microseconds; 254 } 255 256 unsigned long _PyEval_GetSwitchInterval() 257 { 258 return _PyRuntime.ceval.gil.interval; 259 } 260
