1 /* ===---------- emutls.c - Implements __emutls_get_address ---------------=== 2 * 3 * The LLVM Compiler Infrastructure 4 * 5 * This file is dual licensed under the MIT and the University of Illinois Open 6 * Source Licenses. See LICENSE.TXT for details. 7 * 8 * ===----------------------------------------------------------------------=== 9 */ 10 #include <pthread.h> 11 #include <stdint.h> 12 #include <stdlib.h> 13 #include <string.h> 14 15 #include "int_lib.h" 16 #include "int_util.h" 17 18 /* Default is not to use posix_memalign, so systems like Android 19 * can use thread local data without heavier POSIX memory allocators. 20 */ 21 #ifndef EMUTLS_USE_POSIX_MEMALIGN 22 #define EMUTLS_USE_POSIX_MEMALIGN 0 23 #endif 24 25 /* For every TLS variable xyz, 26 * there is one __emutls_control variable named __emutls_v.xyz. 27 * If xyz has non-zero initial value, __emutls_v.xyz's "value" 28 * will point to __emutls_t.xyz, which has the initial value. 29 */ 30 typedef unsigned int gcc_word __attribute__((mode(word))); 31 typedef struct __emutls_control { 32 /* Must use gcc_word here, instead of size_t, to match GCC. When 33 gcc_word is larger than size_t, the upper extra bits are all 34 zeros. We can use variables of size_t to operate on size and 35 align. */ 36 gcc_word size; /* size of the object in bytes */ 37 gcc_word align; /* alignment of the object in bytes */ 38 union { 39 uintptr_t index; /* data[index-1] is the object address */ 40 void* address; /* object address, when in single thread env */ 41 } object; 42 void* value; /* null or non-zero initial value for the object */ 43 } __emutls_control; 44 45 static __inline void *emutls_memalign_alloc(size_t align, size_t size) { 46 void *base; 47 #if EMUTLS_USE_POSIX_MEMALIGN 48 if (posix_memalign(&base, align, size) != 0) 49 abort(); 50 #else 51 #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) 52 char* object; 53 if ((object = malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) 54 abort(); 55 base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) 56 & ~(uintptr_t)(align - 1)); 57 58 ((void**)base)[-1] = object; 59 #endif 60 return base; 61 } 62 63 static __inline void emutls_memalign_free(void *base) { 64 #if EMUTLS_USE_POSIX_MEMALIGN 65 free(base); 66 #else 67 /* The mallocated address is in ((void**)base)[-1] */ 68 free(((void**)base)[-1]); 69 #endif 70 } 71 72 /* Emulated TLS objects are always allocated at run-time. */ 73 static __inline void *emutls_allocate_object(__emutls_control *control) { 74 /* Use standard C types, check with gcc's emutls.o. */ 75 typedef unsigned int gcc_pointer __attribute__((mode(pointer))); 76 COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); 77 COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); 78 79 size_t size = control->size; 80 size_t align = control->align; 81 void* base; 82 if (align < sizeof(void*)) 83 align = sizeof(void*); 84 /* Make sure that align is power of 2. */ 85 if ((align & (align - 1)) != 0) 86 abort(); 87 88 base = emutls_memalign_alloc(align, size); 89 if (control->value) 90 memcpy(base, control->value, size); 91 else 92 memset(base, 0, size); 93 return base; 94 } 95 96 static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; 97 98 static size_t emutls_num_object = 0; /* number of allocated TLS objects */ 99 100 typedef struct emutls_address_array { 101 uintptr_t size; /* number of elements in the 'data' array */ 102 void* data[]; 103 } emutls_address_array; 104 105 static pthread_key_t emutls_pthread_key; 106 107 static void emutls_key_destructor(void* ptr) { 108 emutls_address_array* array = (emutls_address_array*)ptr; 109 uintptr_t i; 110 for (i = 0; i < array->size; ++i) { 111 if (array->data[i]) 112 emutls_memalign_free(array->data[i]); 113 } 114 free(ptr); 115 } 116 117 static void emutls_init(void) { 118 if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) 119 abort(); 120 } 121 122 /* Returns control->object.index; set index if not allocated yet. */ 123 static __inline uintptr_t emutls_get_index(__emutls_control *control) { 124 uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); 125 if (!index) { 126 static pthread_once_t once = PTHREAD_ONCE_INIT; 127 pthread_once(&once, emutls_init); 128 pthread_mutex_lock(&emutls_mutex); 129 index = control->object.index; 130 if (!index) { 131 index = ++emutls_num_object; 132 __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); 133 } 134 pthread_mutex_unlock(&emutls_mutex); 135 } 136 return index; 137 } 138 139 /* Updates newly allocated thread local emutls_address_array. */ 140 static __inline void emutls_check_array_set_size(emutls_address_array *array, 141 uintptr_t size) { 142 if (array == NULL) 143 abort(); 144 array->size = size; 145 pthread_setspecific(emutls_pthread_key, (void*)array); 146 } 147 148 /* Returns the new 'data' array size, number of elements, 149 * which must be no smaller than the given index. 150 */ 151 static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { 152 /* Need to allocate emutls_address_array with one extra slot 153 * to store the data array size. 154 * Round up the emutls_address_array size to multiple of 16. 155 */ 156 return ((index + 1 + 15) & ~((uintptr_t)15)) - 1; 157 } 158 159 /* Returns the thread local emutls_address_array. 160 * Extends its size if necessary to hold address at index. 161 */ 162 static __inline emutls_address_array * 163 emutls_get_address_array(uintptr_t index) { 164 emutls_address_array* array = pthread_getspecific(emutls_pthread_key); 165 if (array == NULL) { 166 uintptr_t new_size = emutls_new_data_array_size(index); 167 array = malloc(new_size * sizeof(void *) + sizeof(emutls_address_array)); 168 if (array) 169 memset(array->data, 0, new_size * sizeof(void*)); 170 emutls_check_array_set_size(array, new_size); 171 } else if (index > array->size) { 172 uintptr_t orig_size = array->size; 173 uintptr_t new_size = emutls_new_data_array_size(index); 174 array = realloc(array, new_size * sizeof(void *) + sizeof(emutls_address_array)); 175 if (array) 176 memset(array->data + orig_size, 0, 177 (new_size - orig_size) * sizeof(void*)); 178 emutls_check_array_set_size(array, new_size); 179 } 180 return array; 181 } 182 183 void* __emutls_get_address(__emutls_control* control) { 184 uintptr_t index = emutls_get_index(control); 185 emutls_address_array* array = emutls_get_address_array(index); 186 if (array->data[index - 1] == NULL) 187 array->data[index - 1] = emutls_allocate_object(control); 188 return array->data[index - 1]; 189 } 190
