1 /* ===---------- emutls.c - Implements __emutls_get_address ---------------=== 2 * 3 * The LLVM Compiler Infrastructure 4 * 5 * This file is dual licensed under the MIT and the University of Illinois Open 6 * Source Licenses. See LICENSE.TXT for details. 7 * 8 * ===----------------------------------------------------------------------=== 9 */ 10 #include <pthread.h> 11 #include <stdint.h> 12 #include <stdlib.h> 13 #include <string.h> 14 15 #include "int_lib.h" 16 #include "int_util.h" 17 18 /* Default is not to use posix_memalign, so systems like Android 19 * can use thread local data without heavier POSIX memory allocators. 20 */ 21 #ifndef EMUTLS_USE_POSIX_MEMALIGN 22 #define EMUTLS_USE_POSIX_MEMALIGN 0 23 #endif 24 25 /* For every TLS variable xyz, 26 * there is one __emutls_control variable named __emutls_v.xyz. 27 * If xyz has non-zero initial value, __emutls_v.xyz's "value" 28 * will point to __emutls_t.xyz, which has the initial value. 29 */ 30 typedef struct __emutls_control { 31 size_t size; /* size of the object in bytes */ 32 size_t align; /* alignment of the object in bytes */ 33 union { 34 uintptr_t index; /* data[index-1] is the object address */ 35 void* address; /* object address, when in single thread env */ 36 } object; 37 void* value; /* null or non-zero initial value for the object */ 38 } __emutls_control; 39 40 static __inline void *emutls_memalign_alloc(size_t align, size_t size) { 41 void *base; 42 #if EMUTLS_USE_POSIX_MEMALIGN 43 if (posix_memalign(&base, align, size) != 0) 44 abort(); 45 #else 46 #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) 47 char* object; 48 if ((object = malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) 49 abort(); 50 base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) 51 & ~(uintptr_t)(align - 1)); 52 53 ((void**)base)[-1] = object; 54 #endif 55 return base; 56 } 57 58 static __inline void emutls_memalign_free(void *base) { 59 #if EMUTLS_USE_POSIX_MEMALIGN 60 free(base); 61 #else 62 /* The mallocated address is in ((void**)base)[-1] */ 63 free(((void**)base)[-1]); 64 #endif 65 } 66 67 /* Emulated TLS objects are always allocated at run-time. */ 68 static __inline void *emutls_allocate_object(__emutls_control *control) { 69 /* Use standard C types, check with gcc's emutls.o. */ 70 typedef unsigned int gcc_word __attribute__((mode(word))); 71 typedef unsigned int gcc_pointer __attribute__((mode(pointer))); 72 COMPILE_TIME_ASSERT(sizeof(size_t) == sizeof(gcc_word)); 73 COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); 74 COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); 75 76 size_t size = control->size; 77 size_t align = control->align; 78 if (align < sizeof(void*)) 79 align = sizeof(void*); 80 /* Make sure that align is power of 2. */ 81 if ((align & (align - 1)) != 0) 82 abort(); 83 84 void* base = emutls_memalign_alloc(align, size); 85 if (control->value) 86 memcpy(base, control->value, size); 87 else 88 memset(base, 0, size); 89 return base; 90 } 91 92 static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; 93 94 static size_t emutls_num_object = 0; /* number of allocated TLS objects */ 95 96 typedef struct emutls_address_array { 97 uintptr_t size; /* number of elements in the 'data' array */ 98 void* data[]; 99 } emutls_address_array; 100 101 static pthread_key_t emutls_pthread_key; 102 103 static void emutls_key_destructor(void* ptr) { 104 emutls_address_array* array = (emutls_address_array*)ptr; 105 uintptr_t i; 106 for (i = 0; i < array->size; ++i) { 107 if (array->data[i]) 108 emutls_memalign_free(array->data[i]); 109 } 110 free(ptr); 111 } 112 113 static void emutls_init(void) { 114 if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) 115 abort(); 116 } 117 118 /* Returns control->object.index; set index if not allocated yet. */ 119 static __inline uintptr_t emutls_get_index(__emutls_control *control) { 120 uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); 121 if (!index) { 122 static pthread_once_t once = PTHREAD_ONCE_INIT; 123 pthread_once(&once, emutls_init); 124 pthread_mutex_lock(&emutls_mutex); 125 index = control->object.index; 126 if (!index) { 127 index = ++emutls_num_object; 128 __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); 129 } 130 pthread_mutex_unlock(&emutls_mutex); 131 } 132 return index; 133 } 134 135 /* Updates newly allocated thread local emutls_address_array. */ 136 static __inline void emutls_check_array_set_size(emutls_address_array *array, 137 uintptr_t size) { 138 if (array == NULL) 139 abort(); 140 array->size = size; 141 pthread_setspecific(emutls_pthread_key, (void*)array); 142 } 143 144 /* Returns the new 'data' array size, number of elements, 145 * which must be no smaller than the given index. 146 */ 147 static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { 148 /* Need to allocate emutls_address_array with one extra slot 149 * to store the data array size. 150 * Round up the emutls_address_array size to multiple of 16. 151 */ 152 return ((index + 1 + 15) & ~((uintptr_t)15)) - 1; 153 } 154 155 /* Returns the thread local emutls_address_array. 156 * Extends its size if necessary to hold address at index. 157 */ 158 static __inline emutls_address_array * 159 emutls_get_address_array(uintptr_t index) { 160 emutls_address_array* array = pthread_getspecific(emutls_pthread_key); 161 if (array == NULL) { 162 uintptr_t new_size = emutls_new_data_array_size(index); 163 array = calloc(new_size + 1, sizeof(void*)); 164 emutls_check_array_set_size(array, new_size); 165 } else if (index > array->size) { 166 uintptr_t orig_size = array->size; 167 uintptr_t new_size = emutls_new_data_array_size(index); 168 array = realloc(array, (new_size + 1) * sizeof(void*)); 169 if (array) 170 memset(array->data + orig_size, 0, 171 (new_size - orig_size) * sizeof(void*)); 172 emutls_check_array_set_size(array, new_size); 173 } 174 return array; 175 } 176 177 void* __emutls_get_address(__emutls_control* control) { 178 uintptr_t index = emutls_get_index(control); 179 emutls_address_array* array = emutls_get_address_array(index); 180 if (array->data[index - 1] == NULL) 181 array->data[index - 1] = emutls_allocate_object(control); 182 return array->data[index - 1]; 183 } 184
