1 //===-- sanitizer_allocator.cc --------------------------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file is shared between AddressSanitizer and ThreadSanitizer 11 // run-time libraries. 12 // This allocator is used inside run-times. 13 //===----------------------------------------------------------------------===// 14 15 #include "sanitizer_allocator.h" 16 #include "sanitizer_allocator_internal.h" 17 #include "sanitizer_common.h" 18 19 namespace __sanitizer { 20 21 // ThreadSanitizer for Go uses libc malloc/free. 22 #if defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) 23 # if SANITIZER_LINUX && !SANITIZER_ANDROID 24 extern "C" void *__libc_malloc(uptr size); 25 extern "C" void *__libc_memalign(uptr alignment, uptr size); 26 extern "C" void *__libc_realloc(void *ptr, uptr size); 27 extern "C" void __libc_free(void *ptr); 28 # else 29 # include <stdlib.h> 30 # define __libc_malloc malloc 31 static void *__libc_memalign(uptr alignment, uptr size) { 32 void *p; 33 uptr error = posix_memalign(&p, alignment, size); 34 if (error) return nullptr; 35 return p; 36 } 37 # define __libc_realloc realloc 38 # define __libc_free free 39 # endif 40 41 static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, 42 uptr alignment) { 43 (void)cache; 44 if (alignment == 0) 45 return __libc_malloc(size); 46 else 47 return __libc_memalign(alignment, size); 48 } 49 50 static void *RawInternalRealloc(void *ptr, uptr size, 51 InternalAllocatorCache *cache) { 52 (void)cache; 53 return __libc_realloc(ptr, size); 54 } 55 56 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { 57 (void)cache; 58 __libc_free(ptr); 59 } 60 61 InternalAllocator *internal_allocator() { 62 return 0; 63 } 64 65 #else // defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) 66 67 static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; 68 static atomic_uint8_t internal_allocator_initialized; 69 static StaticSpinMutex internal_alloc_init_mu; 70 71 static InternalAllocatorCache internal_allocator_cache; 72 static StaticSpinMutex internal_allocator_cache_mu; 73 74 InternalAllocator *internal_allocator() { 75 InternalAllocator *internal_allocator_instance = 76 reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder); 77 if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) { 78 SpinMutexLock l(&internal_alloc_init_mu); 79 if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) == 80 0) { 81 internal_allocator_instance->Init(/* may_return_null*/ false); 82 atomic_store(&internal_allocator_initialized, 1, memory_order_release); 83 } 84 } 85 return internal_allocator_instance; 86 } 87 88 static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, 89 uptr alignment) { 90 if (alignment == 0) alignment = 8; 91 if (cache == 0) { 92 SpinMutexLock l(&internal_allocator_cache_mu); 93 return internal_allocator()->Allocate(&internal_allocator_cache, size, 94 alignment, false); 95 } 96 return internal_allocator()->Allocate(cache, size, alignment, false); 97 } 98 99 static void *RawInternalRealloc(void *ptr, uptr size, 100 InternalAllocatorCache *cache) { 101 uptr alignment = 8; 102 if (cache == 0) { 103 SpinMutexLock l(&internal_allocator_cache_mu); 104 return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, 105 size, alignment); 106 } 107 return internal_allocator()->Reallocate(cache, ptr, size, alignment); 108 } 109 110 static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { 111 if (!cache) { 112 SpinMutexLock l(&internal_allocator_cache_mu); 113 return internal_allocator()->Deallocate(&internal_allocator_cache, ptr); 114 } 115 internal_allocator()->Deallocate(cache, ptr); 116 } 117 118 #endif // defined(SANITIZER_GO) || defined(SANITIZER_USE_MALLOC) 119 120 const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; 121 122 void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { 123 if (size + sizeof(u64) < size) 124 return nullptr; 125 void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment); 126 if (!p) 127 return nullptr; 128 ((u64*)p)[0] = kBlockMagic; 129 return (char*)p + sizeof(u64); 130 } 131 132 void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { 133 if (!addr) 134 return InternalAlloc(size, cache); 135 if (size + sizeof(u64) < size) 136 return nullptr; 137 addr = (char*)addr - sizeof(u64); 138 size = size + sizeof(u64); 139 CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); 140 void *p = RawInternalRealloc(addr, size, cache); 141 if (!p) 142 return nullptr; 143 return (char*)p + sizeof(u64); 144 } 145 146 void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { 147 if (CallocShouldReturnNullDueToOverflow(count, size)) 148 return internal_allocator()->ReturnNullOrDie(); 149 void *p = InternalAlloc(count * size, cache); 150 if (p) internal_memset(p, 0, count * size); 151 return p; 152 } 153 154 void InternalFree(void *addr, InternalAllocatorCache *cache) { 155 if (!addr) 156 return; 157 addr = (char*)addr - sizeof(u64); 158 CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); 159 ((u64*)addr)[0] = 0; 160 RawInternalFree(addr, cache); 161 } 162 163 // LowLevelAllocator 164 static LowLevelAllocateCallback low_level_alloc_callback; 165 166 void *LowLevelAllocator::Allocate(uptr size) { 167 // Align allocation size. 168 size = RoundUpTo(size, 8); 169 if (allocated_end_ - allocated_current_ < (sptr)size) { 170 uptr size_to_allocate = Max(size, GetPageSizeCached()); 171 allocated_current_ = 172 (char*)MmapOrDie(size_to_allocate, __func__); 173 allocated_end_ = allocated_current_ + size_to_allocate; 174 if (low_level_alloc_callback) { 175 low_level_alloc_callback((uptr)allocated_current_, 176 size_to_allocate); 177 } 178 } 179 CHECK(allocated_end_ - allocated_current_ >= (sptr)size); 180 void *res = allocated_current_; 181 allocated_current_ += size; 182 return res; 183 } 184 185 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { 186 low_level_alloc_callback = callback; 187 } 188 189 bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) { 190 if (!size) return false; 191 uptr max = (uptr)-1L; 192 return (max / size) < n; 193 } 194 195 void NORETURN ReportAllocatorCannotReturnNull() { 196 Report("%s's allocator is terminating the process instead of returning 0\n", 197 SanitizerToolName); 198 Report("If you don't like this behavior set allocator_may_return_null=1\n"); 199 CHECK(0); 200 Die(); 201 } 202 203 } // namespace __sanitizer 204
