1 // Copyright (c) 2005, 2007, Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above 11 // copyright notice, this list of conditions and the following disclaimer 12 // in the documentation and/or other materials provided with the 13 // distribution. 14 // * Neither the name of Google Inc. nor the names of its 15 // contributors may be used to endorse or promote products derived from 16 // this software without specific prior written permission. 17 // 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 // --- 31 // Author: Sanjay Ghemawat 32 33 #include "config.h" 34 #if !USE(SYSTEM_MALLOC) 35 #include "TCSystemAlloc.h" 36 37 #include "Assertions.h" 38 #include "CheckedArithmetic.h" 39 #include "TCSpinLock.h" 40 #include "UnusedParam.h" 41 #include "VMTags.h" 42 #include <algorithm> 43 #include <stdint.h> 44 45 #if OS(WINDOWS) 46 #include "windows.h" 47 #else 48 #include <errno.h> 49 #include <unistd.h> 50 #include <sys/mman.h> 51 #endif 52 53 #ifndef MAP_ANONYMOUS 54 #define MAP_ANONYMOUS MAP_ANON 55 #endif 56 57 using namespace std; 58 59 // Structure for discovering alignment 60 union MemoryAligner { 61 void* p; 62 double d; 63 size_t s; 64 }; 65 66 static SpinLock spinlock = SPINLOCK_INITIALIZER; 67 68 // Page size is initialized on demand 69 static size_t pagesize = 0; 70 71 // Configuration parameters. 72 73 #if HAVE(MMAP) 74 static bool use_mmap = true; 75 #endif 76 77 #if HAVE(VIRTUALALLOC) 78 static bool use_VirtualAlloc = true; 79 #endif 80 81 // Flags to keep us from retrying allocators that failed. 82 static bool devmem_failure = false; 83 static bool sbrk_failure = false; 84 static bool mmap_failure = false; 85 static bool VirtualAlloc_failure = false; 86 87 static const int32_t FLAGS_malloc_devmem_start = 0; 88 static const int32_t FLAGS_malloc_devmem_limit = 0; 89 90 #if HAVE(MMAP) 91 92 static void* TryMmap(size_t size, size_t *actual_size, size_t alignment) { 93 // Enforce page alignment 94 if (pagesize == 0) pagesize = getpagesize(); 95 if (alignment < pagesize) alignment = pagesize; 96 size = ((size + alignment - 1) / alignment) * alignment; 97 98 // could theoretically return the "extra" bytes here, but this 99 // is simple and correct. 100 if (actual_size) 101 *actual_size = size; 102 103 // Ask for extra memory if alignment > pagesize 104 size_t extra = 0; 105 if (alignment > pagesize) { 106 extra = alignment - pagesize; 107 } 108 Checked<size_t> mapSize = Checked<size_t>(size) + extra + 2 * pagesize; 109 void* result = mmap(NULL, mapSize.unsafeGet(), 110 PROT_READ | PROT_WRITE, 111 MAP_PRIVATE|MAP_ANONYMOUS, 112 VM_TAG_FOR_TCMALLOC_MEMORY, 0); 113 if (result == reinterpret_cast<void*>(MAP_FAILED)) { 114 mmap_failure = true; 115 return NULL; 116 } 117 mmap(result, pagesize, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, VM_TAG_FOR_TCMALLOC_MEMORY, 0); 118 mmap(static_cast<char*>(result) + (mapSize - pagesize).unsafeGet(), pagesize, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, VM_TAG_FOR_TCMALLOC_MEMORY, 0); 119 result = static_cast<char*>(result) + pagesize; 120 // Adjust the return memory so it is aligned 121 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); 122 size_t adjust = 0; 123 if ((ptr & (alignment - 1)) != 0) { 124 adjust = alignment - (ptr & (alignment - 1)); 125 } 126 127 // Return the unused memory to the system 128 if (adjust > 0) { 129 munmap(reinterpret_cast<void*>(ptr), adjust); 130 } 131 if (adjust < extra) { 132 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); 133 } 134 135 ptr += adjust; 136 return reinterpret_cast<void*>(ptr); 137 } 138 139 #endif /* HAVE(MMAP) */ 140 141 #if HAVE(VIRTUALALLOC) 142 143 static void* TryVirtualAlloc(size_t size, size_t *actual_size, size_t alignment) { 144 // Enforce page alignment 145 if (pagesize == 0) { 146 SYSTEM_INFO system_info; 147 GetSystemInfo(&system_info); 148 pagesize = system_info.dwPageSize; 149 } 150 151 if (alignment < pagesize) alignment = pagesize; 152 size = ((size + alignment - 1) / alignment) * alignment; 153 154 // could theoretically return the "extra" bytes here, but this 155 // is simple and correct. 156 if (actual_size) 157 *actual_size = size; 158 159 // Ask for extra memory if alignment > pagesize 160 size_t extra = 0; 161 if (alignment > pagesize) { 162 extra = alignment - pagesize; 163 } 164 void* result = VirtualAlloc(NULL, size + extra, 165 MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN, 166 PAGE_READWRITE); 167 168 if (result == NULL) { 169 VirtualAlloc_failure = true; 170 return NULL; 171 } 172 173 // Adjust the return memory so it is aligned 174 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); 175 size_t adjust = 0; 176 if ((ptr & (alignment - 1)) != 0) { 177 adjust = alignment - (ptr & (alignment - 1)); 178 } 179 180 // Return the unused memory to the system - we'd like to release but the best we can do 181 // is decommit, since Windows only lets you free the whole allocation. 182 if (adjust > 0) { 183 VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT); 184 } 185 if (adjust < extra) { 186 VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT); 187 } 188 189 ptr += adjust; 190 return reinterpret_cast<void*>(ptr); 191 } 192 193 #endif /* HAVE(MMAP) */ 194 195 void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) { 196 // Discard requests that overflow 197 if (size + alignment < size) return NULL; 198 199 SpinLockHolder lock_holder(&spinlock); 200 201 // Enforce minimum alignment 202 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner); 203 204 // Try twice, once avoiding allocators that failed before, and once 205 // more trying all allocators even if they failed before. 206 for (int i = 0; i < 2; i++) { 207 208 #if HAVE(MMAP) 209 if (use_mmap && !mmap_failure) { 210 void* result = TryMmap(size, actual_size, alignment); 211 if (result != NULL) return result; 212 } 213 #endif 214 215 #if HAVE(VIRTUALALLOC) 216 if (use_VirtualAlloc && !VirtualAlloc_failure) { 217 void* result = TryVirtualAlloc(size, actual_size, alignment); 218 if (result != NULL) return result; 219 } 220 #endif 221 222 // nothing worked - reset failure flags and try again 223 devmem_failure = false; 224 sbrk_failure = false; 225 mmap_failure = false; 226 VirtualAlloc_failure = false; 227 } 228 return NULL; 229 } 230 231 #if HAVE(MADV_FREE_REUSE) 232 233 void TCMalloc_SystemRelease(void* start, size_t length) 234 { 235 int madviseResult; 236 237 while ((madviseResult = madvise(start, length, MADV_FREE_REUSABLE)) == -1 && errno == EAGAIN) { } 238 239 // Although really advisory, if madvise fail, we want to know about it. 240 ASSERT_UNUSED(madviseResult, madviseResult != -1); 241 } 242 243 #elif HAVE(MADV_FREE) || HAVE(MADV_DONTNEED) 244 245 void TCMalloc_SystemRelease(void* start, size_t length) 246 { 247 // MADV_FREE clears the modified bit on pages, which allows 248 // them to be discarded immediately. 249 #if HAVE(MADV_FREE) 250 const int advice = MADV_FREE; 251 #else 252 const int advice = MADV_DONTNEED; 253 #endif 254 if (FLAGS_malloc_devmem_start) { 255 // It's not safe to use MADV_DONTNEED if we've been mapping 256 // /dev/mem for heap memory 257 return; 258 } 259 if (pagesize == 0) pagesize = getpagesize(); 260 const size_t pagemask = pagesize - 1; 261 262 size_t new_start = reinterpret_cast<size_t>(start); 263 size_t end = new_start + length; 264 size_t new_end = end; 265 266 // Round up the starting address and round down the ending address 267 // to be page aligned: 268 new_start = (new_start + pagesize - 1) & ~pagemask; 269 new_end = new_end & ~pagemask; 270 271 ASSERT((new_start & pagemask) == 0); 272 ASSERT((new_end & pagemask) == 0); 273 ASSERT(new_start >= reinterpret_cast<size_t>(start)); 274 ASSERT(new_end <= end); 275 276 if (new_end > new_start) { 277 // Note -- ignoring most return codes, because if this fails it 278 // doesn't matter... 279 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, 280 advice) == -1 && 281 errno == EAGAIN) { 282 // NOP 283 } 284 } 285 } 286 287 #elif HAVE(MMAP) 288 289 void TCMalloc_SystemRelease(void* start, size_t length) 290 { 291 void* newAddress = mmap(start, length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); 292 // If the mmap failed then that's ok, we just won't return the memory to the system. 293 ASSERT_UNUSED(newAddress, newAddress == start || newAddress == reinterpret_cast<void*>(MAP_FAILED)); 294 } 295 296 #elif HAVE(VIRTUALALLOC) 297 298 void TCMalloc_SystemRelease(void* start, size_t length) 299 { 300 if (VirtualFree(start, length, MEM_DECOMMIT)) 301 return; 302 303 // The decommit may fail if the memory region consists of allocations 304 // from more than one call to VirtualAlloc. In this case, fall back to 305 // using VirtualQuery to retrieve the allocation boundaries and decommit 306 // them each individually. 307 308 char* ptr = static_cast<char*>(start); 309 char* end = ptr + length; 310 MEMORY_BASIC_INFORMATION info; 311 while (ptr < end) { 312 size_t resultSize = VirtualQuery(ptr, &info, sizeof(info)); 313 ASSERT_UNUSED(resultSize, resultSize == sizeof(info)); 314 315 size_t decommitSize = min<size_t>(info.RegionSize, end - ptr); 316 BOOL success = VirtualFree(ptr, decommitSize, MEM_DECOMMIT); 317 ASSERT_UNUSED(success, success); 318 ptr += decommitSize; 319 } 320 } 321 322 #else 323 324 // Platforms that don't support returning memory use an empty inline version of TCMalloc_SystemRelease 325 // declared in TCSystemAlloc.h 326 327 #endif 328 329 #if HAVE(MADV_FREE_REUSE) 330 331 void TCMalloc_SystemCommit(void* start, size_t length) 332 { 333 while (madvise(start, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { } 334 } 335 336 #elif HAVE(VIRTUALALLOC) 337 338 void TCMalloc_SystemCommit(void* start, size_t length) 339 { 340 if (VirtualAlloc(start, length, MEM_COMMIT, PAGE_READWRITE) == start) 341 return; 342 343 // The commit may fail if the memory region consists of allocations 344 // from more than one call to VirtualAlloc. In this case, fall back to 345 // using VirtualQuery to retrieve the allocation boundaries and commit them 346 // each individually. 347 348 char* ptr = static_cast<char*>(start); 349 char* end = ptr + length; 350 MEMORY_BASIC_INFORMATION info; 351 while (ptr < end) { 352 size_t resultSize = VirtualQuery(ptr, &info, sizeof(info)); 353 ASSERT_UNUSED(resultSize, resultSize == sizeof(info)); 354 355 size_t commitSize = min<size_t>(info.RegionSize, end - ptr); 356 void* newAddress = VirtualAlloc(ptr, commitSize, MEM_COMMIT, PAGE_READWRITE); 357 ASSERT_UNUSED(newAddress, newAddress == ptr); 358 ptr += commitSize; 359 } 360 } 361 362 #else 363 364 // Platforms that don't need to explicitly commit memory use an empty inline version of TCMalloc_SystemCommit 365 // declared in TCSystemAlloc.h 366 367 #endif 368 369 #endif // #if !USE(SYSTEM_MALLOC) 370 371
