1 // Copyright (c) 2007, Google Inc. 2 // All rights reserved. 3 // 4 // Redistribution and use in source and binary forms, with or without 5 // modification, are permitted provided that the following conditions are 6 // met: 7 // 8 // * Redistributions of source code must retain the above copyright 9 // notice, this list of conditions and the following disclaimer. 10 // * Redistributions in binary form must reproduce the above 11 // copyright notice, this list of conditions and the following disclaimer 12 // in the documentation and/or other materials provided with the 13 // distribution. 14 // * Neither the name of Google Inc. nor the names of its 15 // contributors may be used to endorse or promote products derived from 16 // this software without specific prior written permission. 17 // 18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 // --- 31 // Author: Arun Sharma 32 // 33 // A tcmalloc system allocator that uses a memory based filesystem such as 34 // tmpfs or hugetlbfs 35 // 36 // Since these only exist on linux, we only register this allocator there. 37 38 #ifdef __linux 39 40 #include <config.h> 41 #include <errno.h> // for errno, EINVAL 42 #include <inttypes.h> // for PRId64 43 #include <limits.h> // for PATH_MAX 44 #include <stddef.h> // for size_t, NULL 45 #ifdef HAVE_STDINT_H 46 #include <stdint.h> // for int64_t, uintptr_t 47 #endif 48 #include <stdio.h> // for snprintf 49 #include <stdlib.h> // for mkstemp 50 #include <string.h> // for strerror 51 #include <sys/mman.h> // for mmap, MAP_FAILED, etc 52 #include <sys/statfs.h> // for fstatfs, statfs 53 #include <unistd.h> // for ftruncate, off_t, unlink 54 #include <new> // for operator new 55 #include <string> 56 57 #include <gperftools/malloc_extension.h> 58 #include "base/basictypes.h" 59 #include "base/googleinit.h" 60 #include "base/sysinfo.h" 61 #include "internal_logging.h" 62 63 // TODO(sanjay): Move the code below into the tcmalloc namespace 64 using tcmalloc::kLog; 65 using tcmalloc::kCrash; 66 using tcmalloc::Log; 67 using std::string; 68 69 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""), 70 "Path where hugetlbfs or tmpfs is mounted. The caller is " 71 "responsible for ensuring that the path is unique and does " 72 "not conflict with another process"); 73 DEFINE_int64(memfs_malloc_limit_mb, 74 EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0), 75 "Limit total allocation size to the " 76 "specified number of MiB. 0 == no limit."); 77 DEFINE_bool(memfs_malloc_abort_on_fail, 78 EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false), 79 "abort() whenever memfs_malloc fails to satisfy an allocation " 80 "for any reason."); 81 DEFINE_bool(memfs_malloc_ignore_mmap_fail, 82 EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false), 83 "Ignore failures from mmap"); 84 DEFINE_bool(memfs_malloc_map_private, 85 EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false), 86 "Use MAP_PRIVATE with mmap"); 87 88 // Hugetlbfs based allocator for tcmalloc 89 class HugetlbSysAllocator: public SysAllocator { 90 public: 91 explicit HugetlbSysAllocator(SysAllocator* fallback) 92 : failed_(true), // To disable allocator until Initialize() is called. 93 big_page_size_(0), 94 hugetlb_fd_(-1), 95 hugetlb_base_(0), 96 fallback_(fallback) { 97 } 98 99 void* Alloc(size_t size, size_t *actual_size, size_t alignment); 100 bool Initialize(); 101 102 bool failed_; // Whether failed to allocate memory. 103 104 private: 105 void* AllocInternal(size_t size, size_t *actual_size, size_t alignment); 106 107 int64 big_page_size_; 108 int hugetlb_fd_; // file descriptor for hugetlb 109 off_t hugetlb_base_; 110 111 SysAllocator* fallback_; // Default system allocator to fall back to. 112 }; 113 static char hugetlb_space[sizeof(HugetlbSysAllocator)]; 114 115 // No locking needed here since we assume that tcmalloc calls 116 // us with an internal lock held (see tcmalloc/system-alloc.cc). 117 void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size, 118 size_t alignment) { 119 if (failed_) { 120 return fallback_->Alloc(size, actual_size, alignment); 121 } 122 123 // We don't respond to allocation requests smaller than big_page_size_ unless 124 // the caller is ok to take more than they asked for. Used by MetaDataAlloc. 125 if (actual_size == NULL && size < big_page_size_) { 126 return fallback_->Alloc(size, actual_size, alignment); 127 } 128 129 // Enforce huge page alignment. Be careful to deal with overflow. 130 size_t new_alignment = alignment; 131 if (new_alignment < big_page_size_) new_alignment = big_page_size_; 132 size_t aligned_size = ((size + new_alignment - 1) / 133 new_alignment) * new_alignment; 134 if (aligned_size < size) { 135 return fallback_->Alloc(size, actual_size, alignment); 136 } 137 138 void* result = AllocInternal(aligned_size, actual_size, new_alignment); 139 if (result != NULL) { 140 return result; 141 } 142 Log(kLog, __FILE__, __LINE__, 143 "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_); 144 if (FLAGS_memfs_malloc_abort_on_fail) { 145 Log(kCrash, __FILE__, __LINE__, 146 "memfs_malloc_abort_on_fail is set"); 147 } 148 return fallback_->Alloc(size, actual_size, alignment); 149 } 150 151 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size, 152 size_t alignment) { 153 // Ask for extra memory if alignment > pagesize 154 size_t extra = 0; 155 if (alignment > big_page_size_) { 156 extra = alignment - big_page_size_; 157 } 158 159 // Test if this allocation would put us over the limit. 160 off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024; 161 if (limit > 0 && hugetlb_base_ + size + extra > limit) { 162 // Disable the allocator when there's less than one page left. 163 if (limit - hugetlb_base_ < big_page_size_) { 164 Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb"); 165 failed_ = true; 166 } 167 else { 168 Log(kLog, __FILE__, __LINE__, 169 "alloc too large (size, bytes left)", size, limit-hugetlb_base_); 170 } 171 return NULL; 172 } 173 174 // This is not needed for hugetlbfs, but needed for tmpfs. Annoyingly 175 // hugetlbfs returns EINVAL for ftruncate. 176 int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra); 177 if (ret != 0 && errno != EINVAL) { 178 Log(kLog, __FILE__, __LINE__, 179 "ftruncate failed", strerror(errno)); 180 failed_ = true; 181 return NULL; 182 } 183 184 // Note: size + extra does not overflow since: 185 // size + alignment < (1<<NBITS). 186 // and extra <= alignment 187 // therefore size + extra < (1<<NBITS) 188 void *result; 189 result = mmap(0, size + extra, PROT_WRITE|PROT_READ, 190 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED, 191 hugetlb_fd_, hugetlb_base_); 192 if (result == reinterpret_cast<void*>(MAP_FAILED)) { 193 if (!FLAGS_memfs_malloc_ignore_mmap_fail) { 194 Log(kLog, __FILE__, __LINE__, 195 "mmap failed (size, error)", size + extra, strerror(errno)); 196 failed_ = true; 197 } 198 return NULL; 199 } 200 uintptr_t ptr = reinterpret_cast<uintptr_t>(result); 201 202 // Adjust the return memory so it is aligned 203 size_t adjust = 0; 204 if ((ptr & (alignment - 1)) != 0) { 205 adjust = alignment - (ptr & (alignment - 1)); 206 } 207 ptr += adjust; 208 hugetlb_base_ += (size + extra); 209 210 if (actual_size) { 211 *actual_size = size + extra - adjust; 212 } 213 214 return reinterpret_cast<void*>(ptr); 215 } 216 217 bool HugetlbSysAllocator::Initialize() { 218 char path[PATH_MAX]; 219 const int pathlen = FLAGS_memfs_malloc_path.size(); 220 if (pathlen + 8 > sizeof(path)) { 221 Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long"); 222 return false; 223 } 224 memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen); 225 memcpy(path + pathlen, ".XXXXXX", 8); // Also copies terminating \0 226 227 int hugetlb_fd = mkstemp(path); 228 if (hugetlb_fd == -1) { 229 Log(kLog, __FILE__, __LINE__, 230 "warning: unable to create memfs_malloc_path", 231 path, strerror(errno)); 232 return false; 233 } 234 235 // Cleanup memory on process exit 236 if (unlink(path) == -1) { 237 Log(kCrash, __FILE__, __LINE__, 238 "fatal: error unlinking memfs_malloc_path", path, strerror(errno)); 239 return false; 240 } 241 242 // Use fstatfs to figure out the default page size for memfs 243 struct statfs sfs; 244 if (fstatfs(hugetlb_fd, &sfs) == -1) { 245 Log(kCrash, __FILE__, __LINE__, 246 "fatal: error fstatfs of memfs_malloc_path", strerror(errno)); 247 return false; 248 } 249 int64 page_size = sfs.f_bsize; 250 251 hugetlb_fd_ = hugetlb_fd; 252 big_page_size_ = page_size; 253 failed_ = false; 254 return true; 255 } 256 257 REGISTER_MODULE_INITIALIZER(memfs_malloc, { 258 if (FLAGS_memfs_malloc_path.length()) { 259 SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator(); 260 HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc); 261 if (hp->Initialize()) { 262 MallocExtension::instance()->SetSystemAllocator(hp); 263 } 264 } 265 }); 266 267 #endif /* ifdef __linux */ 268
