Home | History | Annotate | Download | only in src
      1 // Copyright (c) 2007, Google Inc.
      2 // All rights reserved.
      3 //
      4 // Redistribution and use in source and binary forms, with or without
      5 // modification, are permitted provided that the following conditions are
      6 // met:
      7 //
      8 //     * Redistributions of source code must retain the above copyright
      9 // notice, this list of conditions and the following disclaimer.
     10 //     * Redistributions in binary form must reproduce the above
     11 // copyright notice, this list of conditions and the following disclaimer
     12 // in the documentation and/or other materials provided with the
     13 // distribution.
     14 //     * Neither the name of Google Inc. nor the names of its
     15 // contributors may be used to endorse or promote products derived from
     16 // this software without specific prior written permission.
     17 //
     18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 // ---
     31 // Author: Arun Sharma
     32 //
     33 // A tcmalloc system allocator that uses a memory based filesystem such as
     34 // tmpfs or hugetlbfs
     35 //
     36 // Since these only exist on linux, we only register this allocator there.
     37 
     38 #ifdef __linux
     39 
     40 #include <config.h>
     41 #include <errno.h>                      // for errno, EINVAL
     42 #include <inttypes.h>                   // for PRId64
     43 #include <limits.h>                     // for PATH_MAX
     44 #include <stddef.h>                     // for size_t, NULL
     45 #ifdef HAVE_STDINT_H
     46 #include <stdint.h>                     // for int64_t, uintptr_t
     47 #endif
     48 #include <stdio.h>                      // for snprintf
     49 #include <stdlib.h>                     // for mkstemp
     50 #include <string.h>                     // for strerror
     51 #include <sys/mman.h>                   // for mmap, MAP_FAILED, etc
     52 #include <sys/statfs.h>                 // for fstatfs, statfs
     53 #include <unistd.h>                     // for ftruncate, off_t, unlink
     54 #include <new>                          // for operator new
     55 #include <string>
     56 
     57 #include <gperftools/malloc_extension.h>
     58 #include "base/basictypes.h"
     59 #include "base/googleinit.h"
     60 #include "base/sysinfo.h"
     61 #include "internal_logging.h"
     62 
     63 // TODO(sanjay): Move the code below into the tcmalloc namespace
     64 using tcmalloc::kLog;
     65 using tcmalloc::kCrash;
     66 using tcmalloc::Log;
     67 using std::string;
     68 
     69 DEFINE_string(memfs_malloc_path, EnvToString("TCMALLOC_MEMFS_MALLOC_PATH", ""),
     70               "Path where hugetlbfs or tmpfs is mounted. The caller is "
     71               "responsible for ensuring that the path is unique and does "
     72               "not conflict with another process");
     73 DEFINE_int64(memfs_malloc_limit_mb,
     74              EnvToInt("TCMALLOC_MEMFS_LIMIT_MB", 0),
     75              "Limit total allocation size to the "
     76              "specified number of MiB.  0 == no limit.");
     77 DEFINE_bool(memfs_malloc_abort_on_fail,
     78             EnvToBool("TCMALLOC_MEMFS_ABORT_ON_FAIL", false),
     79             "abort whenever memfs_malloc fails to satisfy an allocation "
     80             "for any reason.");
     81 DEFINE_bool(memfs_malloc_ignore_mmap_fail,
     82             EnvToBool("TCMALLOC_MEMFS_IGNORE_MMAP_FAIL", false),
     83             "Ignore failures from mmap");
     84 DEFINE_bool(memfs_malloc_map_private,
     85             EnvToBool("TCMALLOC_MEMFS_MAP_PRIVATE", false),
     86 	    "Use MAP_PRIVATE with mmap");
     87 
     88 // Hugetlbfs based allocator for tcmalloc
     89 class HugetlbSysAllocator: public SysAllocator {
     90 public:
     91   explicit HugetlbSysAllocator(SysAllocator* fallback)
     92     : failed_(true),  // To disable allocator until Initialize() is called.
     93       big_page_size_(0),
     94       hugetlb_fd_(-1),
     95       hugetlb_base_(0),
     96       fallback_(fallback) {
     97   }
     98 
     99   void* Alloc(size_t size, size_t *actual_size, size_t alignment);
    100   bool Initialize();
    101 
    102   bool failed_;          // Whether failed to allocate memory.
    103 
    104 private:
    105   void* AllocInternal(size_t size, size_t *actual_size, size_t alignment);
    106 
    107   int64 big_page_size_;
    108   int hugetlb_fd_;       // file descriptor for hugetlb
    109   off_t hugetlb_base_;
    110 
    111   SysAllocator* fallback_;  // Default system allocator to fall back to.
    112 };
    113 static char hugetlb_space[sizeof(HugetlbSysAllocator)];
    114 
    115 // No locking needed here since we assume that tcmalloc calls
    116 // us with an internal lock held (see tcmalloc/system-alloc.cc).
    117 void* HugetlbSysAllocator::Alloc(size_t size, size_t *actual_size,
    118                                  size_t alignment) {
    119   if (failed_) {
    120     return fallback_->Alloc(size, actual_size, alignment);
    121   }
    122 
    123   // We don't respond to allocation requests smaller than big_page_size_ unless
    124   // the caller is ok to take more than they asked for. Used by MetaDataAlloc.
    125   if (actual_size == NULL && size < big_page_size_) {
    126     return fallback_->Alloc(size, actual_size, alignment);
    127   }
    128 
    129   // Enforce huge page alignment.  Be careful to deal with overflow.
    130   size_t new_alignment = alignment;
    131   if (new_alignment < big_page_size_) new_alignment = big_page_size_;
    132   size_t aligned_size = ((size + new_alignment - 1) /
    133                          new_alignment) * new_alignment;
    134   if (aligned_size < size) {
    135     return fallback_->Alloc(size, actual_size, alignment);
    136   }
    137 
    138   void* result = AllocInternal(aligned_size, actual_size, new_alignment);
    139   if (result != NULL) {
    140     return result;
    141   }
    142   Log(kLog, __FILE__, __LINE__,
    143       "HugetlbSysAllocator: (failed, allocated)", failed_, hugetlb_base_);
    144   if (FLAGS_memfs_malloc_abort_on_fail) {
    145     Log(kCrash, __FILE__, __LINE__,
    146         "memfs_malloc_abort_on_fail is set");
    147   }
    148   return fallback_->Alloc(size, actual_size, alignment);
    149 }
    150 
    151 void* HugetlbSysAllocator::AllocInternal(size_t size, size_t* actual_size,
    152                                          size_t alignment) {
    153   // Ask for extra memory if alignment > pagesize
    154   size_t extra = 0;
    155   if (alignment > big_page_size_) {
    156     extra = alignment - big_page_size_;
    157   }
    158 
    159   // Test if this allocation would put us over the limit.
    160   off_t limit = FLAGS_memfs_malloc_limit_mb*1024*1024;
    161   if (limit > 0 && hugetlb_base_ + size + extra > limit) {
    162     // Disable the allocator when there's less than one page left.
    163     if (limit - hugetlb_base_ < big_page_size_) {
    164       Log(kLog, __FILE__, __LINE__, "reached memfs_malloc_limit_mb");
    165       failed_ = true;
    166     }
    167     else {
    168       Log(kLog, __FILE__, __LINE__,
    169           "alloc too large (size, bytes left)", size, limit-hugetlb_base_);
    170     }
    171     return NULL;
    172   }
    173 
    174   // This is not needed for hugetlbfs, but needed for tmpfs.  Annoyingly
    175   // hugetlbfs returns EINVAL for ftruncate.
    176   int ret = ftruncate(hugetlb_fd_, hugetlb_base_ + size + extra);
    177   if (ret != 0 && errno != EINVAL) {
    178     Log(kLog, __FILE__, __LINE__,
    179         "ftruncate failed", strerror(errno));
    180     failed_ = true;
    181     return NULL;
    182   }
    183 
    184   // Note: size + extra does not overflow since:
    185   //            size + alignment < (1<<NBITS).
    186   // and        extra <= alignment
    187   // therefore  size + extra < (1<<NBITS)
    188   void *result;
    189   result = mmap(0, size + extra, PROT_WRITE|PROT_READ,
    190                 FLAGS_memfs_malloc_map_private ? MAP_PRIVATE : MAP_SHARED,
    191                 hugetlb_fd_, hugetlb_base_);
    192   if (result == reinterpret_cast<void*>(MAP_FAILED)) {
    193     if (!FLAGS_memfs_malloc_ignore_mmap_fail) {
    194       Log(kLog, __FILE__, __LINE__,
    195           "mmap failed (size, error)", size + extra, strerror(errno));
    196       failed_ = true;
    197     }
    198     return NULL;
    199   }
    200   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
    201 
    202   // Adjust the return memory so it is aligned
    203   size_t adjust = 0;
    204   if ((ptr & (alignment - 1)) != 0) {
    205     adjust = alignment - (ptr & (alignment - 1));
    206   }
    207   ptr += adjust;
    208   hugetlb_base_ += (size + extra);
    209 
    210   if (actual_size) {
    211     *actual_size = size + extra - adjust;
    212   }
    213 
    214   return reinterpret_cast<void*>(ptr);
    215 }
    216 
    217 bool HugetlbSysAllocator::Initialize() {
    218   char path[PATH_MAX];
    219   const int pathlen = FLAGS_memfs_malloc_path.size();
    220   if (pathlen + 8 > sizeof(path)) {
    221     Log(kCrash, __FILE__, __LINE__, "XX fatal: memfs_malloc_path too long");
    222     return false;
    223   }
    224   memcpy(path, FLAGS_memfs_malloc_path.data(), pathlen);
    225   memcpy(path + pathlen, ".XXXXXX", 8);  // Also copies terminating \0
    226 
    227   int hugetlb_fd = mkstemp(path);
    228   if (hugetlb_fd == -1) {
    229     Log(kLog, __FILE__, __LINE__,
    230         "warning: unable to create memfs_malloc_path",
    231         path, strerror(errno));
    232     return false;
    233   }
    234 
    235   // Cleanup memory on process exit
    236   if (unlink(path) == -1) {
    237     Log(kCrash, __FILE__, __LINE__,
    238         "fatal: error unlinking memfs_malloc_path", path, strerror(errno));
    239     return false;
    240   }
    241 
    242   // Use fstatfs to figure out the default page size for memfs
    243   struct statfs sfs;
    244   if (fstatfs(hugetlb_fd, &sfs) == -1) {
    245     Log(kCrash, __FILE__, __LINE__,
    246         "fatal: error fstatfs of memfs_malloc_path", strerror(errno));
    247     return false;
    248   }
    249   int64 page_size = sfs.f_bsize;
    250 
    251   hugetlb_fd_ = hugetlb_fd;
    252   big_page_size_ = page_size;
    253   failed_ = false;
    254   return true;
    255 }
    256 
    257 REGISTER_MODULE_INITIALIZER(memfs_malloc, {
    258   if (FLAGS_memfs_malloc_path.length()) {
    259     SysAllocator* alloc = MallocExtension::instance()->GetSystemAllocator();
    260     HugetlbSysAllocator* hp = new (hugetlb_space) HugetlbSysAllocator(alloc);
    261     if (hp->Initialize()) {
    262       MallocExtension::instance()->SetSystemAllocator(hp);
    263     }
    264   }
    265 });
    266 
    267 #endif   /* ifdef __linux */
    268