Home | History | Annotate | Download | only in leveldatabase 
      1 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file. See the AUTHORS file for names of contributors.
      4 
      5 #include <errno.h>
      6 #include <stdio.h>
      7 #include <string.h>
      8 
      9 #include <deque>
     10 
     11 #include "base/at_exit.h"
     12 #include "base/debug/trace_event.h"
     13 #include "base/file_util.h"
     14 #include "base/files/file_enumerator.h"
     15 #include "base/files/file_path.h"
     16 #include "base/lazy_instance.h"
     17 #include "base/memory/ref_counted.h"
     18 #include "base/message_loop/message_loop.h"
     19 #include "base/metrics/histogram.h"
     20 #include "base/platform_file.h"
     21 #include "base/posix/eintr_wrapper.h"
     22 #include "base/strings/utf_string_conversions.h"
     23 #include "base/synchronization/lock.h"
     24 #include "base/sys_info.h"
     25 #include "base/threading/platform_thread.h"
     26 #include "base/threading/thread.h"
     27 #include "chromium_logger.h"
     28 #include "env_chromium.h"
     29 #include "leveldb/env.h"
     30 #include "leveldb/slice.h"
     31 #include "port/port.h"
     32 #include "third_party/re2/re2/re2.h"
     33 #include "util/logging.h"
     34 
     35 #if defined(OS_WIN)
     36 #include <io.h>
     37 #include "base/win/win_util.h"
     38 #endif
     39 
     40 #if defined(OS_POSIX)
     41 #include <dirent.h>
     42 #include <fcntl.h>
     43 #include <sys/resource.h>
     44 #include <sys/time.h>
     45 #endif
     46 
     47 using namespace leveldb;
     48 
     49 namespace leveldb_env {
     50 
     51 namespace {
     52 
     53 const base::FilePath::CharType backup_table_extension[] =
     54     FILE_PATH_LITERAL(".bak");
     55 const base::FilePath::CharType table_extension[] = FILE_PATH_LITERAL(".ldb");
     56 
     57 #if (defined(OS_POSIX) && !defined(OS_LINUX)) || defined(OS_WIN)
     58 // The following are glibc-specific
     59 
     60 size_t fread_unlocked(void *ptr, size_t size, size_t n, FILE *file) {
     61   return fread(ptr, size, n, file);
     62 }
     63 
     64 size_t fwrite_unlocked(const void *ptr, size_t size, size_t n, FILE *file) {
     65   return fwrite(ptr, size, n, file);
     66 }
     67 
     68 int fflush_unlocked(FILE *file) {
     69   return fflush(file);
     70 }
     71 
     72 #if !defined(OS_ANDROID)
     73 int fdatasync(int fildes) {
     74 #if defined(OS_WIN)
     75   return _commit(fildes);
     76 #else
     77   return HANDLE_EINTR(fsync(fildes));
     78 #endif
     79 }
     80 #endif
     81 
     82 #endif
     83 
     84 // Wide-char safe fopen wrapper.
     85 FILE* fopen_internal(const char* fname, const char* mode) {
     86 #if defined(OS_WIN)
     87   return _wfopen(UTF8ToUTF16(fname).c_str(), ASCIIToUTF16(mode).c_str());
     88 #else
     89   return fopen(fname, mode);
     90 #endif
     91 }
     92 
     93 base::FilePath CreateFilePath(const std::string& file_path) {
     94 #if defined(OS_WIN)
     95   return base::FilePath(UTF8ToUTF16(file_path));
     96 #else
     97   return base::FilePath(file_path);
     98 #endif
     99 }
    100 
    101 static const base::FilePath::CharType kLevelDBTestDirectoryPrefix[]
    102     = FILE_PATH_LITERAL("leveldb-test-");
    103 
    104 const char* PlatformFileErrorString(const ::base::PlatformFileError& error) {
    105   switch (error) {
    106     case ::base::PLATFORM_FILE_ERROR_FAILED:
    107       return "No further details.";
    108     case ::base::PLATFORM_FILE_ERROR_IN_USE:
    109       return "File currently in use.";
    110     case ::base::PLATFORM_FILE_ERROR_EXISTS:
    111       return "File already exists.";
    112     case ::base::PLATFORM_FILE_ERROR_NOT_FOUND:
    113       return "File not found.";
    114     case ::base::PLATFORM_FILE_ERROR_ACCESS_DENIED:
    115       return "Access denied.";
    116     case ::base::PLATFORM_FILE_ERROR_TOO_MANY_OPENED:
    117       return "Too many files open.";
    118     case ::base::PLATFORM_FILE_ERROR_NO_MEMORY:
    119       return "Out of memory.";
    120     case ::base::PLATFORM_FILE_ERROR_NO_SPACE:
    121       return "No space left on drive.";
    122     case ::base::PLATFORM_FILE_ERROR_NOT_A_DIRECTORY:
    123       return "Not a directory.";
    124     case ::base::PLATFORM_FILE_ERROR_INVALID_OPERATION:
    125       return "Invalid operation.";
    126     case ::base::PLATFORM_FILE_ERROR_SECURITY:
    127       return "Security error.";
    128     case ::base::PLATFORM_FILE_ERROR_ABORT:
    129       return "File operation aborted.";
    130     case ::base::PLATFORM_FILE_ERROR_NOT_A_FILE:
    131       return "The supplied path was not a file.";
    132     case ::base::PLATFORM_FILE_ERROR_NOT_EMPTY:
    133       return "The file was not empty.";
    134     case ::base::PLATFORM_FILE_ERROR_INVALID_URL:
    135       return "Invalid URL.";
    136     case ::base::PLATFORM_FILE_ERROR_IO:
    137       return "OS or hardware error.";
    138     case ::base::PLATFORM_FILE_OK:
    139       return "OK.";
    140     case ::base::PLATFORM_FILE_ERROR_MAX:
    141       NOTREACHED();
    142   }
    143   NOTIMPLEMENTED();
    144   return "Unknown error.";
    145 }
    146 
    147 class ChromiumSequentialFile: public SequentialFile {
    148  private:
    149   std::string filename_;
    150   FILE* file_;
    151   const UMALogger* uma_logger_;
    152 
    153  public:
    154   ChromiumSequentialFile(const std::string& fname, FILE* f,
    155                          const UMALogger* uma_logger)
    156       : filename_(fname), file_(f), uma_logger_(uma_logger) { }
    157   virtual ~ChromiumSequentialFile() { fclose(file_); }
    158 
    159   virtual Status Read(size_t n, Slice* result, char* scratch) {
    160     Status s;
    161     size_t r = fread_unlocked(scratch, 1, n, file_);
    162     *result = Slice(scratch, r);
    163     if (r < n) {
    164       if (feof(file_)) {
    165         // We leave status as ok if we hit the end of the file
    166       } else {
    167         // A partial read with an error: return a non-ok status
    168         s = MakeIOError(filename_, strerror(errno), kSequentialFileRead, errno);
    169         uma_logger_->RecordErrorAt(kSequentialFileRead);
    170       }
    171     }
    172     return s;
    173   }
    174 
    175   virtual Status Skip(uint64_t n) {
    176     if (fseek(file_, n, SEEK_CUR)) {
    177       int saved_errno = errno;
    178       uma_logger_->RecordErrorAt(kSequentialFileSkip);
    179       return MakeIOError(
    180           filename_, strerror(saved_errno), kSequentialFileSkip, saved_errno);
    181     }
    182     return Status::OK();
    183   }
    184 };
    185 
    186 class ChromiumRandomAccessFile: public RandomAccessFile {
    187  private:
    188   std::string filename_;
    189   ::base::PlatformFile file_;
    190   const UMALogger* uma_logger_;
    191 
    192  public:
    193   ChromiumRandomAccessFile(const std::string& fname, ::base::PlatformFile file,
    194                            const UMALogger* uma_logger)
    195       : filename_(fname), file_(file), uma_logger_(uma_logger) { }
    196   virtual ~ChromiumRandomAccessFile() { ::base::ClosePlatformFile(file_); }
    197 
    198   virtual Status Read(uint64_t offset, size_t n, Slice* result,
    199                       char* scratch) const {
    200     Status s;
    201     int r = ::base::ReadPlatformFile(file_, offset, scratch, n);
    202     *result = Slice(scratch, (r < 0) ? 0 : r);
    203     if (r < 0) {
    204       // An error: return a non-ok status
    205       s = MakeIOError(
    206           filename_, "Could not perform read", kRandomAccessFileRead);
    207       uma_logger_->RecordErrorAt(kRandomAccessFileRead);
    208     }
    209     return s;
    210   }
    211 };
    212 
    213 class ChromiumFileLock : public FileLock {
    214  public:
    215   ::base::PlatformFile file_;
    216   std::string name_;
    217 };
    218 
    219 class Retrier {
    220  public:
    221   Retrier(MethodID method, RetrierProvider* provider)
    222       : start_(base::TimeTicks::Now()),
    223         limit_(start_ + base::TimeDelta::FromMilliseconds(
    224                             provider->MaxRetryTimeMillis())),
    225         last_(start_),
    226         time_to_sleep_(base::TimeDelta::FromMilliseconds(10)),
    227         success_(true),
    228         method_(method),
    229         last_error_(base::PLATFORM_FILE_OK),
    230         provider_(provider) {}
    231   ~Retrier() {
    232     if (success_) {
    233       provider_->GetRetryTimeHistogram(method_)->AddTime(last_ - start_);
    234       if (last_error_ != base::PLATFORM_FILE_OK) {
    235         DCHECK(last_error_ < 0);
    236         provider_->GetRecoveredFromErrorHistogram(method_)->Add(-last_error_);
    237       }
    238     }
    239   }
    240   bool ShouldKeepTrying(base::PlatformFileError last_error) {
    241     DCHECK_NE(last_error, base::PLATFORM_FILE_OK);
    242     last_error_ = last_error;
    243     if (last_ < limit_) {
    244       base::PlatformThread::Sleep(time_to_sleep_);
    245       last_ = base::TimeTicks::Now();
    246       return true;
    247     }
    248     success_ = false;
    249     return false;
    250   }
    251 
    252  private:
    253   base::TimeTicks start_;
    254   base::TimeTicks limit_;
    255   base::TimeTicks last_;
    256   base::TimeDelta time_to_sleep_;
    257   bool success_;
    258   MethodID method_;
    259   base::PlatformFileError last_error_;
    260   RetrierProvider* provider_;
    261 };
    262 
    263 class IDBEnv : public ChromiumEnv {
    264  public:
    265   IDBEnv() : ChromiumEnv() {
    266     name_ = "LevelDBEnv.IDB";
    267     make_backup_ = true;
    268   }
    269 };
    270 
    271 ::base::LazyInstance<IDBEnv>::Leaky idb_env = LAZY_INSTANCE_INITIALIZER;
    272 
    273 ::base::LazyInstance<ChromiumEnv>::Leaky default_env =
    274     LAZY_INSTANCE_INITIALIZER;
    275 
    276 }  // unnamed namespace
    277 
    278 const char* MethodIDToString(MethodID method) {
    279   switch (method) {
    280     case kSequentialFileRead:
    281       return "SequentialFileRead";
    282     case kSequentialFileSkip:
    283       return "SequentialFileSkip";
    284     case kRandomAccessFileRead:
    285       return "RandomAccessFileRead";
    286     case kWritableFileAppend:
    287       return "WritableFileAppend";
    288     case kWritableFileClose:
    289       return "WritableFileClose";
    290     case kWritableFileFlush:
    291       return "WritableFileFlush";
    292     case kWritableFileSync:
    293       return "WritableFileSync";
    294     case kNewSequentialFile:
    295       return "NewSequentialFile";
    296     case kNewRandomAccessFile:
    297       return "NewRandomAccessFile";
    298     case kNewWritableFile:
    299       return "NewWritableFile";
    300     case kDeleteFile:
    301       return "DeleteFile";
    302     case kCreateDir:
    303       return "CreateDir";
    304     case kDeleteDir:
    305       return "DeleteDir";
    306     case kGetFileSize:
    307       return "GetFileSize";
    308     case kRenameFile:
    309       return "RenameFile";
    310     case kLockFile:
    311       return "LockFile";
    312     case kUnlockFile:
    313       return "UnlockFile";
    314     case kGetTestDirectory:
    315       return "GetTestDirectory";
    316     case kNewLogger:
    317       return "NewLogger";
    318     case kSyncParent:
    319       return "SyncParent";
    320     case kGetChildren:
    321       return "GetChildren";
    322     case kNumEntries:
    323       NOTREACHED();
    324       return "kNumEntries";
    325   }
    326   NOTREACHED();
    327   return "Unknown";
    328 }
    329 
    330 Status MakeIOError(Slice filename,
    331                    const char* message,
    332                    MethodID method,
    333                    int saved_errno) {
    334   char buf[512];
    335   snprintf(buf,
    336            sizeof(buf),
    337            "%s (ChromeMethodErrno: %d::%s::%d)",
    338            message,
    339            method,
    340            MethodIDToString(method),
    341            saved_errno);
    342   return Status::IOError(filename, buf);
    343 }
    344 
    345 Status MakeIOError(Slice filename,
    346                    const char* message,
    347                    MethodID method,
    348                    base::PlatformFileError error) {
    349   DCHECK(error < 0);
    350   char buf[512];
    351   snprintf(buf,
    352            sizeof(buf),
    353            "%s (ChromeMethodPFE: %d::%s::%d)",
    354            message,
    355            method,
    356            MethodIDToString(method),
    357            -error);
    358   return Status::IOError(filename, buf);
    359 }
    360 
    361 Status MakeIOError(Slice filename, const char* message, MethodID method) {
    362   char buf[512];
    363   snprintf(buf,
    364            sizeof(buf),
    365            "%s (ChromeMethodOnly: %d::%s)",
    366            message,
    367            method,
    368            MethodIDToString(method));
    369   return Status::IOError(filename, buf);
    370 }
    371 
    372 ErrorParsingResult ParseMethodAndError(const char* string,
    373                                        MethodID* method_param,
    374                                        int* error) {
    375   int method;
    376   if (RE2::PartialMatch(string, "ChromeMethodOnly: (\\d+)", &method)) {
    377     *method_param = static_cast<MethodID>(method);
    378     return METHOD_ONLY;
    379   }
    380   if (RE2::PartialMatch(
    381           string, "ChromeMethodPFE: (\\d+)::.*::(\\d+)", &method, error)) {
    382     *error = -*error;
    383     *method_param = static_cast<MethodID>(method);
    384     return METHOD_AND_PFE;
    385   }
    386   if (RE2::PartialMatch(
    387           string, "ChromeMethodErrno: (\\d+)::.*::(\\d+)", &method, error)) {
    388     *method_param = static_cast<MethodID>(method);
    389     return METHOD_AND_ERRNO;
    390   }
    391   return NONE;
    392 }
    393 
    394 // Keep in sync with LevelDBCorruptionTypes in histograms.xml. Also, don't
    395 // change the order because indices into this array have been recorded in uma
    396 // histograms.
    397 const char* patterns[] = {
    398   "missing files",
    399   "log record too small",
    400   "corrupted internal key",
    401   "partial record",
    402   "missing start of fragmented record",
    403   "error in middle of record",
    404   "unknown record type",
    405   "truncated record at end",
    406   "bad record length",
    407   "VersionEdit",
    408   "FileReader invoked with unexpected value",
    409   "corrupted key",
    410   "CURRENT file does not end with newline",
    411   "no meta-nextfile entry",
    412   "no meta-lognumber entry",
    413   "no last-sequence-number entry",
    414   "malformed WriteBatch",
    415   "bad WriteBatch Put",
    416   "bad WriteBatch Delete",
    417   "unknown WriteBatch tag",
    418   "WriteBatch has wrong count",
    419   "bad entry in block",
    420   "bad block contents",
    421   "bad block handle",
    422   "truncated block read",
    423   "block checksum mismatch",
    424   "checksum mismatch",
    425   "corrupted compressed block contents",
    426   "bad block type",
    427   "bad magic number",
    428   "file is too short",
    429 };
    430 
    431 // Returns 1-based index into the above array or 0 if nothing matches.
    432 int GetCorruptionCode(const leveldb::Status& status) {
    433   DCHECK(!status.IsIOError());
    434   DCHECK(!status.ok());
    435   const int kOtherError = 0;
    436   int error = kOtherError;
    437   const std::string& str_error = status.ToString();
    438   const size_t kNumPatterns = arraysize(patterns);
    439   for (size_t i = 0; i < kNumPatterns; ++i) {
    440     if (str_error.find(patterns[i]) != std::string::npos) {
    441       error = i + 1;
    442       break;
    443     }
    444   }
    445   return error;
    446 }
    447 
    448 int GetNumCorruptionCodes() {
    449   // + 1 for the "other" error that is returned when a corruption message
    450   // doesn't match any of the patterns.
    451   return arraysize(patterns) + 1;
    452 }
    453 
    454 std::string GetCorruptionMessage(const leveldb::Status& status) {
    455   int code = GetCorruptionCode(status);
    456   if (code == 0)
    457     return "Unknown corruption";
    458   return patterns[code - 1];
    459 }
    460 
    461 bool IndicatesDiskFull(const leveldb::Status& status) {
    462   if (status.ok())
    463     return false;
    464   leveldb_env::MethodID method;
    465   int error = -1;
    466   leveldb_env::ErrorParsingResult result = leveldb_env::ParseMethodAndError(
    467       status.ToString().c_str(), &method, &error);
    468   return (result == leveldb_env::METHOD_AND_PFE &&
    469           static_cast<base::PlatformFileError>(error) ==
    470               base::PLATFORM_FILE_ERROR_NO_SPACE) ||
    471          (result == leveldb_env::METHOD_AND_ERRNO && error == ENOSPC);
    472 }
    473 
    474 bool IsIOError(const leveldb::Status& status) {
    475   leveldb_env::MethodID method;
    476   int error = -1;
    477   leveldb_env::ErrorParsingResult result = leveldb_env::ParseMethodAndError(
    478       status.ToString().c_str(), &method, &error);
    479   return result != leveldb_env::NONE;
    480 }
    481 
    482 bool IsCorruption(const leveldb::Status& status) {
    483   // LevelDB returns InvalidArgument when an sst file is truncated but there is
    484   // no IsInvalidArgument() accessor defined.
    485   return status.IsCorruption() || (!status.ok() && !IsIOError(status));
    486 }
    487 
    488 std::string FilePathToString(const base::FilePath& file_path) {
    489 #if defined(OS_WIN)
    490   return UTF16ToUTF8(file_path.value());
    491 #else
    492   return file_path.value();
    493 #endif
    494 }
    495 
    496 ChromiumWritableFile::ChromiumWritableFile(const std::string& fname,
    497                                            FILE* f,
    498                                            const UMALogger* uma_logger,
    499                                            WriteTracker* tracker,
    500                                            bool make_backup)
    501     : filename_(fname),
    502       file_(f),
    503       uma_logger_(uma_logger),
    504       tracker_(tracker),
    505       file_type_(kOther),
    506       make_backup_(make_backup) {
    507   base::FilePath path = base::FilePath::FromUTF8Unsafe(fname);
    508   if (FilePathToString(path.BaseName()).find("MANIFEST") == 0)
    509     file_type_ = kManifest;
    510   else if (path.MatchesExtension(table_extension))
    511     file_type_ = kTable;
    512   if (file_type_ != kManifest)
    513     tracker_->DidCreateNewFile(filename_);
    514   parent_dir_ = FilePathToString(CreateFilePath(fname).DirName());
    515 }
    516 
    517 ChromiumWritableFile::~ChromiumWritableFile() {
    518   if (file_ != NULL) {
    519     // Ignoring any potential errors
    520     fclose(file_);
    521   }
    522 }
    523 
    524 Status ChromiumWritableFile::SyncParent() {
    525   Status s;
    526 #if !defined(OS_WIN)
    527   TRACE_EVENT0("leveldb", "SyncParent");
    528 
    529   int parent_fd =
    530       HANDLE_EINTR(open(parent_dir_.c_str(), O_RDONLY));
    531   if (parent_fd < 0) {
    532     int saved_errno = errno;
    533     return MakeIOError(
    534         parent_dir_, strerror(saved_errno), kSyncParent, saved_errno);
    535   }
    536   if (HANDLE_EINTR(fsync(parent_fd)) != 0) {
    537     int saved_errno = errno;
    538     s = MakeIOError(
    539         parent_dir_, strerror(saved_errno), kSyncParent, saved_errno);
    540   };
    541   close(parent_fd);
    542 #endif
    543   return s;
    544 }
    545 
    546 Status ChromiumWritableFile::Append(const Slice& data) {
    547   if (file_type_ == kManifest && tracker_->DoesDirNeedSync(filename_)) {
    548     Status s = SyncParent();
    549     if (!s.ok())
    550       return s;
    551     tracker_->DidSyncDir(filename_);
    552   }
    553 
    554   size_t r = fwrite_unlocked(data.data(), 1, data.size(), file_);
    555   if (r != data.size()) {
    556     int saved_errno = errno;
    557     uma_logger_->RecordOSError(kWritableFileAppend, saved_errno);
    558     return MakeIOError(
    559         filename_, strerror(saved_errno), kWritableFileAppend, saved_errno);
    560   }
    561   return Status::OK();
    562 }
    563 
    564 Status ChromiumWritableFile::Close() {
    565   Status result;
    566   if (fclose(file_) != 0) {
    567     result = MakeIOError(filename_, strerror(errno), kWritableFileClose, errno);
    568     uma_logger_->RecordErrorAt(kWritableFileClose);
    569   }
    570   file_ = NULL;
    571   return result;
    572 }
    573 
    574 Status ChromiumWritableFile::Flush() {
    575   Status result;
    576   if (HANDLE_EINTR(fflush_unlocked(file_))) {
    577     int saved_errno = errno;
    578     result = MakeIOError(
    579         filename_, strerror(saved_errno), kWritableFileFlush, saved_errno);
    580     uma_logger_->RecordOSError(kWritableFileFlush, saved_errno);
    581   }
    582   return result;
    583 }
    584 
    585 static bool MakeBackup(const std::string& fname) {
    586   base::FilePath original_table_name = CreateFilePath(fname);
    587   base::FilePath backup_table_name =
    588       original_table_name.ReplaceExtension(backup_table_extension);
    589   return base::CopyFile(original_table_name, backup_table_name);
    590 }
    591 
    592 Status ChromiumWritableFile::Sync() {
    593   TRACE_EVENT0("leveldb", "ChromiumEnv::Sync");
    594   Status result;
    595   int error = 0;
    596 
    597   if (HANDLE_EINTR(fflush_unlocked(file_)))
    598     error = errno;
    599   // Sync even if fflush gave an error; perhaps the data actually got out,
    600   // even though something went wrong.
    601   if (fdatasync(fileno(file_)) && !error)
    602     error = errno;
    603   // Report the first error we found.
    604   if (error) {
    605     result = MakeIOError(filename_, strerror(error), kWritableFileSync, error);
    606     uma_logger_->RecordErrorAt(kWritableFileSync);
    607   } else if (make_backup_ && file_type_ == kTable) {
    608     bool success = MakeBackup(filename_);
    609     uma_logger_->RecordBackupResult(success);
    610   }
    611   return result;
    612 }
    613 
    614 ChromiumEnv::ChromiumEnv()
    615     : name_("LevelDBEnv"),
    616       make_backup_(false),
    617       bgsignal_(&mu_),
    618       started_bgthread_(false),
    619       kMaxRetryTimeMillis(1000) {
    620 }
    621 
    622 ChromiumEnv::~ChromiumEnv() {
    623   // In chromium, ChromiumEnv is leaked. It'd be nice to add NOTREACHED here to
    624   // ensure that behavior isn't accidentally changed, but there's an instance in
    625   // a unit test that is deleted.
    626 }
    627 
    628 Status ChromiumEnv::NewSequentialFile(const std::string& fname,
    629                                       SequentialFile** result) {
    630   FILE* f = fopen_internal(fname.c_str(), "rb");
    631   if (f == NULL) {
    632     *result = NULL;
    633     int saved_errno = errno;
    634     RecordOSError(kNewSequentialFile, saved_errno);
    635     return MakeIOError(
    636         fname, strerror(saved_errno), kNewSequentialFile, saved_errno);
    637   } else {
    638     *result = new ChromiumSequentialFile(fname, f, this);
    639     return Status::OK();
    640   }
    641 }
    642 
    643 void ChromiumEnv::RecordOpenFilesLimit(const std::string& type) {
    644 #if defined(OS_POSIX)
    645   struct rlimit nofile;
    646   if (getrlimit(RLIMIT_NOFILE, &nofile))
    647     return;
    648   GetMaxFDHistogram(type)->Add(nofile.rlim_cur);
    649 #endif
    650 }
    651 
    652 Status ChromiumEnv::NewRandomAccessFile(const std::string& fname,
    653                                         RandomAccessFile** result) {
    654   int flags = ::base::PLATFORM_FILE_READ | ::base::PLATFORM_FILE_OPEN;
    655   bool created;
    656   ::base::PlatformFileError error_code;
    657   ::base::PlatformFile file = ::base::CreatePlatformFile(
    658       CreateFilePath(fname), flags, &created, &error_code);
    659   if (error_code == ::base::PLATFORM_FILE_OK) {
    660     *result = new ChromiumRandomAccessFile(fname, file, this);
    661     RecordOpenFilesLimit("Success");
    662     return Status::OK();
    663   }
    664   if (error_code == ::base::PLATFORM_FILE_ERROR_TOO_MANY_OPENED)
    665     RecordOpenFilesLimit("TooManyOpened");
    666   else
    667     RecordOpenFilesLimit("OtherError");
    668   *result = NULL;
    669   RecordOSError(kNewRandomAccessFile, error_code);
    670   return MakeIOError(fname,
    671                      PlatformFileErrorString(error_code),
    672                      kNewRandomAccessFile,
    673                      error_code);
    674 }
    675 
    676 Status ChromiumEnv::NewWritableFile(const std::string& fname,
    677                                     WritableFile** result) {
    678   *result = NULL;
    679   FILE* f = fopen_internal(fname.c_str(), "wb");
    680   if (f == NULL) {
    681     int saved_errno = errno;
    682     RecordErrorAt(kNewWritableFile);
    683     return MakeIOError(
    684         fname, strerror(saved_errno), kNewWritableFile, saved_errno);
    685   } else {
    686     *result = new ChromiumWritableFile(fname, f, this, this, make_backup_);
    687     return Status::OK();
    688   }
    689 }
    690 
    691 bool ChromiumEnv::FileExists(const std::string& fname) {
    692   return ::base::PathExists(CreateFilePath(fname));
    693 }
    694 
    695 base::FilePath ChromiumEnv::RestoreFromBackup(const base::FilePath& base_name) {
    696   base::FilePath table_name =
    697       base_name.AddExtension(table_extension);
    698   bool result = base::CopyFile(base_name.AddExtension(backup_table_extension),
    699                                table_name);
    700   std::string uma_name(name_);
    701   uma_name.append(".TableRestore");
    702   base::BooleanHistogram::FactoryGet(
    703       uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
    704   return table_name;
    705 }
    706 
    707 void ChromiumEnv::RestoreIfNecessary(const std::string& dir,
    708                                      std::vector<std::string>* result) {
    709   std::set<base::FilePath> tables_found;
    710   std::set<base::FilePath> backups_found;
    711   for (std::vector<std::string>::iterator it = result->begin();
    712        it != result->end();
    713        ++it) {
    714     base::FilePath current = CreateFilePath(*it);
    715     if (current.MatchesExtension(table_extension))
    716       tables_found.insert(current.RemoveExtension());
    717     if (current.MatchesExtension(backup_table_extension))
    718       backups_found.insert(current.RemoveExtension());
    719   }
    720   std::set<base::FilePath> backups_only;
    721   std::set_difference(backups_found.begin(),
    722                       backups_found.end(),
    723                       tables_found.begin(),
    724                       tables_found.end(),
    725                       std::inserter(backups_only, backups_only.begin()));
    726   if (backups_only.size()) {
    727     std::string uma_name(name_);
    728     uma_name.append(".MissingFiles");
    729     int num_missing_files =
    730         backups_only.size() > INT_MAX ? INT_MAX : backups_only.size();
    731     base::Histogram::FactoryGet(uma_name,
    732                                 1 /*min*/,
    733                                 100 /*max*/,
    734                                 8 /*num_buckets*/,
    735                                 base::Histogram::kUmaTargetedHistogramFlag)
    736         ->Add(num_missing_files);
    737   }
    738   base::FilePath dir_filepath = base::FilePath::FromUTF8Unsafe(dir);
    739   for (std::set<base::FilePath>::iterator it = backups_only.begin();
    740        it != backups_only.end();
    741        ++it) {
    742     base::FilePath restored_table_name =
    743         RestoreFromBackup(dir_filepath.Append(*it));
    744     result->push_back(FilePathToString(restored_table_name.BaseName()));
    745   }
    746 }
    747 
    748 namespace {
    749 #if defined(OS_WIN)
    750 static base::PlatformFileError GetDirectoryEntries(
    751     const base::FilePath& dir_param,
    752     std::vector<base::FilePath>* result) {
    753   result->clear();
    754   base::FilePath dir_filepath = dir_param.Append(FILE_PATH_LITERAL("*"));
    755   WIN32_FIND_DATA find_data;
    756   HANDLE find_handle = FindFirstFile(dir_filepath.value().c_str(), &find_data);
    757   if (find_handle == INVALID_HANDLE_VALUE) {
    758     DWORD last_error = GetLastError();
    759     if (last_error == ERROR_FILE_NOT_FOUND)
    760       return base::PLATFORM_FILE_OK;
    761     return base::LastErrorToPlatformFileError(last_error);
    762   }
    763   do {
    764     base::FilePath filepath(find_data.cFileName);
    765     base::FilePath::StringType basename = filepath.BaseName().value();
    766     if (basename == FILE_PATH_LITERAL(".") ||
    767         basename == FILE_PATH_LITERAL(".."))
    768       continue;
    769     result->push_back(filepath.BaseName());
    770   } while (FindNextFile(find_handle, &find_data));
    771   DWORD last_error = GetLastError();
    772   base::PlatformFileError return_value = base::PLATFORM_FILE_OK;
    773   if (last_error != ERROR_NO_MORE_FILES)
    774     return_value = base::LastErrorToPlatformFileError(last_error);
    775   FindClose(find_handle);
    776   return return_value;
    777 }
    778 #else
    779 static base::PlatformFileError GetDirectoryEntries(
    780     const base::FilePath& dir_filepath,
    781     std::vector<base::FilePath>* result) {
    782   const std::string dir_string = FilePathToString(dir_filepath);
    783   result->clear();
    784   DIR* dir = opendir(dir_string.c_str());
    785   if (!dir)
    786     return base::ErrnoToPlatformFileError(errno);
    787   struct dirent dent_buf;
    788   struct dirent* dent;
    789   int readdir_result;
    790   while ((readdir_result = readdir_r(dir, &dent_buf, &dent)) == 0 && dent) {
    791     if (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)
    792       continue;
    793     result->push_back(CreateFilePath(dent->d_name));
    794   }
    795   int saved_errno = errno;
    796   closedir(dir);
    797   if (readdir_result != 0)
    798     return base::ErrnoToPlatformFileError(saved_errno);
    799   return base::PLATFORM_FILE_OK;
    800 }
    801 #endif
    802 }
    803 
    804 Status ChromiumEnv::GetChildren(const std::string& dir_string,
    805                                 std::vector<std::string>* result) {
    806   std::vector<base::FilePath> entries;
    807   base::PlatformFileError error =
    808       GetDirectoryEntries(CreateFilePath(dir_string), &entries);
    809   if (error != base::PLATFORM_FILE_OK) {
    810     RecordOSError(kGetChildren, error);
    811     return MakeIOError(
    812         dir_string, "Could not open/read directory", kGetChildren, error);
    813   }
    814   result->clear();
    815   for (std::vector<base::FilePath>::iterator it = entries.begin();
    816        it != entries.end();
    817        ++it) {
    818     result->push_back(FilePathToString(*it));
    819   }
    820 
    821   if (make_backup_)
    822     RestoreIfNecessary(dir_string, result);
    823   return Status::OK();
    824 }
    825 
    826 Status ChromiumEnv::DeleteFile(const std::string& fname) {
    827   Status result;
    828   base::FilePath fname_filepath = CreateFilePath(fname);
    829   // TODO(jorlow): Should we assert this is a file?
    830   if (!::base::DeleteFile(fname_filepath, false)) {
    831     result = MakeIOError(fname, "Could not delete file.", kDeleteFile);
    832     RecordErrorAt(kDeleteFile);
    833   }
    834   if (make_backup_ && fname_filepath.MatchesExtension(table_extension)) {
    835     base::DeleteFile(fname_filepath.ReplaceExtension(backup_table_extension),
    836                      false);
    837   }
    838   return result;
    839 }
    840 
    841 Status ChromiumEnv::CreateDir(const std::string& name) {
    842   Status result;
    843   base::PlatformFileError error = base::PLATFORM_FILE_OK;
    844   Retrier retrier(kCreateDir, this);
    845   do {
    846     if (base::CreateDirectoryAndGetError(CreateFilePath(name), &error))
    847       return result;
    848   } while (retrier.ShouldKeepTrying(error));
    849   result = MakeIOError(name, "Could not create directory.", kCreateDir, error);
    850   RecordOSError(kCreateDir, error);
    851   return result;
    852 }
    853 
    854 Status ChromiumEnv::DeleteDir(const std::string& name) {
    855   Status result;
    856   // TODO(jorlow): Should we assert this is a directory?
    857   if (!::base::DeleteFile(CreateFilePath(name), false)) {
    858     result = MakeIOError(name, "Could not delete directory.", kDeleteDir);
    859     RecordErrorAt(kDeleteDir);
    860   }
    861   return result;
    862 }
    863 
    864 Status ChromiumEnv::GetFileSize(const std::string& fname, uint64_t* size) {
    865   Status s;
    866   int64_t signed_size;
    867   if (!::base::GetFileSize(CreateFilePath(fname), &signed_size)) {
    868     *size = 0;
    869     s = MakeIOError(fname, "Could not determine file size.", kGetFileSize);
    870     RecordErrorAt(kGetFileSize);
    871   } else {
    872     *size = static_cast<uint64_t>(signed_size);
    873   }
    874   return s;
    875 }
    876 
    877 Status ChromiumEnv::RenameFile(const std::string& src, const std::string& dst) {
    878   Status result;
    879   base::FilePath src_file_path = CreateFilePath(src);
    880   if (!::base::PathExists(src_file_path))
    881     return result;
    882   base::FilePath destination = CreateFilePath(dst);
    883 
    884   Retrier retrier(kRenameFile, this);
    885   base::PlatformFileError error = base::PLATFORM_FILE_OK;
    886   do {
    887     if (base::ReplaceFile(src_file_path, destination, &error))
    888       return result;
    889   } while (retrier.ShouldKeepTrying(error));
    890 
    891   DCHECK(error != base::PLATFORM_FILE_OK);
    892   RecordOSError(kRenameFile, error);
    893   char buf[100];
    894   snprintf(buf,
    895            sizeof(buf),
    896            "Could not rename file: %s",
    897            PlatformFileErrorString(error));
    898   return MakeIOError(src, buf, kRenameFile, error);
    899 }
    900 
    901 Status ChromiumEnv::LockFile(const std::string& fname, FileLock** lock) {
    902   *lock = NULL;
    903   Status result;
    904   int flags = ::base::PLATFORM_FILE_OPEN_ALWAYS |
    905               ::base::PLATFORM_FILE_READ |
    906               ::base::PLATFORM_FILE_WRITE;
    907   bool created;
    908   ::base::PlatformFileError error_code;
    909   ::base::PlatformFile file;
    910   Retrier retrier(kLockFile, this);
    911   do {
    912     file = ::base::CreatePlatformFile(
    913         CreateFilePath(fname), flags, &created, &error_code);
    914   } while (error_code != ::base::PLATFORM_FILE_OK &&
    915            retrier.ShouldKeepTrying(error_code));
    916 
    917   if (error_code == ::base::PLATFORM_FILE_ERROR_NOT_FOUND) {
    918     ::base::FilePath parent = CreateFilePath(fname).DirName();
    919     ::base::FilePath last_parent;
    920     int num_missing_ancestors = 0;
    921     do {
    922       if (base::DirectoryExists(parent))
    923         break;
    924       ++num_missing_ancestors;
    925       last_parent = parent;
    926       parent = parent.DirName();
    927     } while (parent != last_parent);
    928     RecordLockFileAncestors(num_missing_ancestors);
    929   }
    930 
    931   if (error_code != ::base::PLATFORM_FILE_OK) {
    932     result = MakeIOError(
    933         fname, PlatformFileErrorString(error_code), kLockFile, error_code);
    934     RecordOSError(kLockFile, error_code);
    935     return result;
    936   }
    937 
    938   if (!locks_.Insert(fname)) {
    939     result = MakeIOError(fname, "Lock file already locked.", kLockFile);
    940     ::base::ClosePlatformFile(file);
    941     return result;
    942   }
    943 
    944   Retrier lock_retrier = Retrier(kLockFile, this);
    945   do {
    946     error_code = ::base::LockPlatformFile(file);
    947   } while (error_code != ::base::PLATFORM_FILE_OK &&
    948            retrier.ShouldKeepTrying(error_code));
    949 
    950   if (error_code != ::base::PLATFORM_FILE_OK) {
    951     ::base::ClosePlatformFile(file);
    952     locks_.Remove(fname);
    953     result = MakeIOError(
    954         fname, PlatformFileErrorString(error_code), kLockFile, error_code);
    955     RecordOSError(kLockFile, error_code);
    956     return result;
    957   }
    958 
    959   ChromiumFileLock* my_lock = new ChromiumFileLock;
    960   my_lock->file_ = file;
    961   my_lock->name_ = fname;
    962   *lock = my_lock;
    963   return result;
    964 }
    965 
    966 Status ChromiumEnv::UnlockFile(FileLock* lock) {
    967   ChromiumFileLock* my_lock = reinterpret_cast<ChromiumFileLock*>(lock);
    968   Status result;
    969 
    970   ::base::PlatformFileError error_code =
    971       ::base::UnlockPlatformFile(my_lock->file_);
    972   if (error_code != ::base::PLATFORM_FILE_OK) {
    973     result =
    974         MakeIOError(my_lock->name_, "Could not unlock lock file.", kUnlockFile);
    975     RecordOSError(kUnlockFile, error_code);
    976     ::base::ClosePlatformFile(my_lock->file_);
    977   } else if (!::base::ClosePlatformFile(my_lock->file_)) {
    978     result =
    979         MakeIOError(my_lock->name_, "Could not close lock file.", kUnlockFile);
    980     RecordErrorAt(kUnlockFile);
    981   }
    982   bool removed = locks_.Remove(my_lock->name_);
    983   DCHECK(removed);
    984   delete my_lock;
    985   return result;
    986 }
    987 
    988 Status ChromiumEnv::GetTestDirectory(std::string* path) {
    989   mu_.Acquire();
    990   if (test_directory_.empty()) {
    991     if (!base::CreateNewTempDirectory(kLevelDBTestDirectoryPrefix,
    992                                       &test_directory_)) {
    993       mu_.Release();
    994       RecordErrorAt(kGetTestDirectory);
    995       return MakeIOError(
    996           "Could not create temp directory.", "", kGetTestDirectory);
    997     }
    998   }
    999   *path = FilePathToString(test_directory_);
   1000   mu_.Release();
   1001   return Status::OK();
   1002 }
   1003 
   1004 Status ChromiumEnv::NewLogger(const std::string& fname, Logger** result) {
   1005   FILE* f = fopen_internal(fname.c_str(), "w");
   1006   if (f == NULL) {
   1007     *result = NULL;
   1008     int saved_errno = errno;
   1009     RecordOSError(kNewLogger, saved_errno);
   1010     return MakeIOError(fname, strerror(saved_errno), kNewLogger, saved_errno);
   1011   } else {
   1012     *result = new ChromiumLogger(f);
   1013     return Status::OK();
   1014   }
   1015 }
   1016 
   1017 uint64_t ChromiumEnv::NowMicros() {
   1018   return ::base::TimeTicks::Now().ToInternalValue();
   1019 }
   1020 
   1021 void ChromiumEnv::SleepForMicroseconds(int micros) {
   1022   // Round up to the next millisecond.
   1023   ::base::PlatformThread::Sleep(::base::TimeDelta::FromMicroseconds(micros));
   1024 }
   1025 
   1026 void ChromiumEnv::RecordErrorAt(MethodID method) const {
   1027   GetMethodIOErrorHistogram()->Add(method);
   1028 }
   1029 
   1030 void ChromiumEnv::RecordLockFileAncestors(int num_missing_ancestors) const {
   1031   GetLockFileAncestorHistogram()->Add(num_missing_ancestors);
   1032 }
   1033 
   1034 void ChromiumEnv::RecordOSError(MethodID method,
   1035                                 base::PlatformFileError error) const {
   1036   DCHECK(error < 0);
   1037   RecordErrorAt(method);
   1038   GetOSErrorHistogram(method, -base::PLATFORM_FILE_ERROR_MAX)->Add(-error);
   1039 }
   1040 
   1041 void ChromiumEnv::RecordOSError(MethodID method, int error) const {
   1042   DCHECK(error > 0);
   1043   RecordErrorAt(method);
   1044   GetOSErrorHistogram(method, ERANGE + 1)->Add(error);
   1045 }
   1046 
   1047 void ChromiumEnv::RecordBackupResult(bool result) const {
   1048   std::string uma_name(name_);
   1049   uma_name.append(".TableBackup");
   1050   base::BooleanHistogram::FactoryGet(
   1051       uma_name, base::Histogram::kUmaTargetedHistogramFlag)->AddBoolean(result);
   1052 }
   1053 
   1054 base::HistogramBase* ChromiumEnv::GetOSErrorHistogram(MethodID method,
   1055                                                       int limit) const {
   1056   std::string uma_name(name_);
   1057   // TODO(dgrogan): This is probably not the best way to concatenate strings.
   1058   uma_name.append(".IOError.").append(MethodIDToString(method));
   1059   return base::LinearHistogram::FactoryGet(uma_name, 1, limit, limit + 1,
   1060       base::Histogram::kUmaTargetedHistogramFlag);
   1061 }
   1062 
   1063 base::HistogramBase* ChromiumEnv::GetMethodIOErrorHistogram() const {
   1064   std::string uma_name(name_);
   1065   uma_name.append(".IOError");
   1066   return base::LinearHistogram::FactoryGet(uma_name, 1, kNumEntries,
   1067       kNumEntries + 1, base::Histogram::kUmaTargetedHistogramFlag);
   1068 }
   1069 
   1070 base::HistogramBase* ChromiumEnv::GetMaxFDHistogram(
   1071     const std::string& type) const {
   1072   std::string uma_name(name_);
   1073   uma_name.append(".MaxFDs.").append(type);
   1074   // These numbers make each bucket twice as large as the previous bucket.
   1075   const int kFirstEntry = 1;
   1076   const int kLastEntry = 65536;
   1077   const int kNumBuckets = 18;
   1078   return base::Histogram::FactoryGet(
   1079       uma_name, kFirstEntry, kLastEntry, kNumBuckets,
   1080       base::Histogram::kUmaTargetedHistogramFlag);
   1081 }
   1082 
   1083 base::HistogramBase* ChromiumEnv::GetLockFileAncestorHistogram() const {
   1084   std::string uma_name(name_);
   1085   uma_name.append(".LockFileAncestorsNotFound");
   1086   const int kMin = 1;
   1087   const int kMax = 10;
   1088   const int kNumBuckets = 11;
   1089   return base::LinearHistogram::FactoryGet(
   1090       uma_name, kMin, kMax, kNumBuckets,
   1091       base::Histogram::kUmaTargetedHistogramFlag);
   1092 }
   1093 
   1094 base::HistogramBase* ChromiumEnv::GetRetryTimeHistogram(MethodID method) const {
   1095   std::string uma_name(name_);
   1096   // TODO(dgrogan): This is probably not the best way to concatenate strings.
   1097   uma_name.append(".TimeUntilSuccessFor").append(MethodIDToString(method));
   1098 
   1099   const int kBucketSizeMillis = 25;
   1100   // Add 2, 1 for each of the buckets <1 and >max.
   1101   const int kNumBuckets = kMaxRetryTimeMillis / kBucketSizeMillis + 2;
   1102   return base::Histogram::FactoryTimeGet(
   1103       uma_name, base::TimeDelta::FromMilliseconds(1),
   1104       base::TimeDelta::FromMilliseconds(kMaxRetryTimeMillis + 1),
   1105       kNumBuckets,
   1106       base::Histogram::kUmaTargetedHistogramFlag);
   1107 }
   1108 
   1109 base::HistogramBase* ChromiumEnv::GetRecoveredFromErrorHistogram(
   1110     MethodID method) const {
   1111   std::string uma_name(name_);
   1112   uma_name.append(".RetryRecoveredFromErrorIn")
   1113       .append(MethodIDToString(method));
   1114   return base::LinearHistogram::FactoryGet(uma_name, 1, kNumEntries,
   1115       kNumEntries + 1, base::Histogram::kUmaTargetedHistogramFlag);
   1116 }
   1117 
   1118 class Thread : public ::base::PlatformThread::Delegate {
   1119  public:
   1120   Thread(void (*function)(void* arg), void* arg)
   1121       : function_(function), arg_(arg) {
   1122     ::base::PlatformThreadHandle handle;
   1123     bool success = ::base::PlatformThread::Create(0, this, &handle);
   1124     DCHECK(success);
   1125   }
   1126   virtual ~Thread() {}
   1127   virtual void ThreadMain() {
   1128     (*function_)(arg_);
   1129     delete this;
   1130   }
   1131 
   1132  private:
   1133   void (*function_)(void* arg);
   1134   void* arg_;
   1135 };
   1136 
   1137 void ChromiumEnv::Schedule(void (*function)(void*), void* arg) {
   1138   mu_.Acquire();
   1139 
   1140   // Start background thread if necessary
   1141   if (!started_bgthread_) {
   1142     started_bgthread_ = true;
   1143     StartThread(&ChromiumEnv::BGThreadWrapper, this);
   1144   }
   1145 
   1146   // If the queue is currently empty, the background thread may currently be
   1147   // waiting.
   1148   if (queue_.empty()) {
   1149     bgsignal_.Signal();
   1150   }
   1151 
   1152   // Add to priority queue
   1153   queue_.push_back(BGItem());
   1154   queue_.back().function = function;
   1155   queue_.back().arg = arg;
   1156 
   1157   mu_.Release();
   1158 }
   1159 
   1160 void ChromiumEnv::BGThread() {
   1161   base::PlatformThread::SetName(name_.c_str());
   1162 
   1163   while (true) {
   1164     // Wait until there is an item that is ready to run
   1165     mu_.Acquire();
   1166     while (queue_.empty()) {
   1167       bgsignal_.Wait();
   1168     }
   1169 
   1170     void (*function)(void*) = queue_.front().function;
   1171     void* arg = queue_.front().arg;
   1172     queue_.pop_front();
   1173 
   1174     mu_.Release();
   1175     TRACE_EVENT0("leveldb", "ChromiumEnv::BGThread-Task");
   1176     (*function)(arg);
   1177   }
   1178 }
   1179 
   1180 void ChromiumEnv::StartThread(void (*function)(void* arg), void* arg) {
   1181   new Thread(function, arg); // Will self-delete.
   1182 }
   1183 
   1184 static std::string GetDirName(const std::string& filename) {
   1185   base::FilePath file = base::FilePath::FromUTF8Unsafe(filename);
   1186   return FilePathToString(file.DirName());
   1187 }
   1188 
   1189 void ChromiumEnv::DidCreateNewFile(const std::string& filename) {
   1190   base::AutoLock auto_lock(map_lock_);
   1191   needs_sync_map_[GetDirName(filename)] = true;
   1192 }
   1193 
   1194 bool ChromiumEnv::DoesDirNeedSync(const std::string& filename) {
   1195   base::AutoLock auto_lock(map_lock_);
   1196   return needs_sync_map_.find(GetDirName(filename)) != needs_sync_map_.end();
   1197 }
   1198 
   1199 void ChromiumEnv::DidSyncDir(const std::string& filename) {
   1200   base::AutoLock auto_lock(map_lock_);
   1201   needs_sync_map_.erase(GetDirName(filename));
   1202 }
   1203 
   1204 }  // namespace leveldb_env
   1205 
   1206 namespace leveldb {
   1207 
   1208 Env* IDBEnv() {
   1209   return leveldb_env::idb_env.Pointer();
   1210 }
   1211 
   1212 Env* Env::Default() {
   1213   return leveldb_env::default_env.Pointer();
   1214 }
   1215 
   1216 }  // namespace leveldb
   1217 
   1218