Home | History | Annotate | Download | only in leveldb 
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
      6 
      7 #include <cerrno>
      8 
      9 #include "base/basictypes.h"
     10 #include "base/logging.h"
     11 #include "base/memory/scoped_ptr.h"
     12 #include "base/metrics/histogram.h"
     13 #include "base/strings/string16.h"
     14 #include "base/strings/string_piece.h"
     15 #include "base/strings/stringprintf.h"
     16 #include "base/strings/utf_string_conversions.h"
     17 #include "base/sys_info.h"
     18 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
     19 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
     20 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
     21 #include "third_party/leveldatabase/env_chromium.h"
     22 #include "third_party/leveldatabase/env_idb.h"
     23 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
     24 #include "third_party/leveldatabase/src/include/leveldb/comparator.h"
     25 #include "third_party/leveldatabase/src/include/leveldb/db.h"
     26 #include "third_party/leveldatabase/src/include/leveldb/env.h"
     27 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
     28 
     29 using base::StringPiece;
     30 
     31 namespace content {
     32 
     33 // Forcing flushes to disk at the end of a transaction guarantees that the
     34 // data hit disk, but drastically impacts throughput when the filesystem is
     35 // busy with background compactions. Not syncing trades off reliability for
     36 // performance. Note that background compactions which move data from the
     37 // log to SSTs are always done with reliable writes.
     38 //
     39 // Sync writes are necessary on Windows for quota calculations; POSIX
     40 // calculates file sizes correctly even when not synced to disk.
     41 #if defined(OS_WIN)
     42 static const bool kSyncWrites = true;
     43 #else
     44 // TODO(dgrogan): Either remove the #if block or change this back to false.
     45 // See http://crbug.com/338385.
     46 static const bool kSyncWrites = true;
     47 #endif
     48 
     49 static leveldb::Slice MakeSlice(const StringPiece& s) {
     50   return leveldb::Slice(s.begin(), s.size());
     51 }
     52 
     53 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
     54   return StringPiece(s.data(), s.size());
     55 }
     56 
     57 class ComparatorAdapter : public leveldb::Comparator {
     58  public:
     59   explicit ComparatorAdapter(const LevelDBComparator* comparator)
     60       : comparator_(comparator) {}
     61 
     62   virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
     63       OVERRIDE {
     64     return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
     65   }
     66 
     67   virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
     68 
     69   // TODO(jsbell): Support the methods below in the future.
     70   virtual void FindShortestSeparator(std::string* start,
     71                                      const leveldb::Slice& limit) const
     72       OVERRIDE {}
     73   virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
     74 
     75  private:
     76   const LevelDBComparator* comparator_;
     77 };
     78 
     79 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
     80     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
     81 
     82 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
     83 
     84 LevelDBDatabase::LevelDBDatabase() {}
     85 
     86 LevelDBDatabase::~LevelDBDatabase() {
     87   // db_'s destructor uses comparator_adapter_; order of deletion is important.
     88   db_.reset();
     89   comparator_adapter_.reset();
     90   env_.reset();
     91 }
     92 
     93 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
     94                               leveldb::Env* env,
     95                               const base::FilePath& path,
     96                               leveldb::DB** db) {
     97   leveldb::Options options;
     98   options.comparator = comparator;
     99   options.create_if_missing = true;
    100   options.paranoid_checks = true;
    101   options.compression = leveldb::kSnappyCompression;
    102 
    103   // For info about the troubles we've run into with this parameter, see:
    104   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
    105   options.max_open_files = 80;
    106   options.env = env;
    107 
    108   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
    109   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
    110 }
    111 
    112 bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
    113   leveldb::Options options;
    114   options.env = leveldb::IDBEnv();
    115   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
    116   const leveldb::Status s =
    117       leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
    118   return s.ok();
    119 }
    120 
    121 namespace {
    122 class LockImpl : public LevelDBLock {
    123  public:
    124   explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
    125       : env_(env), lock_(lock) {}
    126   virtual ~LockImpl() { env_->UnlockFile(lock_); }
    127  private:
    128   leveldb::Env* env_;
    129   leveldb::FileLock* lock_;
    130 };
    131 }
    132 
    133 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
    134     const base::FilePath& file_name) {
    135   leveldb::Env* env = leveldb::IDBEnv();
    136   base::FilePath lock_path = file_name.AppendASCII("LOCK");
    137   leveldb::FileLock* lock = NULL;
    138   leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
    139   if (!status.ok())
    140     return scoped_ptr<LevelDBLock>();
    141   DCHECK(lock);
    142   return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
    143 }
    144 
    145 static int CheckFreeSpace(const char* const type,
    146                           const base::FilePath& file_name) {
    147   std::string name =
    148       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
    149   int64 free_disk_space_in_k_bytes =
    150       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
    151   if (free_disk_space_in_k_bytes < 0) {
    152     base::Histogram::FactoryGet(
    153         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
    154         1,
    155         2 /*boundary*/,
    156         2 /*boundary*/ + 1,
    157         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
    158     return -1;
    159   }
    160   int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
    161                                        ? INT_MAX
    162                                        : free_disk_space_in_k_bytes;
    163   const uint64 histogram_max = static_cast<uint64>(1e9);
    164   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
    165   base::Histogram::FactoryGet(name,
    166                               1,
    167                               histogram_max,
    168                               11 /*buckets*/,
    169                               base::HistogramBase::kUmaTargetedHistogramFlag)
    170       ->Add(clamped_disk_space_k_bytes);
    171   return clamped_disk_space_k_bytes;
    172 }
    173 
    174 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
    175                                             const leveldb::Status& s) {
    176   leveldb_env::MethodID method;
    177   int error = -1;
    178   leveldb_env::ErrorParsingResult result =
    179       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
    180   if (result == leveldb_env::NONE)
    181     return;
    182   std::string method_histogram_name(histogram_name);
    183   method_histogram_name.append(".EnvMethod");
    184   base::LinearHistogram::FactoryGet(
    185       method_histogram_name,
    186       1,
    187       leveldb_env::kNumEntries,
    188       leveldb_env::kNumEntries + 1,
    189       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
    190 
    191   std::string error_histogram_name(histogram_name);
    192 
    193   if (result == leveldb_env::METHOD_AND_PFE) {
    194     DCHECK(error < 0);
    195     error_histogram_name.append(std::string(".PFE.") +
    196                                 leveldb_env::MethodIDToString(method));
    197     base::LinearHistogram::FactoryGet(
    198         error_histogram_name,
    199         1,
    200         -base::PLATFORM_FILE_ERROR_MAX,
    201         -base::PLATFORM_FILE_ERROR_MAX + 1,
    202         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
    203   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
    204     error_histogram_name.append(std::string(".Errno.") +
    205                                 leveldb_env::MethodIDToString(method));
    206     base::LinearHistogram::FactoryGet(
    207         error_histogram_name,
    208         1,
    209         ERANGE + 1,
    210         ERANGE + 2,
    211         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
    212   }
    213 }
    214 
    215 static void ParseAndHistogramCorruptionDetails(
    216     const std::string& histogram_name,
    217     const leveldb::Status& status) {
    218   int error = leveldb_env::GetCorruptionCode(status);
    219   DCHECK(error >= 0);
    220   std::string corruption_histogram_name(histogram_name);
    221   corruption_histogram_name.append(".Corruption");
    222   const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
    223   base::LinearHistogram::FactoryGet(
    224       corruption_histogram_name,
    225       1,
    226       kNumPatterns,
    227       kNumPatterns + 1,
    228       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
    229 }
    230 
    231 static void HistogramLevelDBError(const std::string& histogram_name,
    232                                   const leveldb::Status& s) {
    233   if (s.ok()) {
    234     NOTREACHED();
    235     return;
    236   }
    237   enum {
    238     LEVEL_DB_NOT_FOUND,
    239     LEVEL_DB_CORRUPTION,
    240     LEVEL_DB_IO_ERROR,
    241     LEVEL_DB_OTHER,
    242     LEVEL_DB_MAX_ERROR
    243   };
    244   int leveldb_error = LEVEL_DB_OTHER;
    245   if (s.IsNotFound())
    246     leveldb_error = LEVEL_DB_NOT_FOUND;
    247   else if (s.IsCorruption())
    248     leveldb_error = LEVEL_DB_CORRUPTION;
    249   else if (s.IsIOError())
    250     leveldb_error = LEVEL_DB_IO_ERROR;
    251   base::Histogram::FactoryGet(histogram_name,
    252                               1,
    253                               LEVEL_DB_MAX_ERROR,
    254                               LEVEL_DB_MAX_ERROR + 1,
    255                               base::HistogramBase::kUmaTargetedHistogramFlag)
    256       ->Add(leveldb_error);
    257   if (s.IsIOError())
    258     ParseAndHistogramIOErrorDetails(histogram_name, s);
    259   else
    260     ParseAndHistogramCorruptionDetails(histogram_name, s);
    261 }
    262 
    263 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
    264                                       const LevelDBComparator* comparator,
    265                                       scoped_ptr<LevelDBDatabase>* result,
    266                                       bool* is_disk_full) {
    267   scoped_ptr<ComparatorAdapter> comparator_adapter(
    268       new ComparatorAdapter(comparator));
    269 
    270   leveldb::DB* db;
    271   const leveldb::Status s =
    272       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
    273 
    274   if (!s.ok()) {
    275     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
    276     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
    277     // Disks with <100k of free space almost never succeed in opening a
    278     // leveldb database.
    279     if (is_disk_full)
    280       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
    281 
    282     LOG(ERROR) << "Failed to open LevelDB database from "
    283                << file_name.AsUTF8Unsafe() << "," << s.ToString();
    284     return s;
    285   }
    286 
    287   CheckFreeSpace("Success", file_name);
    288 
    289   (*result).reset(new LevelDBDatabase);
    290   (*result)->db_ = make_scoped_ptr(db);
    291   (*result)->comparator_adapter_ = comparator_adapter.Pass();
    292   (*result)->comparator_ = comparator;
    293 
    294   return s;
    295 }
    296 
    297 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
    298     const LevelDBComparator* comparator) {
    299   scoped_ptr<ComparatorAdapter> comparator_adapter(
    300       new ComparatorAdapter(comparator));
    301   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
    302 
    303   leveldb::DB* db;
    304   const leveldb::Status s = OpenDB(
    305       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
    306 
    307   if (!s.ok()) {
    308     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
    309     return scoped_ptr<LevelDBDatabase>();
    310   }
    311 
    312   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
    313   result->env_ = in_memory_env.Pass();
    314   result->db_ = make_scoped_ptr(db);
    315   result->comparator_adapter_ = comparator_adapter.Pass();
    316   result->comparator_ = comparator;
    317 
    318   return result.Pass();
    319 }
    320 
    321 bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) {
    322   leveldb::WriteOptions write_options;
    323   write_options.sync = kSyncWrites;
    324 
    325   const leveldb::Status s =
    326       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
    327   if (s.ok())
    328     return true;
    329   LOG(ERROR) << "LevelDB put failed: " << s.ToString();
    330   return false;
    331 }
    332 
    333 bool LevelDBDatabase::Remove(const StringPiece& key) {
    334   leveldb::WriteOptions write_options;
    335   write_options.sync = kSyncWrites;
    336 
    337   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
    338   if (s.ok())
    339     return true;
    340   if (s.IsNotFound())
    341     return false;
    342   LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
    343   return false;
    344 }
    345 
    346 bool LevelDBDatabase::Get(const StringPiece& key,
    347                           std::string* value,
    348                           bool* found,
    349                           const LevelDBSnapshot* snapshot) {
    350   *found = false;
    351   leveldb::ReadOptions read_options;
    352   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
    353                                          // performance impact is too great.
    354   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
    355 
    356   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
    357   if (s.ok()) {
    358     *found = true;
    359     return true;
    360   }
    361   if (s.IsNotFound())
    362     return true;
    363   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
    364   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
    365   return false;
    366 }
    367 
    368 bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
    369   leveldb::WriteOptions write_options;
    370   write_options.sync = kSyncWrites;
    371 
    372   const leveldb::Status s =
    373       db_->Write(write_options, write_batch.write_batch_.get());
    374   if (s.ok())
    375     return true;
    376   HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
    377   LOG(ERROR) << "LevelDB write failed: " << s.ToString();
    378   return false;
    379 }
    380 
    381 namespace {
    382 class IteratorImpl : public LevelDBIterator {
    383  public:
    384   virtual ~IteratorImpl() {}
    385 
    386   virtual bool IsValid() const OVERRIDE;
    387   virtual void SeekToLast() OVERRIDE;
    388   virtual void Seek(const StringPiece& target) OVERRIDE;
    389   virtual void Next() OVERRIDE;
    390   virtual void Prev() OVERRIDE;
    391   virtual StringPiece Key() const OVERRIDE;
    392   virtual StringPiece Value() const OVERRIDE;
    393 
    394  private:
    395   friend class content::LevelDBDatabase;
    396   explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
    397   void CheckStatus();
    398 
    399   scoped_ptr<leveldb::Iterator> iterator_;
    400 };
    401 }
    402 
    403 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
    404     : iterator_(it.Pass()) {}
    405 
    406 void IteratorImpl::CheckStatus() {
    407   const leveldb::Status s = iterator_->status();
    408   if (!s.ok())
    409     LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
    410 }
    411 
    412 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
    413 
    414 void IteratorImpl::SeekToLast() {
    415   iterator_->SeekToLast();
    416   CheckStatus();
    417 }
    418 
    419 void IteratorImpl::Seek(const StringPiece& target) {
    420   iterator_->Seek(MakeSlice(target));
    421   CheckStatus();
    422 }
    423 
    424 void IteratorImpl::Next() {
    425   DCHECK(IsValid());
    426   iterator_->Next();
    427   CheckStatus();
    428 }
    429 
    430 void IteratorImpl::Prev() {
    431   DCHECK(IsValid());
    432   iterator_->Prev();
    433   CheckStatus();
    434 }
    435 
    436 StringPiece IteratorImpl::Key() const {
    437   DCHECK(IsValid());
    438   return MakeStringPiece(iterator_->key());
    439 }
    440 
    441 StringPiece IteratorImpl::Value() const {
    442   DCHECK(IsValid());
    443   return MakeStringPiece(iterator_->value());
    444 }
    445 
    446 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
    447     const LevelDBSnapshot* snapshot) {
    448   leveldb::ReadOptions read_options;
    449   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
    450                                          // performance impact is too great.
    451   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
    452 
    453   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
    454   return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
    455 }
    456 
    457 const LevelDBComparator* LevelDBDatabase::Comparator() const {
    458   return comparator_;
    459 }
    460 
    461 }  // namespace content
    462