Home | History | Annotate | Download | only in leveldb 
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
      6 
      7 #include <cerrno>
      8 
      9 #include "base/basictypes.h"
     10 #include "base/files/file.h"
     11 #include "base/logging.h"
     12 #include "base/memory/scoped_ptr.h"
     13 #include "base/metrics/histogram.h"
     14 #include "base/strings/string16.h"
     15 #include "base/strings/string_piece.h"
     16 #include "base/strings/stringprintf.h"
     17 #include "base/strings/utf_string_conversions.h"
     18 #include "base/sys_info.h"
     19 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
     20 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
     21 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
     22 #include "third_party/leveldatabase/env_chromium.h"
     23 #include "third_party/leveldatabase/env_idb.h"
     24 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
     25 #include "third_party/leveldatabase/src/include/leveldb/db.h"
     26 #include "third_party/leveldatabase/src/include/leveldb/env.h"
     27 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
     28 
     29 using base::StringPiece;
     30 
     31 namespace content {
     32 
     33 // Forcing flushes to disk at the end of a transaction guarantees that the
     34 // data hit disk, but drastically impacts throughput when the filesystem is
     35 // busy with background compactions. Not syncing trades off reliability for
     36 // performance. Note that background compactions which move data from the
     37 // log to SSTs are always done with reliable writes.
     38 //
     39 // Sync writes are necessary on Windows for quota calculations; POSIX
     40 // calculates file sizes correctly even when not synced to disk.
     41 #if defined(OS_WIN)
     42 static const bool kSyncWrites = true;
     43 #else
     44 // TODO(dgrogan): Either remove the #if block or change this back to false.
     45 // See http://crbug.com/338385.
     46 static const bool kSyncWrites = true;
     47 #endif
     48 
     49 static leveldb::Slice MakeSlice(const StringPiece& s) {
     50   return leveldb::Slice(s.begin(), s.size());
     51 }
     52 
     53 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
     54   return StringPiece(s.data(), s.size());
     55 }
     56 
     57 LevelDBDatabase::ComparatorAdapter::ComparatorAdapter(
     58     const LevelDBComparator* comparator)
     59     : comparator_(comparator) {}
     60 
     61 int LevelDBDatabase::ComparatorAdapter::Compare(const leveldb::Slice& a,
     62                                                 const leveldb::Slice& b) const {
     63   return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
     64 }
     65 
     66 const char* LevelDBDatabase::ComparatorAdapter::Name() const {
     67   return comparator_->Name();
     68 }
     69 
     70 // TODO(jsbell): Support the methods below in the future.
     71 void LevelDBDatabase::ComparatorAdapter::FindShortestSeparator(
     72     std::string* start,
     73     const leveldb::Slice& limit) const {}
     74 
     75 void LevelDBDatabase::ComparatorAdapter::FindShortSuccessor(
     76     std::string* key) const {}
     77 
     78 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
     79     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
     80 
     81 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
     82 
     83 LevelDBDatabase::LevelDBDatabase() {}
     84 
     85 LevelDBDatabase::~LevelDBDatabase() {
     86   // db_'s destructor uses comparator_adapter_; order of deletion is important.
     87   db_.reset();
     88   comparator_adapter_.reset();
     89   env_.reset();
     90 }
     91 
     92 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
     93                               leveldb::Env* env,
     94                               const base::FilePath& path,
     95                               leveldb::DB** db) {
     96   leveldb::Options options;
     97   options.comparator = comparator;
     98   options.create_if_missing = true;
     99   options.paranoid_checks = true;
    100   options.compression = leveldb::kSnappyCompression;
    101 
    102   // For info about the troubles we've run into with this parameter, see:
    103   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
    104   options.max_open_files = 80;
    105   options.env = env;
    106 
    107   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
    108   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
    109 }
    110 
    111 leveldb::Status LevelDBDatabase::Destroy(const base::FilePath& file_name) {
    112   leveldb::Options options;
    113   options.env = leveldb::IDBEnv();
    114   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
    115   return leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
    116 }
    117 
    118 namespace {
    119 class LockImpl : public LevelDBLock {
    120  public:
    121   explicit LockImpl(leveldb::Env* env, leveldb::FileLock* lock)
    122       : env_(env), lock_(lock) {}
    123   virtual ~LockImpl() { env_->UnlockFile(lock_); }
    124  private:
    125   leveldb::Env* env_;
    126   leveldb::FileLock* lock_;
    127 
    128   DISALLOW_COPY_AND_ASSIGN(LockImpl);
    129 };
    130 }  // namespace
    131 
    132 scoped_ptr<LevelDBLock> LevelDBDatabase::LockForTesting(
    133     const base::FilePath& file_name) {
    134   leveldb::Env* env = leveldb::IDBEnv();
    135   base::FilePath lock_path = file_name.AppendASCII("LOCK");
    136   leveldb::FileLock* lock = NULL;
    137   leveldb::Status status = env->LockFile(lock_path.AsUTF8Unsafe(), &lock);
    138   if (!status.ok())
    139     return scoped_ptr<LevelDBLock>();
    140   DCHECK(lock);
    141   return scoped_ptr<LevelDBLock>(new LockImpl(env, lock));
    142 }
    143 
    144 static int CheckFreeSpace(const char* const type,
    145                           const base::FilePath& file_name) {
    146   std::string name =
    147       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
    148   int64 free_disk_space_in_k_bytes =
    149       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
    150   if (free_disk_space_in_k_bytes < 0) {
    151     base::Histogram::FactoryGet(
    152         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
    153         1,
    154         2 /*boundary*/,
    155         2 /*boundary*/ + 1,
    156         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
    157     return -1;
    158   }
    159   int clamped_disk_space_k_bytes = free_disk_space_in_k_bytes > INT_MAX
    160                                        ? INT_MAX
    161                                        : free_disk_space_in_k_bytes;
    162   const uint64 histogram_max = static_cast<uint64>(1e9);
    163   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
    164   base::Histogram::FactoryGet(name,
    165                               1,
    166                               histogram_max,
    167                               11 /*buckets*/,
    168                               base::HistogramBase::kUmaTargetedHistogramFlag)
    169       ->Add(clamped_disk_space_k_bytes);
    170   return clamped_disk_space_k_bytes;
    171 }
    172 
    173 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
    174                                             const leveldb::Status& s) {
    175   leveldb_env::MethodID method;
    176   int error = -1;
    177   leveldb_env::ErrorParsingResult result =
    178       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
    179   if (result == leveldb_env::NONE)
    180     return;
    181   std::string method_histogram_name(histogram_name);
    182   method_histogram_name.append(".EnvMethod");
    183   base::LinearHistogram::FactoryGet(
    184       method_histogram_name,
    185       1,
    186       leveldb_env::kNumEntries,
    187       leveldb_env::kNumEntries + 1,
    188       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
    189 
    190   std::string error_histogram_name(histogram_name);
    191 
    192   if (result == leveldb_env::METHOD_AND_PFE) {
    193     DCHECK_LT(error, 0);
    194     error_histogram_name.append(std::string(".PFE.") +
    195                                 leveldb_env::MethodIDToString(method));
    196     base::LinearHistogram::FactoryGet(
    197         error_histogram_name,
    198         1,
    199         -base::File::FILE_ERROR_MAX,
    200         -base::File::FILE_ERROR_MAX + 1,
    201         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
    202   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
    203     error_histogram_name.append(std::string(".Errno.") +
    204                                 leveldb_env::MethodIDToString(method));
    205     base::LinearHistogram::FactoryGet(
    206         error_histogram_name,
    207         1,
    208         ERANGE + 1,
    209         ERANGE + 2,
    210         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
    211   }
    212 }
    213 
    214 static void ParseAndHistogramCorruptionDetails(
    215     const std::string& histogram_name,
    216     const leveldb::Status& status) {
    217   int error = leveldb_env::GetCorruptionCode(status);
    218   DCHECK_GE(error, 0);
    219   std::string corruption_histogram_name(histogram_name);
    220   corruption_histogram_name.append(".Corruption");
    221   const int kNumPatterns = leveldb_env::GetNumCorruptionCodes();
    222   base::LinearHistogram::FactoryGet(
    223       corruption_histogram_name,
    224       1,
    225       kNumPatterns,
    226       kNumPatterns + 1,
    227       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
    228 }
    229 
    230 static void HistogramLevelDBError(const std::string& histogram_name,
    231                                   const leveldb::Status& s) {
    232   if (s.ok()) {
    233     NOTREACHED();
    234     return;
    235   }
    236   enum {
    237     LEVEL_DB_NOT_FOUND,
    238     LEVEL_DB_CORRUPTION,
    239     LEVEL_DB_IO_ERROR,
    240     LEVEL_DB_OTHER,
    241     LEVEL_DB_MAX_ERROR
    242   };
    243   int leveldb_error = LEVEL_DB_OTHER;
    244   if (s.IsNotFound())
    245     leveldb_error = LEVEL_DB_NOT_FOUND;
    246   else if (s.IsCorruption())
    247     leveldb_error = LEVEL_DB_CORRUPTION;
    248   else if (s.IsIOError())
    249     leveldb_error = LEVEL_DB_IO_ERROR;
    250   base::Histogram::FactoryGet(histogram_name,
    251                               1,
    252                               LEVEL_DB_MAX_ERROR,
    253                               LEVEL_DB_MAX_ERROR + 1,
    254                               base::HistogramBase::kUmaTargetedHistogramFlag)
    255       ->Add(leveldb_error);
    256   if (s.IsIOError())
    257     ParseAndHistogramIOErrorDetails(histogram_name, s);
    258   else
    259     ParseAndHistogramCorruptionDetails(histogram_name, s);
    260 }
    261 
    262 leveldb::Status LevelDBDatabase::Open(const base::FilePath& file_name,
    263                                       const LevelDBComparator* comparator,
    264                                       scoped_ptr<LevelDBDatabase>* result,
    265                                       bool* is_disk_full) {
    266   scoped_ptr<ComparatorAdapter> comparator_adapter(
    267       new ComparatorAdapter(comparator));
    268 
    269   leveldb::DB* db;
    270   const leveldb::Status s =
    271       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
    272 
    273   if (!s.ok()) {
    274     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
    275     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
    276     // Disks with <100k of free space almost never succeed in opening a
    277     // leveldb database.
    278     if (is_disk_full)
    279       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
    280 
    281     LOG(ERROR) << "Failed to open LevelDB database from "
    282                << file_name.AsUTF8Unsafe() << "," << s.ToString();
    283     return s;
    284   }
    285 
    286   CheckFreeSpace("Success", file_name);
    287 
    288   (*result).reset(new LevelDBDatabase);
    289   (*result)->db_ = make_scoped_ptr(db);
    290   (*result)->comparator_adapter_ = comparator_adapter.Pass();
    291   (*result)->comparator_ = comparator;
    292 
    293   return s;
    294 }
    295 
    296 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
    297     const LevelDBComparator* comparator) {
    298   scoped_ptr<ComparatorAdapter> comparator_adapter(
    299       new ComparatorAdapter(comparator));
    300   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
    301 
    302   leveldb::DB* db;
    303   const leveldb::Status s = OpenDB(
    304       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
    305 
    306   if (!s.ok()) {
    307     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
    308     return scoped_ptr<LevelDBDatabase>();
    309   }
    310 
    311   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
    312   result->env_ = in_memory_env.Pass();
    313   result->db_ = make_scoped_ptr(db);
    314   result->comparator_adapter_ = comparator_adapter.Pass();
    315   result->comparator_ = comparator;
    316 
    317   return result.Pass();
    318 }
    319 
    320 leveldb::Status LevelDBDatabase::Put(const StringPiece& key,
    321                                      std::string* value) {
    322   leveldb::WriteOptions write_options;
    323   write_options.sync = kSyncWrites;
    324 
    325   const leveldb::Status s =
    326       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
    327   if (!s.ok())
    328     LOG(ERROR) << "LevelDB put failed: " << s.ToString();
    329   return s;
    330 }
    331 
    332 leveldb::Status LevelDBDatabase::Remove(const StringPiece& key) {
    333   leveldb::WriteOptions write_options;
    334   write_options.sync = kSyncWrites;
    335 
    336   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
    337   if (!s.IsNotFound())
    338     LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
    339   return s;
    340 }
    341 
    342 leveldb::Status LevelDBDatabase::Get(const StringPiece& key,
    343                                      std::string* value,
    344                                      bool* found,
    345                                      const LevelDBSnapshot* snapshot) {
    346   *found = false;
    347   leveldb::ReadOptions read_options;
    348   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
    349                                          // performance impact is too great.
    350   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
    351 
    352   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
    353   if (s.ok()) {
    354     *found = true;
    355     return s;
    356   }
    357   if (s.IsNotFound())
    358     return leveldb::Status::OK();
    359   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
    360   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
    361   return s;
    362 }
    363 
    364 leveldb::Status LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
    365   leveldb::WriteOptions write_options;
    366   write_options.sync = kSyncWrites;
    367 
    368   const leveldb::Status s =
    369       db_->Write(write_options, write_batch.write_batch_.get());
    370   if (!s.ok()) {
    371     HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
    372     LOG(ERROR) << "LevelDB write failed: " << s.ToString();
    373   }
    374   return s;
    375 }
    376 
    377 namespace {
    378 class IteratorImpl : public LevelDBIterator {
    379  public:
    380   virtual ~IteratorImpl() {}
    381 
    382   virtual bool IsValid() const OVERRIDE;
    383   virtual leveldb::Status SeekToLast() OVERRIDE;
    384   virtual leveldb::Status Seek(const StringPiece& target) OVERRIDE;
    385   virtual leveldb::Status Next() OVERRIDE;
    386   virtual leveldb::Status Prev() OVERRIDE;
    387   virtual StringPiece Key() const OVERRIDE;
    388   virtual StringPiece Value() const OVERRIDE;
    389 
    390  private:
    391   friend class content::LevelDBDatabase;
    392   explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
    393   void CheckStatus();
    394 
    395   scoped_ptr<leveldb::Iterator> iterator_;
    396 
    397   DISALLOW_COPY_AND_ASSIGN(IteratorImpl);
    398 };
    399 }  // namespace
    400 
    401 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
    402     : iterator_(it.Pass()) {}
    403 
    404 void IteratorImpl::CheckStatus() {
    405   const leveldb::Status& s = iterator_->status();
    406   if (!s.ok())
    407     LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
    408 }
    409 
    410 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
    411 
    412 leveldb::Status IteratorImpl::SeekToLast() {
    413   iterator_->SeekToLast();
    414   CheckStatus();
    415   return iterator_->status();
    416 }
    417 
    418 leveldb::Status IteratorImpl::Seek(const StringPiece& target) {
    419   iterator_->Seek(MakeSlice(target));
    420   CheckStatus();
    421   return iterator_->status();
    422 }
    423 
    424 leveldb::Status IteratorImpl::Next() {
    425   DCHECK(IsValid());
    426   iterator_->Next();
    427   CheckStatus();
    428   return iterator_->status();
    429 }
    430 
    431 leveldb::Status IteratorImpl::Prev() {
    432   DCHECK(IsValid());
    433   iterator_->Prev();
    434   CheckStatus();
    435   return iterator_->status();
    436 }
    437 
    438 StringPiece IteratorImpl::Key() const {
    439   DCHECK(IsValid());
    440   return MakeStringPiece(iterator_->key());
    441 }
    442 
    443 StringPiece IteratorImpl::Value() const {
    444   DCHECK(IsValid());
    445   return MakeStringPiece(iterator_->value());
    446 }
    447 
    448 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
    449     const LevelDBSnapshot* snapshot) {
    450   leveldb::ReadOptions read_options;
    451   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
    452                                          // performance impact is too great.
    453   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
    454 
    455   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
    456   return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
    457 }
    458 
    459 const LevelDBComparator* LevelDBDatabase::Comparator() const {
    460   return comparator_;
    461 }
    462 
    463 void LevelDBDatabase::Compact(const base::StringPiece& start,
    464                               const base::StringPiece& stop) {
    465   const leveldb::Slice start_slice = MakeSlice(start);
    466   const leveldb::Slice stop_slice = MakeSlice(stop);
    467   db_->CompactRange(&start_slice, &stop_slice);
    468 }
    469 
    470 void LevelDBDatabase::CompactAll() { db_->CompactRange(NULL, NULL); }
    471 
    472 }  // namespace content
    473