Home | History | Annotate | Download | only in leveldb 
      1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/browser/indexed_db/leveldb/leveldb_database.h"
      6 
      7 #include <cerrno>
      8 
      9 #include "base/basictypes.h"
     10 #include "base/logging.h"
     11 #include "base/memory/scoped_ptr.h"
     12 #include "base/metrics/histogram.h"
     13 #include "base/strings/string16.h"
     14 #include "base/strings/string_piece.h"
     15 #include "base/strings/stringprintf.h"
     16 #include "base/strings/utf_string_conversions.h"
     17 #include "base/sys_info.h"
     18 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h"
     19 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h"
     20 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h"
     21 #include "third_party/leveldatabase/env_chromium.h"
     22 #include "third_party/leveldatabase/env_idb.h"
     23 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h"
     24 #include "third_party/leveldatabase/src/include/leveldb/comparator.h"
     25 #include "third_party/leveldatabase/src/include/leveldb/db.h"
     26 #include "third_party/leveldatabase/src/include/leveldb/env.h"
     27 #include "third_party/leveldatabase/src/include/leveldb/slice.h"
     28 
     29 using base::StringPiece;
     30 
     31 namespace content {
     32 
     33 static leveldb::Slice MakeSlice(const StringPiece& s) {
     34   return leveldb::Slice(s.begin(), s.size());
     35 }
     36 
     37 static StringPiece MakeStringPiece(const leveldb::Slice& s) {
     38   return StringPiece(s.data(), s.size());
     39 }
     40 
     41 class ComparatorAdapter : public leveldb::Comparator {
     42  public:
     43   explicit ComparatorAdapter(const LevelDBComparator* comparator)
     44       : comparator_(comparator) {}
     45 
     46   virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const
     47       OVERRIDE {
     48     return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b));
     49   }
     50 
     51   virtual const char* Name() const OVERRIDE { return comparator_->Name(); }
     52 
     53   // TODO(jsbell): Support the methods below in the future.
     54   virtual void FindShortestSeparator(std::string* start,
     55                                      const leveldb::Slice& limit) const
     56       OVERRIDE {}
     57   virtual void FindShortSuccessor(std::string* key) const OVERRIDE {}
     58 
     59  private:
     60   const LevelDBComparator* comparator_;
     61 };
     62 
     63 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db)
     64     : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {}
     65 
     66 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); }
     67 
     68 LevelDBDatabase::LevelDBDatabase() {}
     69 
     70 LevelDBDatabase::~LevelDBDatabase() {
     71   // db_'s destructor uses comparator_adapter_; order of deletion is important.
     72   db_.reset();
     73   comparator_adapter_.reset();
     74   env_.reset();
     75 }
     76 
     77 static leveldb::Status OpenDB(leveldb::Comparator* comparator,
     78                               leveldb::Env* env,
     79                               const base::FilePath& path,
     80                               leveldb::DB** db) {
     81   leveldb::Options options;
     82   options.comparator = comparator;
     83   options.create_if_missing = true;
     84   options.paranoid_checks = true;
     85 
     86   // Marking compression as explicitly off so snappy support can be
     87   // compiled in for other leveldb clients without implicitly enabling
     88   // it for IndexedDB. http://crbug.com/81384
     89   options.compression = leveldb::kNoCompression;
     90 
     91   // For info about the troubles we've run into with this parameter, see:
     92   // https://code.google.com/p/chromium/issues/detail?id=227313#c11
     93   options.max_open_files = 80;
     94   options.env = env;
     95 
     96   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
     97   return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db);
     98 }
     99 
    100 bool LevelDBDatabase::Destroy(const base::FilePath& file_name) {
    101   leveldb::Options options;
    102   options.env = leveldb::IDBEnv();
    103   // ChromiumEnv assumes UTF8, converts back to FilePath before using.
    104   const leveldb::Status s =
    105       leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options);
    106   return s.ok();
    107 }
    108 
    109 static int CheckFreeSpace(const char* const type,
    110                           const base::FilePath& file_name) {
    111   std::string name =
    112       std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace";
    113   int64 free_disk_space_in_k_bytes =
    114       base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024;
    115   if (free_disk_space_in_k_bytes < 0) {
    116     base::Histogram::FactoryGet(
    117         "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure",
    118         1,
    119         2 /*boundary*/,
    120         2 /*boundary*/ + 1,
    121         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/);
    122     return -1;
    123   }
    124   int clamped_disk_space_k_bytes =
    125       free_disk_space_in_k_bytes > INT_MAX ? INT_MAX
    126                                            : free_disk_space_in_k_bytes;
    127   const uint64 histogram_max = static_cast<uint64>(1e9);
    128   COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big);
    129   base::Histogram::FactoryGet(name,
    130                               1,
    131                               histogram_max,
    132                               11 /*buckets*/,
    133                               base::HistogramBase::kUmaTargetedHistogramFlag)
    134       ->Add(clamped_disk_space_k_bytes);
    135   return clamped_disk_space_k_bytes;
    136 }
    137 
    138 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name,
    139                                             const leveldb::Status& s) {
    140   leveldb_env::MethodID method;
    141   int error = -1;
    142   leveldb_env::ErrorParsingResult result =
    143       leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error);
    144   if (result == leveldb_env::NONE)
    145     return;
    146   std::string method_histogram_name(histogram_name);
    147   method_histogram_name.append(".EnvMethod");
    148   base::LinearHistogram::FactoryGet(
    149       method_histogram_name,
    150       1,
    151       leveldb_env::kNumEntries,
    152       leveldb_env::kNumEntries + 1,
    153       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method);
    154 
    155   std::string error_histogram_name(histogram_name);
    156 
    157   if (result == leveldb_env::METHOD_AND_PFE) {
    158     DCHECK(error < 0);
    159     error_histogram_name.append(std::string(".PFE.") +
    160                                 leveldb_env::MethodIDToString(method));
    161     base::LinearHistogram::FactoryGet(
    162         error_histogram_name,
    163         1,
    164         -base::PLATFORM_FILE_ERROR_MAX,
    165         -base::PLATFORM_FILE_ERROR_MAX + 1,
    166         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error);
    167   } else if (result == leveldb_env::METHOD_AND_ERRNO) {
    168     error_histogram_name.append(std::string(".Errno.") +
    169                                 leveldb_env::MethodIDToString(method));
    170     base::LinearHistogram::FactoryGet(
    171         error_histogram_name,
    172         1,
    173         ERANGE + 1,
    174         ERANGE + 2,
    175         base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
    176   }
    177 }
    178 
    179 static void ParseAndHistogramCorruptionDetails(
    180     const std::string& histogram_name,
    181     const leveldb::Status& status) {
    182   DCHECK(!status.IsIOError());
    183   DCHECK(!status.ok());
    184   const int kOtherError = 0;
    185   int error = kOtherError;
    186   const std::string& str_error = status.ToString();
    187   // Keep in sync with LevelDBCorruptionTypes in histograms.xml.
    188   const char* patterns[] = {
    189     "missing files",
    190     "log record too small",
    191     "corrupted internal key",
    192     "partial record",
    193     "missing start of fragmented record",
    194     "error in middle of record",
    195     "unknown record type",
    196     "truncated record at end",
    197     "bad record length",
    198     "VersionEdit",
    199     "FileReader invoked with unexpected value",
    200     "corrupted key",
    201     "CURRENT file does not end with newline",
    202     "no meta-nextfile entry",
    203     "no meta-lognumber entry",
    204     "no last-sequence-number entry",
    205     "malformed WriteBatch",
    206     "bad WriteBatch Put",
    207     "bad WriteBatch Delete",
    208     "unknown WriteBatch tag",
    209     "WriteBatch has wrong count",
    210     "bad entry in block",
    211     "bad block contents",
    212     "bad block handle",
    213     "truncated block read",
    214     "block checksum mismatch",
    215     "checksum mismatch",
    216     "corrupted compressed block contents",
    217     "bad block type",
    218     "bad magic number",
    219     "file is too short",
    220   };
    221   const size_t kNumPatterns = arraysize(patterns);
    222   for (size_t i = 0; i < kNumPatterns; ++i) {
    223     if (str_error.find(patterns[i]) != std::string::npos) {
    224       error = i + 1;
    225       break;
    226     }
    227   }
    228   DCHECK(error >= 0);
    229   std::string corruption_histogram_name(histogram_name);
    230   corruption_histogram_name.append(".Corruption");
    231   base::LinearHistogram::FactoryGet(
    232       corruption_histogram_name,
    233       1,
    234       kNumPatterns + 1,
    235       kNumPatterns + 2,
    236       base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error);
    237 }
    238 
    239 static void HistogramLevelDBError(const std::string& histogram_name,
    240                                   const leveldb::Status& s) {
    241   if (s.ok()) {
    242     NOTREACHED();
    243     return;
    244   }
    245   enum {
    246     LEVEL_DB_NOT_FOUND,
    247     LEVEL_DB_CORRUPTION,
    248     LEVEL_DB_IO_ERROR,
    249     LEVEL_DB_OTHER,
    250     LEVEL_DB_MAX_ERROR
    251   };
    252   int leveldb_error = LEVEL_DB_OTHER;
    253   if (s.IsNotFound())
    254     leveldb_error = LEVEL_DB_NOT_FOUND;
    255   else if (s.IsCorruption())
    256     leveldb_error = LEVEL_DB_CORRUPTION;
    257   else if (s.IsIOError())
    258     leveldb_error = LEVEL_DB_IO_ERROR;
    259   base::Histogram::FactoryGet(histogram_name,
    260                               1,
    261                               LEVEL_DB_MAX_ERROR,
    262                               LEVEL_DB_MAX_ERROR + 1,
    263                               base::HistogramBase::kUmaTargetedHistogramFlag)
    264       ->Add(leveldb_error);
    265   if (s.IsIOError())
    266     ParseAndHistogramIOErrorDetails(histogram_name, s);
    267   else
    268     ParseAndHistogramCorruptionDetails(histogram_name, s);
    269 }
    270 
    271 leveldb::Status LevelDBDatabase::Open(
    272     const base::FilePath& file_name,
    273     const LevelDBComparator* comparator,
    274     scoped_ptr<LevelDBDatabase>* result,
    275     bool* is_disk_full) {
    276   scoped_ptr<ComparatorAdapter> comparator_adapter(
    277       new ComparatorAdapter(comparator));
    278 
    279   leveldb::DB* db;
    280   const leveldb::Status s =
    281       OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db);
    282 
    283   if (!s.ok()) {
    284     HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s);
    285     int free_space_k_bytes = CheckFreeSpace("Failure", file_name);
    286     // Disks with <100k of free space almost never succeed in opening a
    287     // leveldb database.
    288     if (is_disk_full)
    289       *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100;
    290 
    291     LOG(ERROR) << "Failed to open LevelDB database from "
    292                << file_name.AsUTF8Unsafe() << "," << s.ToString();
    293     return s;
    294   }
    295 
    296   CheckFreeSpace("Success", file_name);
    297 
    298   (*result).reset(new LevelDBDatabase);
    299   (*result)->db_ = make_scoped_ptr(db);
    300   (*result)->comparator_adapter_ = comparator_adapter.Pass();
    301   (*result)->comparator_ = comparator;
    302 
    303   return s;
    304 }
    305 
    306 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory(
    307     const LevelDBComparator* comparator) {
    308   scoped_ptr<ComparatorAdapter> comparator_adapter(
    309       new ComparatorAdapter(comparator));
    310   scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv()));
    311 
    312   leveldb::DB* db;
    313   const leveldb::Status s = OpenDB(
    314       comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db);
    315 
    316   if (!s.ok()) {
    317     LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString();
    318     return scoped_ptr<LevelDBDatabase>();
    319   }
    320 
    321   scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase);
    322   result->env_ = in_memory_env.Pass();
    323   result->db_ = make_scoped_ptr(db);
    324   result->comparator_adapter_ = comparator_adapter.Pass();
    325   result->comparator_ = comparator;
    326 
    327   return result.Pass();
    328 }
    329 
    330 bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) {
    331   leveldb::WriteOptions write_options;
    332   write_options.sync = true;
    333 
    334   const leveldb::Status s =
    335       db_->Put(write_options, MakeSlice(key), MakeSlice(*value));
    336   if (s.ok())
    337     return true;
    338   LOG(ERROR) << "LevelDB put failed: " << s.ToString();
    339   return false;
    340 }
    341 
    342 bool LevelDBDatabase::Remove(const StringPiece& key) {
    343   leveldb::WriteOptions write_options;
    344   write_options.sync = true;
    345 
    346   const leveldb::Status s = db_->Delete(write_options, MakeSlice(key));
    347   if (s.ok())
    348     return true;
    349   if (s.IsNotFound())
    350     return false;
    351   LOG(ERROR) << "LevelDB remove failed: " << s.ToString();
    352   return false;
    353 }
    354 
    355 bool LevelDBDatabase::Get(const StringPiece& key,
    356                           std::string* value,
    357                           bool* found,
    358                           const LevelDBSnapshot* snapshot) {
    359   *found = false;
    360   leveldb::ReadOptions read_options;
    361   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
    362                                          // performance impact is too great.
    363   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
    364 
    365   const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value);
    366   if (s.ok()) {
    367     *found = true;
    368     return true;
    369   }
    370   if (s.IsNotFound())
    371     return true;
    372   HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s);
    373   LOG(ERROR) << "LevelDB get failed: " << s.ToString();
    374   return false;
    375 }
    376 
    377 bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) {
    378   leveldb::WriteOptions write_options;
    379   write_options.sync = true;
    380 
    381   const leveldb::Status s =
    382       db_->Write(write_options, write_batch.write_batch_.get());
    383   if (s.ok())
    384     return true;
    385   HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s);
    386   LOG(ERROR) << "LevelDB write failed: " << s.ToString();
    387   return false;
    388 }
    389 
    390 namespace {
    391 class IteratorImpl : public LevelDBIterator {
    392  public:
    393   virtual ~IteratorImpl() {}
    394 
    395   virtual bool IsValid() const OVERRIDE;
    396   virtual void SeekToLast() OVERRIDE;
    397   virtual void Seek(const StringPiece& target) OVERRIDE;
    398   virtual void Next() OVERRIDE;
    399   virtual void Prev() OVERRIDE;
    400   virtual StringPiece Key() const OVERRIDE;
    401   virtual StringPiece Value() const OVERRIDE;
    402 
    403  private:
    404   friend class content::LevelDBDatabase;
    405   explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator);
    406   void CheckStatus();
    407 
    408   scoped_ptr<leveldb::Iterator> iterator_;
    409 };
    410 }
    411 
    412 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it)
    413     : iterator_(it.Pass()) {}
    414 
    415 void IteratorImpl::CheckStatus() {
    416   const leveldb::Status s = iterator_->status();
    417   if (!s.ok())
    418     LOG(ERROR) << "LevelDB iterator error: " << s.ToString();
    419 }
    420 
    421 bool IteratorImpl::IsValid() const { return iterator_->Valid(); }
    422 
    423 void IteratorImpl::SeekToLast() {
    424   iterator_->SeekToLast();
    425   CheckStatus();
    426 }
    427 
    428 void IteratorImpl::Seek(const StringPiece& target) {
    429   iterator_->Seek(MakeSlice(target));
    430   CheckStatus();
    431 }
    432 
    433 void IteratorImpl::Next() {
    434   DCHECK(IsValid());
    435   iterator_->Next();
    436   CheckStatus();
    437 }
    438 
    439 void IteratorImpl::Prev() {
    440   DCHECK(IsValid());
    441   iterator_->Prev();
    442   CheckStatus();
    443 }
    444 
    445 StringPiece IteratorImpl::Key() const {
    446   DCHECK(IsValid());
    447   return MakeStringPiece(iterator_->key());
    448 }
    449 
    450 StringPiece IteratorImpl::Value() const {
    451   DCHECK(IsValid());
    452   return MakeStringPiece(iterator_->value());
    453 }
    454 
    455 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator(
    456     const LevelDBSnapshot* snapshot) {
    457   leveldb::ReadOptions read_options;
    458   read_options.verify_checksums = true;  // TODO(jsbell): Disable this if the
    459                                          // performance impact is too great.
    460   read_options.snapshot = snapshot ? snapshot->snapshot_ : 0;
    461 
    462   scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options));
    463   return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass()));
    464 }
    465 
    466 const LevelDBComparator* LevelDBDatabase::Comparator() const {
    467   return comparator_;
    468 }
    469 
    470 }  // namespace content
    471