1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/browser/indexed_db/leveldb/leveldb_database.h" 6 7 #include <cerrno> 8 9 #include "base/basictypes.h" 10 #include "base/logging.h" 11 #include "base/memory/scoped_ptr.h" 12 #include "base/metrics/histogram.h" 13 #include "base/strings/string16.h" 14 #include "base/strings/string_piece.h" 15 #include "base/strings/stringprintf.h" 16 #include "base/strings/utf_string_conversions.h" 17 #include "base/sys_info.h" 18 #include "content/browser/indexed_db/leveldb/leveldb_comparator.h" 19 #include "content/browser/indexed_db/leveldb/leveldb_iterator.h" 20 #include "content/browser/indexed_db/leveldb/leveldb_write_batch.h" 21 #include "third_party/leveldatabase/env_chromium.h" 22 #include "third_party/leveldatabase/env_idb.h" 23 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h" 24 #include "third_party/leveldatabase/src/include/leveldb/comparator.h" 25 #include "third_party/leveldatabase/src/include/leveldb/db.h" 26 #include "third_party/leveldatabase/src/include/leveldb/env.h" 27 #include "third_party/leveldatabase/src/include/leveldb/slice.h" 28 29 using base::StringPiece; 30 31 namespace content { 32 33 static leveldb::Slice MakeSlice(const StringPiece& s) { 34 return leveldb::Slice(s.begin(), s.size()); 35 } 36 37 static StringPiece MakeStringPiece(const leveldb::Slice& s) { 38 return StringPiece(s.data(), s.size()); 39 } 40 41 class ComparatorAdapter : public leveldb::Comparator { 42 public: 43 explicit ComparatorAdapter(const LevelDBComparator* comparator) 44 : comparator_(comparator) {} 45 46 virtual int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const 47 OVERRIDE { 48 return comparator_->Compare(MakeStringPiece(a), MakeStringPiece(b)); 49 } 50 51 virtual const char* Name() const OVERRIDE { return comparator_->Name(); } 52 53 // TODO(jsbell): Support the methods below in the future. 54 virtual void FindShortestSeparator(std::string* start, 55 const leveldb::Slice& limit) const 56 OVERRIDE {} 57 virtual void FindShortSuccessor(std::string* key) const OVERRIDE {} 58 59 private: 60 const LevelDBComparator* comparator_; 61 }; 62 63 LevelDBSnapshot::LevelDBSnapshot(LevelDBDatabase* db) 64 : db_(db->db_.get()), snapshot_(db_->GetSnapshot()) {} 65 66 LevelDBSnapshot::~LevelDBSnapshot() { db_->ReleaseSnapshot(snapshot_); } 67 68 LevelDBDatabase::LevelDBDatabase() {} 69 70 LevelDBDatabase::~LevelDBDatabase() { 71 // db_'s destructor uses comparator_adapter_; order of deletion is important. 72 db_.reset(); 73 comparator_adapter_.reset(); 74 env_.reset(); 75 } 76 77 static leveldb::Status OpenDB(leveldb::Comparator* comparator, 78 leveldb::Env* env, 79 const base::FilePath& path, 80 leveldb::DB** db) { 81 leveldb::Options options; 82 options.comparator = comparator; 83 options.create_if_missing = true; 84 options.paranoid_checks = true; 85 86 // Marking compression as explicitly off so snappy support can be 87 // compiled in for other leveldb clients without implicitly enabling 88 // it for IndexedDB. http://crbug.com/81384 89 options.compression = leveldb::kNoCompression; 90 91 // For info about the troubles we've run into with this parameter, see: 92 // https://code.google.com/p/chromium/issues/detail?id=227313#c11 93 options.max_open_files = 80; 94 options.env = env; 95 96 // ChromiumEnv assumes UTF8, converts back to FilePath before using. 97 return leveldb::DB::Open(options, path.AsUTF8Unsafe(), db); 98 } 99 100 bool LevelDBDatabase::Destroy(const base::FilePath& file_name) { 101 leveldb::Options options; 102 options.env = leveldb::IDBEnv(); 103 // ChromiumEnv assumes UTF8, converts back to FilePath before using. 104 const leveldb::Status s = 105 leveldb::DestroyDB(file_name.AsUTF8Unsafe(), options); 106 return s.ok(); 107 } 108 109 static int CheckFreeSpace(const char* const type, 110 const base::FilePath& file_name) { 111 std::string name = 112 std::string("WebCore.IndexedDB.LevelDB.Open") + type + "FreeDiskSpace"; 113 int64 free_disk_space_in_k_bytes = 114 base::SysInfo::AmountOfFreeDiskSpace(file_name) / 1024; 115 if (free_disk_space_in_k_bytes < 0) { 116 base::Histogram::FactoryGet( 117 "WebCore.IndexedDB.LevelDB.FreeDiskSpaceFailure", 118 1, 119 2 /*boundary*/, 120 2 /*boundary*/ + 1, 121 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(1 /*sample*/); 122 return -1; 123 } 124 int clamped_disk_space_k_bytes = 125 free_disk_space_in_k_bytes > INT_MAX ? INT_MAX 126 : free_disk_space_in_k_bytes; 127 const uint64 histogram_max = static_cast<uint64>(1e9); 128 COMPILE_ASSERT(histogram_max <= INT_MAX, histogram_max_too_big); 129 base::Histogram::FactoryGet(name, 130 1, 131 histogram_max, 132 11 /*buckets*/, 133 base::HistogramBase::kUmaTargetedHistogramFlag) 134 ->Add(clamped_disk_space_k_bytes); 135 return clamped_disk_space_k_bytes; 136 } 137 138 static void ParseAndHistogramIOErrorDetails(const std::string& histogram_name, 139 const leveldb::Status& s) { 140 leveldb_env::MethodID method; 141 int error = -1; 142 leveldb_env::ErrorParsingResult result = 143 leveldb_env::ParseMethodAndError(s.ToString().c_str(), &method, &error); 144 if (result == leveldb_env::NONE) 145 return; 146 std::string method_histogram_name(histogram_name); 147 method_histogram_name.append(".EnvMethod"); 148 base::LinearHistogram::FactoryGet( 149 method_histogram_name, 150 1, 151 leveldb_env::kNumEntries, 152 leveldb_env::kNumEntries + 1, 153 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(method); 154 155 std::string error_histogram_name(histogram_name); 156 157 if (result == leveldb_env::METHOD_AND_PFE) { 158 DCHECK(error < 0); 159 error_histogram_name.append(std::string(".PFE.") + 160 leveldb_env::MethodIDToString(method)); 161 base::LinearHistogram::FactoryGet( 162 error_histogram_name, 163 1, 164 -base::PLATFORM_FILE_ERROR_MAX, 165 -base::PLATFORM_FILE_ERROR_MAX + 1, 166 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(-error); 167 } else if (result == leveldb_env::METHOD_AND_ERRNO) { 168 error_histogram_name.append(std::string(".Errno.") + 169 leveldb_env::MethodIDToString(method)); 170 base::LinearHistogram::FactoryGet( 171 error_histogram_name, 172 1, 173 ERANGE + 1, 174 ERANGE + 2, 175 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error); 176 } 177 } 178 179 static void ParseAndHistogramCorruptionDetails( 180 const std::string& histogram_name, 181 const leveldb::Status& status) { 182 DCHECK(!status.IsIOError()); 183 DCHECK(!status.ok()); 184 const int kOtherError = 0; 185 int error = kOtherError; 186 const std::string& str_error = status.ToString(); 187 // Keep in sync with LevelDBCorruptionTypes in histograms.xml. 188 const char* patterns[] = { 189 "missing files", 190 "log record too small", 191 "corrupted internal key", 192 "partial record", 193 "missing start of fragmented record", 194 "error in middle of record", 195 "unknown record type", 196 "truncated record at end", 197 "bad record length", 198 "VersionEdit", 199 "FileReader invoked with unexpected value", 200 "corrupted key", 201 "CURRENT file does not end with newline", 202 "no meta-nextfile entry", 203 "no meta-lognumber entry", 204 "no last-sequence-number entry", 205 "malformed WriteBatch", 206 "bad WriteBatch Put", 207 "bad WriteBatch Delete", 208 "unknown WriteBatch tag", 209 "WriteBatch has wrong count", 210 "bad entry in block", 211 "bad block contents", 212 "bad block handle", 213 "truncated block read", 214 "block checksum mismatch", 215 "checksum mismatch", 216 "corrupted compressed block contents", 217 "bad block type", 218 "bad magic number", 219 "file is too short", 220 }; 221 const size_t kNumPatterns = arraysize(patterns); 222 for (size_t i = 0; i < kNumPatterns; ++i) { 223 if (str_error.find(patterns[i]) != std::string::npos) { 224 error = i + 1; 225 break; 226 } 227 } 228 DCHECK(error >= 0); 229 std::string corruption_histogram_name(histogram_name); 230 corruption_histogram_name.append(".Corruption"); 231 base::LinearHistogram::FactoryGet( 232 corruption_histogram_name, 233 1, 234 kNumPatterns + 1, 235 kNumPatterns + 2, 236 base::HistogramBase::kUmaTargetedHistogramFlag)->Add(error); 237 } 238 239 static void HistogramLevelDBError(const std::string& histogram_name, 240 const leveldb::Status& s) { 241 if (s.ok()) { 242 NOTREACHED(); 243 return; 244 } 245 enum { 246 LEVEL_DB_NOT_FOUND, 247 LEVEL_DB_CORRUPTION, 248 LEVEL_DB_IO_ERROR, 249 LEVEL_DB_OTHER, 250 LEVEL_DB_MAX_ERROR 251 }; 252 int leveldb_error = LEVEL_DB_OTHER; 253 if (s.IsNotFound()) 254 leveldb_error = LEVEL_DB_NOT_FOUND; 255 else if (s.IsCorruption()) 256 leveldb_error = LEVEL_DB_CORRUPTION; 257 else if (s.IsIOError()) 258 leveldb_error = LEVEL_DB_IO_ERROR; 259 base::Histogram::FactoryGet(histogram_name, 260 1, 261 LEVEL_DB_MAX_ERROR, 262 LEVEL_DB_MAX_ERROR + 1, 263 base::HistogramBase::kUmaTargetedHistogramFlag) 264 ->Add(leveldb_error); 265 if (s.IsIOError()) 266 ParseAndHistogramIOErrorDetails(histogram_name, s); 267 else 268 ParseAndHistogramCorruptionDetails(histogram_name, s); 269 } 270 271 leveldb::Status LevelDBDatabase::Open( 272 const base::FilePath& file_name, 273 const LevelDBComparator* comparator, 274 scoped_ptr<LevelDBDatabase>* result, 275 bool* is_disk_full) { 276 scoped_ptr<ComparatorAdapter> comparator_adapter( 277 new ComparatorAdapter(comparator)); 278 279 leveldb::DB* db; 280 const leveldb::Status s = 281 OpenDB(comparator_adapter.get(), leveldb::IDBEnv(), file_name, &db); 282 283 if (!s.ok()) { 284 HistogramLevelDBError("WebCore.IndexedDB.LevelDBOpenErrors", s); 285 int free_space_k_bytes = CheckFreeSpace("Failure", file_name); 286 // Disks with <100k of free space almost never succeed in opening a 287 // leveldb database. 288 if (is_disk_full) 289 *is_disk_full = free_space_k_bytes >= 0 && free_space_k_bytes < 100; 290 291 LOG(ERROR) << "Failed to open LevelDB database from " 292 << file_name.AsUTF8Unsafe() << "," << s.ToString(); 293 return s; 294 } 295 296 CheckFreeSpace("Success", file_name); 297 298 (*result).reset(new LevelDBDatabase); 299 (*result)->db_ = make_scoped_ptr(db); 300 (*result)->comparator_adapter_ = comparator_adapter.Pass(); 301 (*result)->comparator_ = comparator; 302 303 return s; 304 } 305 306 scoped_ptr<LevelDBDatabase> LevelDBDatabase::OpenInMemory( 307 const LevelDBComparator* comparator) { 308 scoped_ptr<ComparatorAdapter> comparator_adapter( 309 new ComparatorAdapter(comparator)); 310 scoped_ptr<leveldb::Env> in_memory_env(leveldb::NewMemEnv(leveldb::IDBEnv())); 311 312 leveldb::DB* db; 313 const leveldb::Status s = OpenDB( 314 comparator_adapter.get(), in_memory_env.get(), base::FilePath(), &db); 315 316 if (!s.ok()) { 317 LOG(ERROR) << "Failed to open in-memory LevelDB database: " << s.ToString(); 318 return scoped_ptr<LevelDBDatabase>(); 319 } 320 321 scoped_ptr<LevelDBDatabase> result(new LevelDBDatabase); 322 result->env_ = in_memory_env.Pass(); 323 result->db_ = make_scoped_ptr(db); 324 result->comparator_adapter_ = comparator_adapter.Pass(); 325 result->comparator_ = comparator; 326 327 return result.Pass(); 328 } 329 330 bool LevelDBDatabase::Put(const StringPiece& key, std::string* value) { 331 leveldb::WriteOptions write_options; 332 write_options.sync = true; 333 334 const leveldb::Status s = 335 db_->Put(write_options, MakeSlice(key), MakeSlice(*value)); 336 if (s.ok()) 337 return true; 338 LOG(ERROR) << "LevelDB put failed: " << s.ToString(); 339 return false; 340 } 341 342 bool LevelDBDatabase::Remove(const StringPiece& key) { 343 leveldb::WriteOptions write_options; 344 write_options.sync = true; 345 346 const leveldb::Status s = db_->Delete(write_options, MakeSlice(key)); 347 if (s.ok()) 348 return true; 349 if (s.IsNotFound()) 350 return false; 351 LOG(ERROR) << "LevelDB remove failed: " << s.ToString(); 352 return false; 353 } 354 355 bool LevelDBDatabase::Get(const StringPiece& key, 356 std::string* value, 357 bool* found, 358 const LevelDBSnapshot* snapshot) { 359 *found = false; 360 leveldb::ReadOptions read_options; 361 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the 362 // performance impact is too great. 363 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0; 364 365 const leveldb::Status s = db_->Get(read_options, MakeSlice(key), value); 366 if (s.ok()) { 367 *found = true; 368 return true; 369 } 370 if (s.IsNotFound()) 371 return true; 372 HistogramLevelDBError("WebCore.IndexedDB.LevelDBReadErrors", s); 373 LOG(ERROR) << "LevelDB get failed: " << s.ToString(); 374 return false; 375 } 376 377 bool LevelDBDatabase::Write(const LevelDBWriteBatch& write_batch) { 378 leveldb::WriteOptions write_options; 379 write_options.sync = true; 380 381 const leveldb::Status s = 382 db_->Write(write_options, write_batch.write_batch_.get()); 383 if (s.ok()) 384 return true; 385 HistogramLevelDBError("WebCore.IndexedDB.LevelDBWriteErrors", s); 386 LOG(ERROR) << "LevelDB write failed: " << s.ToString(); 387 return false; 388 } 389 390 namespace { 391 class IteratorImpl : public LevelDBIterator { 392 public: 393 virtual ~IteratorImpl() {} 394 395 virtual bool IsValid() const OVERRIDE; 396 virtual void SeekToLast() OVERRIDE; 397 virtual void Seek(const StringPiece& target) OVERRIDE; 398 virtual void Next() OVERRIDE; 399 virtual void Prev() OVERRIDE; 400 virtual StringPiece Key() const OVERRIDE; 401 virtual StringPiece Value() const OVERRIDE; 402 403 private: 404 friend class content::LevelDBDatabase; 405 explicit IteratorImpl(scoped_ptr<leveldb::Iterator> iterator); 406 void CheckStatus(); 407 408 scoped_ptr<leveldb::Iterator> iterator_; 409 }; 410 } 411 412 IteratorImpl::IteratorImpl(scoped_ptr<leveldb::Iterator> it) 413 : iterator_(it.Pass()) {} 414 415 void IteratorImpl::CheckStatus() { 416 const leveldb::Status s = iterator_->status(); 417 if (!s.ok()) 418 LOG(ERROR) << "LevelDB iterator error: " << s.ToString(); 419 } 420 421 bool IteratorImpl::IsValid() const { return iterator_->Valid(); } 422 423 void IteratorImpl::SeekToLast() { 424 iterator_->SeekToLast(); 425 CheckStatus(); 426 } 427 428 void IteratorImpl::Seek(const StringPiece& target) { 429 iterator_->Seek(MakeSlice(target)); 430 CheckStatus(); 431 } 432 433 void IteratorImpl::Next() { 434 DCHECK(IsValid()); 435 iterator_->Next(); 436 CheckStatus(); 437 } 438 439 void IteratorImpl::Prev() { 440 DCHECK(IsValid()); 441 iterator_->Prev(); 442 CheckStatus(); 443 } 444 445 StringPiece IteratorImpl::Key() const { 446 DCHECK(IsValid()); 447 return MakeStringPiece(iterator_->key()); 448 } 449 450 StringPiece IteratorImpl::Value() const { 451 DCHECK(IsValid()); 452 return MakeStringPiece(iterator_->value()); 453 } 454 455 scoped_ptr<LevelDBIterator> LevelDBDatabase::CreateIterator( 456 const LevelDBSnapshot* snapshot) { 457 leveldb::ReadOptions read_options; 458 read_options.verify_checksums = true; // TODO(jsbell): Disable this if the 459 // performance impact is too great. 460 read_options.snapshot = snapshot ? snapshot->snapshot_ : 0; 461 462 scoped_ptr<leveldb::Iterator> i(db_->NewIterator(read_options)); 463 return scoped_ptr<LevelDBIterator>(new IteratorImpl(i.Pass())); 464 } 465 466 const LevelDBComparator* LevelDBDatabase::Comparator() const { 467 return comparator_; 468 } 469 470 } // namespace content 471