Home | History | Annotate | Download | only in blockfile 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/disk_cache/blockfile/backend_worker_v3.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/bind_helpers.h"
      9 #include "base/file_util.h"
     10 #include "base/files/file_path.h"
     11 #include "base/message_loop/message_loop.h"
     12 #include "base/strings/string_util.h"
     13 #include "base/strings/stringprintf.h"
     14 #include "base/time/time.h"
     15 #include "base/timer/timer.h"
     16 #include "net/base/net_errors.h"
     17 #include "net/disk_cache/blockfile/errors.h"
     18 #include "net/disk_cache/blockfile/experiments.h"
     19 #include "net/disk_cache/blockfile/file.h"
     20 
     21 using base::Time;
     22 using base::TimeDelta;
     23 using base::TimeTicks;
     24 
     25 namespace {
     26 
     27 #if defined(V3_NOT_JUST_YET_READY)
     28 
     29 const char* kIndexName = "index";
     30 
     31 // Seems like ~240 MB correspond to less than 50k entries for 99% of the people.
     32 // Note that the actual target is to keep the index table load factor under 55%
     33 // for most users.
     34 const int k64kEntriesStore = 240 * 1000 * 1000;
     35 const int kBaseTableLen = 64 * 1024;
     36 const int kDefaultCacheSize = 80 * 1024 * 1024;
     37 
     38 // Avoid trimming the cache for the first 5 minutes (10 timer ticks).
     39 const int kTrimDelay = 10;
     40 
     41 int DesiredIndexTableLen(int32 storage_size) {
     42   if (storage_size <= k64kEntriesStore)
     43     return kBaseTableLen;
     44   if (storage_size <= k64kEntriesStore * 2)
     45     return kBaseTableLen * 2;
     46   if (storage_size <= k64kEntriesStore * 4)
     47     return kBaseTableLen * 4;
     48   if (storage_size <= k64kEntriesStore * 8)
     49     return kBaseTableLen * 8;
     50 
     51   // The biggest storage_size for int32 requires a 4 MB table.
     52   return kBaseTableLen * 16;
     53 }
     54 
     55 int MaxStorageSizeForTable(int table_len) {
     56   return table_len * (k64kEntriesStore / kBaseTableLen);
     57 }
     58 
     59 size_t GetIndexSize(int table_len) {
     60   size_t table_size = sizeof(disk_cache::CacheAddr) * table_len;
     61   return sizeof(disk_cache::IndexHeader) + table_size;
     62 }
     63 
     64 // ------------------------------------------------------------------------
     65 
     66 // Sets group for the current experiment. Returns false if the files should be
     67 // discarded.
     68 bool InitExperiment(disk_cache::IndexHeader* header, bool cache_created) {
     69   if (header->experiment == disk_cache::EXPERIMENT_OLD_FILE1 ||
     70       header->experiment == disk_cache::EXPERIMENT_OLD_FILE2) {
     71     // Discard current cache.
     72     return false;
     73   }
     74 
     75   if (base::FieldTrialList::FindFullName("SimpleCacheTrial") ==
     76           "ExperimentControl") {
     77     if (cache_created) {
     78       header->experiment = disk_cache::EXPERIMENT_SIMPLE_CONTROL;
     79       return true;
     80     } else if (header->experiment != disk_cache::EXPERIMENT_SIMPLE_CONTROL) {
     81       return false;
     82     }
     83   }
     84 
     85   header->experiment = disk_cache::NO_EXPERIMENT;
     86   return true;
     87 }
     88 #endif  // defined(V3_NOT_JUST_YET_READY).
     89 
     90 }  // namespace
     91 
     92 // ------------------------------------------------------------------------
     93 
     94 namespace disk_cache {
     95 
     96 BackendImplV3::Worker::Worker(const base::FilePath& path,
     97                               base::MessageLoopProxy* main_thread)
     98       : path_(path),
     99         block_files_(path),
    100         init_(false) {
    101 }
    102 
    103 #if defined(V3_NOT_JUST_YET_READY)
    104 
    105 int BackendImpl::SyncInit() {
    106 #if defined(NET_BUILD_STRESS_CACHE)
    107   // Start evictions right away.
    108   up_ticks_ = kTrimDelay * 2;
    109 #endif
    110   DCHECK(!init_);
    111   if (init_)
    112     return net::ERR_FAILED;
    113 
    114   bool create_files = false;
    115   if (!InitBackingStore(&create_files)) {
    116     ReportError(ERR_STORAGE_ERROR);
    117     return net::ERR_FAILED;
    118   }
    119 
    120   num_refs_ = num_pending_io_ = max_refs_ = 0;
    121   entry_count_ = byte_count_ = 0;
    122 
    123   if (!restarted_) {
    124     buffer_bytes_ = 0;
    125     trace_object_ = TraceObject::GetTraceObject();
    126     // Create a recurrent timer of 30 secs.
    127     int timer_delay = unit_test_ ? 1000 : 30000;
    128     timer_.reset(new base::RepeatingTimer<BackendImpl>());
    129     timer_->Start(FROM_HERE, TimeDelta::FromMilliseconds(timer_delay), this,
    130                   &BackendImpl::OnStatsTimer);
    131   }
    132 
    133   init_ = true;
    134   Trace("Init");
    135 
    136   if (data_->header.experiment != NO_EXPERIMENT &&
    137       cache_type_ != net::DISK_CACHE) {
    138     // No experiment for other caches.
    139     return net::ERR_FAILED;
    140   }
    141 
    142   if (!(user_flags_ & kNoRandom)) {
    143     // The unit test controls directly what to test.
    144     new_eviction_ = (cache_type_ == net::DISK_CACHE);
    145   }
    146 
    147   if (!CheckIndex()) {
    148     ReportError(ERR_INIT_FAILED);
    149     return net::ERR_FAILED;
    150   }
    151 
    152   if (!restarted_ && (create_files || !data_->header.num_entries))
    153     ReportError(ERR_CACHE_CREATED);
    154 
    155   if (!(user_flags_ & kNoRandom) && cache_type_ == net::DISK_CACHE &&
    156       !InitExperiment(&data_->header, create_files)) {
    157     return net::ERR_FAILED;
    158   }
    159 
    160   // We don't care if the value overflows. The only thing we care about is that
    161   // the id cannot be zero, because that value is used as "not dirty".
    162   // Increasing the value once per second gives us many years before we start
    163   // having collisions.
    164   data_->header.this_id++;
    165   if (!data_->header.this_id)
    166     data_->header.this_id++;
    167 
    168   bool previous_crash = (data_->header.crash != 0);
    169   data_->header.crash = 1;
    170 
    171   if (!block_files_.Init(create_files))
    172     return net::ERR_FAILED;
    173 
    174   // We want to minimize the changes to cache for an AppCache.
    175   if (cache_type() == net::APP_CACHE) {
    176     DCHECK(!new_eviction_);
    177     read_only_ = true;
    178   } else if (cache_type() == net::SHADER_CACHE) {
    179     DCHECK(!new_eviction_);
    180   }
    181 
    182   eviction_.Init(this);
    183 
    184   // stats_ and rankings_ may end up calling back to us so we better be enabled.
    185   disabled_ = false;
    186   if (!InitStats())
    187     return net::ERR_FAILED;
    188 
    189   disabled_ = !rankings_.Init(this, new_eviction_);
    190 
    191 #if defined(STRESS_CACHE_EXTENDED_VALIDATION)
    192   trace_object_->EnableTracing(false);
    193   int sc = SelfCheck();
    194   if (sc < 0 && sc != ERR_NUM_ENTRIES_MISMATCH)
    195     NOTREACHED();
    196   trace_object_->EnableTracing(true);
    197 #endif
    198 
    199   if (previous_crash) {
    200     ReportError(ERR_PREVIOUS_CRASH);
    201   } else if (!restarted_) {
    202     ReportError(ERR_NO_ERROR);
    203   }
    204 
    205   FlushIndex();
    206 
    207   return disabled_ ? net::ERR_FAILED : net::OK;
    208 }
    209 
    210 void BackendImpl::PrepareForRestart() {
    211   // Reset the mask_ if it was not given by the user.
    212   if (!(user_flags_ & kMask))
    213     mask_ = 0;
    214 
    215   if (!(user_flags_ & kNewEviction))
    216     new_eviction_ = false;
    217 
    218   disabled_ = true;
    219   data_->header.crash = 0;
    220   index_->Flush();
    221   index_ = NULL;
    222   data_ = NULL;
    223   block_files_.CloseFiles();
    224   rankings_.Reset();
    225   init_ = false;
    226   restarted_ = true;
    227 }
    228 
    229 BackendImpl::~BackendImpl() {
    230   if (user_flags_ & kNoRandom) {
    231     // This is a unit test, so we want to be strict about not leaking entries
    232     // and completing all the work.
    233     background_queue_.WaitForPendingIO();
    234   } else {
    235     // This is most likely not a test, so we want to do as little work as
    236     // possible at this time, at the price of leaving dirty entries behind.
    237     background_queue_.DropPendingIO();
    238   }
    239 
    240   if (background_queue_.BackgroundIsCurrentThread()) {
    241     // Unit tests may use the same thread for everything.
    242     CleanupCache();
    243   } else {
    244     background_queue_.background_thread()->PostTask(
    245         FROM_HERE, base::Bind(&FinalCleanupCallback, base::Unretained(this)));
    246     // http://crbug.com/74623
    247     base::ThreadRestrictions::ScopedAllowWait allow_wait;
    248     done_.Wait();
    249   }
    250 }
    251 
    252 void BackendImpl::CleanupCache() {
    253   Trace("Backend Cleanup");
    254   eviction_.Stop();
    255   timer_.reset();
    256 
    257   if (init_) {
    258     StoreStats();
    259     if (data_)
    260       data_->header.crash = 0;
    261 
    262     if (user_flags_ & kNoRandom) {
    263       // This is a net_unittest, verify that we are not 'leaking' entries.
    264       File::WaitForPendingIO(&num_pending_io_);
    265       DCHECK(!num_refs_);
    266     } else {
    267       File::DropPendingIO();
    268     }
    269   }
    270   block_files_.CloseFiles();
    271   FlushIndex();
    272   index_ = NULL;
    273   ptr_factory_.InvalidateWeakPtrs();
    274   done_.Signal();
    275 }
    276 
    277 base::FilePath BackendImpl::GetFileName(Addr address) const {
    278   if (!address.is_separate_file() || !address.is_initialized()) {
    279     NOTREACHED();
    280     return base::FilePath();
    281   }
    282 
    283   std::string tmp = base::StringPrintf("f_%06x", address.FileNumber());
    284   return path_.AppendASCII(tmp);
    285 }
    286 
    287 // We just created a new file so we're going to write the header and set the
    288 // file length to include the hash table (zero filled).
    289 bool BackendImpl::CreateBackingStore(disk_cache::File* file) {
    290   AdjustMaxCacheSize(0);
    291 
    292   IndexHeader header;
    293   header.table_len = DesiredIndexTableLen(max_size_);
    294 
    295   // We need file version 2.1 for the new eviction algorithm.
    296   if (new_eviction_)
    297     header.version = 0x20001;
    298 
    299   header.create_time = Time::Now().ToInternalValue();
    300 
    301   if (!file->Write(&header, sizeof(header), 0))
    302     return false;
    303 
    304   return file->SetLength(GetIndexSize(header.table_len));
    305 }
    306 
    307 bool BackendImpl::InitBackingStore(bool* file_created) {
    308   if (!base::CreateDirectory(path_))
    309     return false;
    310 
    311   base::FilePath index_name = path_.AppendASCII(kIndexName);
    312 
    313   int flags = base::PLATFORM_FILE_READ |
    314               base::PLATFORM_FILE_WRITE |
    315               base::PLATFORM_FILE_OPEN_ALWAYS |
    316               base::PLATFORM_FILE_EXCLUSIVE_WRITE;
    317   scoped_refptr<disk_cache::File> file(new disk_cache::File(
    318       base::CreatePlatformFile(index_name, flags, file_created, NULL)));
    319 
    320   if (!file->IsValid())
    321     return false;
    322 
    323   bool ret = true;
    324   if (*file_created)
    325     ret = CreateBackingStore(file.get());
    326 
    327   file = NULL;
    328   if (!ret)
    329     return false;
    330 
    331   index_ = new MappedFile();
    332   data_ = reinterpret_cast<Index*>(index_->Init(index_name, 0));
    333   if (!data_) {
    334     LOG(ERROR) << "Unable to map Index file";
    335     return false;
    336   }
    337 
    338   if (index_->GetLength() < sizeof(Index)) {
    339     // We verify this again on CheckIndex() but it's easier to make sure now
    340     // that the header is there.
    341     LOG(ERROR) << "Corrupt Index file";
    342     return false;
    343   }
    344 
    345   return true;
    346 }
    347 
    348 void BackendImpl::ReportError(int error) {
    349   STRESS_DCHECK(!error || error == ERR_PREVIOUS_CRASH ||
    350                 error == ERR_CACHE_CREATED);
    351 
    352   // We transmit positive numbers, instead of direct error codes.
    353   DCHECK_LE(error, 0);
    354   CACHE_UMA(CACHE_ERROR, "Error", 0, error * -1);
    355 }
    356 
    357 
    358 bool BackendImpl::CheckIndex() {
    359   DCHECK(data_);
    360 
    361   size_t current_size = index_->GetLength();
    362   if (current_size < sizeof(Index)) {
    363     LOG(ERROR) << "Corrupt Index file";
    364     return false;
    365   }
    366 
    367   if (new_eviction_) {
    368     // We support versions 2.0 and 2.1, upgrading 2.0 to 2.1.
    369     if (kIndexMagic != data_->header.magic ||
    370         kCurrentVersion >> 16 != data_->header.version >> 16) {
    371       LOG(ERROR) << "Invalid file version or magic";
    372       return false;
    373     }
    374     if (kCurrentVersion == data_->header.version) {
    375       // We need file version 2.1 for the new eviction algorithm.
    376       UpgradeTo2_1();
    377     }
    378   } else {
    379     if (kIndexMagic != data_->header.magic ||
    380         kCurrentVersion != data_->header.version) {
    381       LOG(ERROR) << "Invalid file version or magic";
    382       return false;
    383     }
    384   }
    385 
    386   if (!data_->header.table_len) {
    387     LOG(ERROR) << "Invalid table size";
    388     return false;
    389   }
    390 
    391   if (current_size < GetIndexSize(data_->header.table_len) ||
    392       data_->header.table_len & (kBaseTableLen - 1)) {
    393     LOG(ERROR) << "Corrupt Index file";
    394     return false;
    395   }
    396 
    397   AdjustMaxCacheSize(data_->header.table_len);
    398 
    399 #if !defined(NET_BUILD_STRESS_CACHE)
    400   if (data_->header.num_bytes < 0 ||
    401       (max_size_ < kint32max - kDefaultCacheSize &&
    402        data_->header.num_bytes > max_size_ + kDefaultCacheSize)) {
    403     LOG(ERROR) << "Invalid cache (current) size";
    404     return false;
    405   }
    406 #endif
    407 
    408   if (data_->header.num_entries < 0) {
    409     LOG(ERROR) << "Invalid number of entries";
    410     return false;
    411   }
    412 
    413   if (!mask_)
    414     mask_ = data_->header.table_len - 1;
    415 
    416   // Load the table into memory with a single read.
    417   scoped_ptr<char[]> buf(new char[current_size]);
    418   return index_->Read(buf.get(), current_size, 0);
    419 }
    420 
    421 bool BackendImpl::InitStats() {
    422   Addr address(data_->header.stats);
    423   int size = stats_.StorageSize();
    424 
    425   if (!address.is_initialized()) {
    426     FileType file_type = Addr::RequiredFileType(size);
    427     DCHECK_NE(file_type, EXTERNAL);
    428     int num_blocks = Addr::RequiredBlocks(size, file_type);
    429 
    430     if (!CreateBlock(file_type, num_blocks, &address))
    431       return false;
    432     return stats_.Init(NULL, 0, address);
    433   }
    434 
    435   if (!address.is_block_file()) {
    436     NOTREACHED();
    437     return false;
    438   }
    439 
    440   // Load the required data.
    441   size = address.num_blocks() * address.BlockSize();
    442   MappedFile* file = File(address);
    443   if (!file)
    444     return false;
    445 
    446   scoped_ptr<char[]> data(new char[size]);
    447   size_t offset = address.start_block() * address.BlockSize() +
    448                   kBlockHeaderSize;
    449   if (!file->Read(data.get(), size, offset))
    450     return false;
    451 
    452   if (!stats_.Init(data.get(), size, address))
    453     return false;
    454   if (cache_type_ == net::DISK_CACHE && ShouldReportAgain())
    455     stats_.InitSizeHistogram();
    456   return true;
    457 }
    458 
    459 #endif  // defined(V3_NOT_JUST_YET_READY).
    460 
    461 int BackendImplV3::Worker::Init(const CompletionCallback& callback) {
    462   return net::ERR_FAILED;
    463 }
    464 
    465 BackendImplV3::Worker::~Worker() {
    466 }
    467 
    468 }  // namespace disk_cache
    469