Home | History | Annotate | Download | only in history 
      1 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/history/history_database.h"
      6 
      7 #include <algorithm>
      8 #include <set>
      9 #include <string>
     10 #include "app/sql/transaction.h"
     11 #include "base/command_line.h"
     12 #include "base/file_util.h"
     13 #include "base/metrics/histogram.h"
     14 #include "base/rand_util.h"
     15 #include "base/string_util.h"
     16 #include "chrome/browser/diagnostics/sqlite_diagnostics.h"
     17 
     18 #if defined(OS_MACOSX)
     19 #include "base/mac/mac_util.h"
     20 #endif
     21 
     22 namespace history {
     23 
     24 namespace {
     25 
     26 // Current version number. We write databases at the "current" version number,
     27 // but any previous version that can read the "compatible" one can make do with
     28 // or database without *too* many bad effects.
     29 static const int kCurrentVersionNumber = 20;
     30 static const int kCompatibleVersionNumber = 16;
     31 static const char kEarlyExpirationThresholdKey[] = "early_expiration_threshold";
     32 
     33 // Key in the meta table used to determine if we need to migrate thumbnails out
     34 // of history.
     35 static const char kNeedsThumbnailMigrationKey[] = "needs_thumbnail_migration";
     36 
     37 void ComputeDatabaseMetrics(const FilePath& history_name,
     38                             sql::Connection& db) {
     39   if (base::RandInt(1, 100) != 50)
     40     return;  // Only do this computation sometimes since it can be expensive.
     41 
     42   int64 file_size = 0;
     43   if (!file_util::GetFileSize(history_name, &file_size))
     44     return;
     45   int file_mb = static_cast<int>(file_size / (1024 * 1024));
     46   UMA_HISTOGRAM_MEMORY_MB("History.DatabaseFileMB", file_mb);
     47 
     48   sql::Statement url_count(db.GetUniqueStatement("SELECT count(*) FROM urls"));
     49   if (!url_count || !url_count.Step())
     50     return;
     51   UMA_HISTOGRAM_COUNTS("History.URLTableCount", url_count.ColumnInt(0));
     52 
     53   sql::Statement visit_count(db.GetUniqueStatement(
     54       "SELECT count(*) FROM visits"));
     55   if (!visit_count || !visit_count.Step())
     56     return;
     57   UMA_HISTOGRAM_COUNTS("History.VisitTableCount", visit_count.ColumnInt(0));
     58 }
     59 
     60 }  // namespace
     61 
     62 HistoryDatabase::HistoryDatabase()
     63     : needs_version_17_migration_(false) {
     64 }
     65 
     66 HistoryDatabase::~HistoryDatabase() {
     67 }
     68 
     69 sql::InitStatus HistoryDatabase::Init(const FilePath& history_name,
     70                                       const FilePath& bookmarks_path) {
     71   // Set the exceptional sqlite error handler.
     72   db_.set_error_delegate(GetErrorHandlerForHistoryDb());
     73 
     74   // Set the database page size to something a little larger to give us
     75   // better performance (we're typically seek rather than bandwidth limited).
     76   // This only has an effect before any tables have been created, otherwise
     77   // this is a NOP. Must be a power of 2 and a max of 8192.
     78   db_.set_page_size(4096);
     79 
     80   // Increase the cache size. The page size, plus a little extra, times this
     81   // value, tells us how much memory the cache will use maximum.
     82   // 6000 * 4MB = 24MB
     83   // TODO(brettw) scale this value to the amount of available memory.
     84   db_.set_cache_size(6000);
     85 
     86   // Note that we don't set exclusive locking here. That's done by
     87   // BeginExclusiveMode below which is called later (we have to be in shared
     88   // mode to start out for the in-memory backend to read the data).
     89 
     90   if (!db_.Open(history_name))
     91     return sql::INIT_FAILURE;
     92 
     93   // Wrap the rest of init in a tranaction. This will prevent the database from
     94   // getting corrupted if we crash in the middle of initialization or migration.
     95   sql::Transaction committer(&db_);
     96   if (!committer.Begin())
     97     return sql::INIT_FAILURE;
     98 
     99 #if defined(OS_MACOSX)
    100   // Exclude the history file and its journal from backups.
    101   base::mac::SetFileBackupExclusion(history_name, true);
    102   FilePath::StringType history_name_string(history_name.value());
    103   history_name_string += "-journal";
    104   FilePath history_journal_name(history_name_string);
    105   base::mac::SetFileBackupExclusion(history_journal_name, true);
    106 #endif
    107 
    108   // Prime the cache.
    109   db_.Preload();
    110 
    111   // Create the tables and indices.
    112   // NOTE: If you add something here, also add it to
    113   //       RecreateAllButStarAndURLTables.
    114   if (!meta_table_.Init(&db_, GetCurrentVersion(), kCompatibleVersionNumber))
    115     return sql::INIT_FAILURE;
    116   if (!CreateURLTable(false) || !InitVisitTable() ||
    117       !InitKeywordSearchTermsTable() || !InitDownloadTable() ||
    118       !InitSegmentTables())
    119     return sql::INIT_FAILURE;
    120   CreateMainURLIndex();
    121   CreateKeywordSearchTermsIndices();
    122 
    123   // Version check.
    124   sql::InitStatus version_status = EnsureCurrentVersion(bookmarks_path);
    125   if (version_status != sql::INIT_OK)
    126     return version_status;
    127 
    128   ComputeDatabaseMetrics(history_name, db_);
    129   return committer.Commit() ? sql::INIT_OK : sql::INIT_FAILURE;
    130 }
    131 
    132 void HistoryDatabase::BeginExclusiveMode() {
    133   // We can't use set_exclusive_locking() since that only has an effect before
    134   // the DB is opened.
    135   db_.Execute("PRAGMA locking_mode=EXCLUSIVE");
    136 }
    137 
    138 // static
    139 int HistoryDatabase::GetCurrentVersion() {
    140   return kCurrentVersionNumber;
    141 }
    142 
    143 void HistoryDatabase::BeginTransaction() {
    144   db_.BeginTransaction();
    145 }
    146 
    147 void HistoryDatabase::CommitTransaction() {
    148   db_.CommitTransaction();
    149 }
    150 
    151 bool HistoryDatabase::RecreateAllTablesButURL() {
    152   if (!DropVisitTable())
    153     return false;
    154   if (!InitVisitTable())
    155     return false;
    156 
    157   if (!DropKeywordSearchTermsTable())
    158     return false;
    159   if (!InitKeywordSearchTermsTable())
    160     return false;
    161 
    162   if (!DropSegmentTables())
    163     return false;
    164   if (!InitSegmentTables())
    165     return false;
    166 
    167   // We also add the supplementary URL indices at this point. This index is
    168   // over parts of the URL table that weren't automatically created when the
    169   // temporary URL table was
    170   CreateKeywordSearchTermsIndices();
    171   return true;
    172 }
    173 
    174 void HistoryDatabase::Vacuum() {
    175   DCHECK_EQ(0, db_.transaction_nesting()) <<
    176       "Can not have a transaction when vacuuming.";
    177   db_.Execute("VACUUM");
    178 }
    179 
    180 void HistoryDatabase::ThumbnailMigrationDone() {
    181   meta_table_.SetValue(kNeedsThumbnailMigrationKey, 0);
    182 }
    183 
    184 bool HistoryDatabase::GetNeedsThumbnailMigration() {
    185   int value = 0;
    186   return (meta_table_.GetValue(kNeedsThumbnailMigrationKey, &value) &&
    187           value != 0);
    188 }
    189 
    190 bool HistoryDatabase::SetSegmentID(VisitID visit_id, SegmentID segment_id) {
    191   sql::Statement s(db_.GetCachedStatement(SQL_FROM_HERE,
    192       "UPDATE visits SET segment_id = ? WHERE id = ?"));
    193   if (!s) {
    194     NOTREACHED() << db_.GetErrorMessage();
    195     return false;
    196   }
    197   s.BindInt64(0, segment_id);
    198   s.BindInt64(1, visit_id);
    199   DCHECK(db_.GetLastChangeCount() == 1);
    200   return s.Run();
    201 }
    202 
    203 SegmentID HistoryDatabase::GetSegmentID(VisitID visit_id) {
    204   sql::Statement s(db_.GetCachedStatement(SQL_FROM_HERE,
    205       "SELECT segment_id FROM visits WHERE id = ?"));
    206   if (!s) {
    207     NOTREACHED() << db_.GetErrorMessage();
    208     return 0;
    209   }
    210 
    211   s.BindInt64(0, visit_id);
    212   if (s.Step()) {
    213     if (s.ColumnType(0) == sql::COLUMN_TYPE_NULL)
    214       return 0;
    215     else
    216       return s.ColumnInt64(0);
    217   }
    218   return 0;
    219 }
    220 
    221 base::Time HistoryDatabase::GetEarlyExpirationThreshold() {
    222   if (!cached_early_expiration_threshold_.is_null())
    223     return cached_early_expiration_threshold_;
    224 
    225   int64 threshold;
    226   if (!meta_table_.GetValue(kEarlyExpirationThresholdKey, &threshold)) {
    227     // Set to a very early non-zero time, so it's before all history, but not
    228     // zero to avoid re-retrieval.
    229     threshold = 1L;
    230   }
    231 
    232   cached_early_expiration_threshold_ = base::Time::FromInternalValue(threshold);
    233   return cached_early_expiration_threshold_;
    234 }
    235 
    236 void HistoryDatabase::UpdateEarlyExpirationThreshold(base::Time threshold) {
    237   meta_table_.SetValue(kEarlyExpirationThresholdKey,
    238                        threshold.ToInternalValue());
    239   cached_early_expiration_threshold_ = threshold;
    240 }
    241 
    242 sql::Connection& HistoryDatabase::GetDB() {
    243   return db_;
    244 }
    245 
    246 // Migration -------------------------------------------------------------------
    247 
    248 sql::InitStatus HistoryDatabase::EnsureCurrentVersion(
    249     const FilePath& tmp_bookmarks_path) {
    250   // We can't read databases newer than we were designed for.
    251   if (meta_table_.GetCompatibleVersionNumber() > kCurrentVersionNumber) {
    252     LOG(WARNING) << "History database is too new.";
    253     return sql::INIT_TOO_NEW;
    254   }
    255 
    256   // NOTICE: If you are changing structures for things shared with the archived
    257   // history file like URLs, visits, or downloads, that will need migration as
    258   // well. Instead of putting such migration code in this class, it should be
    259   // in the corresponding file (url_database.cc, etc.) and called from here and
    260   // from the archived_database.cc.
    261 
    262   int cur_version = meta_table_.GetVersionNumber();
    263 
    264   // Put migration code here
    265 
    266   if (cur_version == 15) {
    267     if (!MigrateBookmarksToFile(tmp_bookmarks_path) ||
    268         !DropStarredIDFromURLs()) {
    269       LOG(WARNING) << "Unable to update history database to version 16.";
    270       return sql::INIT_FAILURE;
    271     }
    272     ++cur_version;
    273     meta_table_.SetVersionNumber(cur_version);
    274     meta_table_.SetCompatibleVersionNumber(
    275         std::min(cur_version, kCompatibleVersionNumber));
    276   }
    277 
    278   if (cur_version == 16) {
    279 #if !defined(OS_WIN)
    280     // In this version we bring the time format on Mac & Linux in sync with the
    281     // Windows version so that profiles can be moved between computers.
    282     MigrateTimeEpoch();
    283 #endif
    284     // On all platforms we bump the version number, so on Windows this
    285     // migration is a NOP. We keep the compatible version at 16 since things
    286     // will basically still work, just history will be in the future if an
    287     // old version reads it.
    288     ++cur_version;
    289     meta_table_.SetVersionNumber(cur_version);
    290   }
    291 
    292   if (cur_version == 17) {
    293     // Version 17 was for thumbnails to top sites migration. We ended up
    294     // disabling it though, so 17->18 does nothing.
    295     ++cur_version;
    296     meta_table_.SetVersionNumber(cur_version);
    297   }
    298 
    299   if (cur_version == 18) {
    300     // This is the version prior to adding url_source column. We need to
    301     // migrate the database.
    302     cur_version = 19;
    303     meta_table_.SetVersionNumber(cur_version);
    304   }
    305 
    306   if (cur_version == 19) {
    307     cur_version++;
    308     meta_table_.SetVersionNumber(cur_version);
    309     // Set a key indicating we need to migrate thumbnails. When successfull the
    310     // key is removed (ThumbnailMigrationDone).
    311     meta_table_.SetValue(kNeedsThumbnailMigrationKey, 1);
    312   }
    313 
    314   // When the version is too old, we just try to continue anyway, there should
    315   // not be a released product that makes a database too old for us to handle.
    316   LOG_IF(WARNING, cur_version < GetCurrentVersion()) <<
    317          "History database version " << cur_version << " is too old to handle.";
    318 
    319   return sql::INIT_OK;
    320 }
    321 
    322 #if !defined(OS_WIN)
    323 void HistoryDatabase::MigrateTimeEpoch() {
    324   // Update all the times in the URLs and visits table in the main database.
    325   // For visits, clear the indexed flag since we'll delete the FTS databases in
    326   // the next step.
    327   db_.Execute(
    328       "UPDATE urls "
    329       "SET last_visit_time = last_visit_time + 11644473600000000 "
    330       "WHERE id IN (SELECT id FROM urls WHERE last_visit_time > 0);");
    331   db_.Execute(
    332       "UPDATE visits "
    333       "SET visit_time = visit_time + 11644473600000000, is_indexed = 0 "
    334       "WHERE id IN (SELECT id FROM visits WHERE visit_time > 0);");
    335   db_.Execute(
    336       "UPDATE segment_usage "
    337       "SET time_slot = time_slot + 11644473600000000 "
    338       "WHERE id IN (SELECT id FROM segment_usage WHERE time_slot > 0);");
    339 
    340   // Erase all the full text index files. These will take a while to update and
    341   // are less important, so we just blow them away. Same with the archived
    342   // database.
    343   needs_version_17_migration_ = true;
    344 }
    345 #endif
    346 
    347 }  // namespace history
    348