Home | History | Annotate | Download | only in importer 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/importer/firefox3_importer.h"
      6 
      7 #include <set>
      8 
      9 #include "app/sql/connection.h"
     10 #include "app/sql/statement.h"
     11 #include "base/file_util.h"
     12 #include "base/memory/scoped_ptr.h"
     13 #include "base/message_loop.h"
     14 #include "base/stl_util-inl.h"
     15 #include "base/string_util.h"
     16 #include "base/utf_string_conversions.h"
     17 #include "chrome/browser/browser_process.h"
     18 #include "chrome/browser/history/history_types.h"
     19 #include "chrome/browser/importer/firefox2_importer.h"
     20 #include "chrome/browser/importer/firefox_importer_utils.h"
     21 #include "chrome/browser/importer/importer_bridge.h"
     22 #include "chrome/browser/importer/nss_decryptor.h"
     23 #include "chrome/browser/search_engines/template_url.h"
     24 #include "chrome/common/time_format.h"
     25 #include "content/browser/browser_thread.h"
     26 #include "googleurl/src/gurl.h"
     27 #include "grit/generated_resources.h"
     28 #include "webkit/glue/password_form.h"
     29 
     30 namespace {
     31 
     32 // Original definition is in http://mxr.mozilla.org/firefox/source/toolkit/
     33 //  components/places/public/nsINavBookmarksService.idl
     34 enum BookmarkItemType {
     35   TYPE_BOOKMARK = 1,
     36   TYPE_FOLDER = 2,
     37   TYPE_SEPARATOR = 3,
     38   TYPE_DYNAMIC_CONTAINER = 4
     39 };
     40 
     41 }  // namespace
     42 
     43 struct Firefox3Importer::BookmarkItem {
     44   int parent;
     45   int id;
     46   GURL url;
     47   string16 title;
     48   BookmarkItemType type;
     49   std::string keyword;
     50   base::Time date_added;
     51   int64 favicon;
     52   bool empty_folder;
     53 };
     54 
     55 Firefox3Importer::Firefox3Importer() {
     56 #if defined(OS_LINUX)
     57   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
     58   locale_ = g_browser_process->GetApplicationLocale();
     59 #endif
     60 }
     61 
     62 Firefox3Importer::~Firefox3Importer() {
     63 }
     64 
     65 void Firefox3Importer::StartImport(
     66     const importer::SourceProfile& source_profile,
     67     uint16 items,
     68     ImporterBridge* bridge) {
     69 #if defined(OS_LINUX)
     70   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
     71 #endif
     72   bridge_ = bridge;
     73   source_path_ = source_profile.source_path;
     74   app_path_ = source_profile.app_path;
     75 
     76   // The order here is important!
     77   bridge_->NotifyStarted();
     78   if ((items & importer::HOME_PAGE) && !cancelled())
     79     ImportHomepage();  // Doesn't have a UI item.
     80 
     81   // Note history should be imported before bookmarks because bookmark import
     82   // will also import favicons and we store favicon for a URL only if the URL
     83   // exist in history or bookmarks.
     84   if ((items & importer::HISTORY) && !cancelled()) {
     85     bridge_->NotifyItemStarted(importer::HISTORY);
     86     ImportHistory();
     87     bridge_->NotifyItemEnded(importer::HISTORY);
     88   }
     89 
     90   if ((items & importer::FAVORITES) && !cancelled()) {
     91     bridge_->NotifyItemStarted(importer::FAVORITES);
     92     ImportBookmarks();
     93     bridge_->NotifyItemEnded(importer::FAVORITES);
     94   }
     95   if ((items & importer::SEARCH_ENGINES) && !cancelled()) {
     96     bridge_->NotifyItemStarted(importer::SEARCH_ENGINES);
     97     ImportSearchEngines();
     98     bridge_->NotifyItemEnded(importer::SEARCH_ENGINES);
     99   }
    100   if ((items & importer::PASSWORDS) && !cancelled()) {
    101     bridge_->NotifyItemStarted(importer::PASSWORDS);
    102     ImportPasswords();
    103     bridge_->NotifyItemEnded(importer::PASSWORDS);
    104   }
    105   bridge_->NotifyEnded();
    106 }
    107 
    108 void Firefox3Importer::ImportHistory() {
    109   FilePath file = source_path_.AppendASCII("places.sqlite");
    110   if (!file_util::PathExists(file))
    111     return;
    112 
    113   sql::Connection db;
    114   if (!db.Open(file))
    115     return;
    116 
    117   // |visit_type| represent the transition type of URLs (typed, click,
    118   // redirect, bookmark, etc.) We eliminate some URLs like sub-frames and
    119   // redirects, since we don't want them to appear in history.
    120   // Firefox transition types are defined in:
    121   //   toolkit/components/places/public/nsINavHistoryService.idl
    122   const char* query = "SELECT h.url, h.title, h.visit_count, "
    123                       "h.hidden, h.typed, v.visit_date "
    124                       "FROM moz_places h JOIN moz_historyvisits v "
    125                       "ON h.id = v.place_id "
    126                       "WHERE v.visit_type <= 3";
    127 
    128   sql::Statement s(db.GetUniqueStatement(query));
    129   if (!s)
    130     return;
    131 
    132   std::vector<history::URLRow> rows;
    133   while (s.Step() && !cancelled()) {
    134     GURL url(s.ColumnString(0));
    135 
    136     // Filter out unwanted URLs.
    137     if (!CanImportURL(url))
    138       continue;
    139 
    140     history::URLRow row(url);
    141     row.set_title(s.ColumnString16(1));
    142     row.set_visit_count(s.ColumnInt(2));
    143     row.set_hidden(s.ColumnInt(3) == 1);
    144     row.set_typed_count(s.ColumnInt(4));
    145     row.set_last_visit(base::Time::FromTimeT(s.ColumnInt64(5)/1000000));
    146 
    147     rows.push_back(row);
    148   }
    149 
    150   if (!rows.empty() && !cancelled())
    151     bridge_->SetHistoryItems(rows, history::SOURCE_FIREFOX_IMPORTED);
    152 }
    153 
    154 void Firefox3Importer::ImportBookmarks() {
    155   FilePath file = source_path_.AppendASCII("places.sqlite");
    156   if (!file_util::PathExists(file))
    157     return;
    158 
    159   sql::Connection db;
    160   if (!db.Open(file))
    161     return;
    162 
    163   // Get the bookmark folders that we are interested in.
    164   int toolbar_folder_id = -1;
    165   int menu_folder_id = -1;
    166   int unsorted_folder_id = -1;
    167   LoadRootNodeID(&db, &toolbar_folder_id, &menu_folder_id, &unsorted_folder_id);
    168 
    169   // Load livemark IDs.
    170   std::set<int> livemark_id;
    171   LoadLivemarkIDs(&db, &livemark_id);
    172 
    173   // Load the default bookmarks. Its storage is the same as Firefox 2.
    174   std::set<GURL> default_urls;
    175   Firefox2Importer::LoadDefaultBookmarks(app_path_, &default_urls);
    176 
    177   BookmarkList list;
    178   GetTopBookmarkFolder(&db, toolbar_folder_id, &list);
    179   GetTopBookmarkFolder(&db, menu_folder_id, &list);
    180   GetTopBookmarkFolder(&db, unsorted_folder_id, &list);
    181   size_t count = list.size();
    182   for (size_t i = 0; i < count; ++i)
    183     GetWholeBookmarkFolder(&db, &list, i, NULL);
    184 
    185   std::vector<ProfileWriter::BookmarkEntry> bookmarks;
    186   std::vector<TemplateURL*> template_urls;
    187   FaviconMap favicon_map;
    188 
    189   // TODO(jcampan): http://b/issue?id=1196285 we do not support POST based
    190   //                keywords yet.  We won't include them in the list.
    191   std::set<int> post_keyword_ids;
    192   const char* query = "SELECT b.id FROM moz_bookmarks b "
    193       "INNER JOIN moz_items_annos ia ON ia.item_id = b.id "
    194       "INNER JOIN moz_anno_attributes aa ON ia.anno_attribute_id = aa.id "
    195       "WHERE aa.name = 'bookmarkProperties/POSTData'";
    196   sql::Statement s(db.GetUniqueStatement(query));
    197   if (s) {
    198     while (s.Step() && !cancelled())
    199       post_keyword_ids.insert(s.ColumnInt(0));
    200   } else {
    201     NOTREACHED();
    202     return;
    203   }
    204 
    205   string16 firefox_folder =
    206       bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
    207   for (size_t i = 0; i < list.size(); ++i) {
    208     BookmarkItem* item = list[i];
    209 
    210     if (item->type == TYPE_FOLDER) {
    211       // Folders are added implicitly on adding children,
    212       // so now we pass only empty folders to add them explicitly.
    213       if (!item->empty_folder)
    214         continue;
    215     } else if (item->type == TYPE_BOOKMARK) {
    216       // Import only valid bookmarks
    217       if (!CanImportURL(item->url))
    218         continue;
    219     } else {
    220       continue;
    221     }
    222 
    223     // Skip the default bookmarks and unwanted URLs.
    224     if (default_urls.find(item->url) != default_urls.end() ||
    225         post_keyword_ids.find(item->id) != post_keyword_ids.end())
    226       continue;
    227 
    228     // Find the bookmark path by tracing their links to parent folders.
    229     std::vector<string16> path;
    230     BookmarkItem* child = item;
    231     bool found_path = false;
    232     bool is_in_toolbar = false;
    233     while (child->parent >= 0) {
    234       BookmarkItem* parent = list[child->parent];
    235       if (parent->id == toolbar_folder_id) {
    236         // This bookmark entry should be put in the bookmark bar.
    237         // But we put it in the Firefox group after first run, so
    238         // that do not mess up the bookmark bar.
    239         if (import_to_bookmark_bar()) {
    240           is_in_toolbar = true;
    241         } else {
    242           path.insert(path.begin(), parent->title);
    243           path.insert(path.begin(), firefox_folder);
    244         }
    245         found_path = true;
    246         break;
    247       } else if (parent->id == menu_folder_id ||
    248                  parent->id == unsorted_folder_id) {
    249         // After the first run, the item will be placed in a folder in
    250         // the "Other bookmarks".
    251         if (!import_to_bookmark_bar())
    252           path.insert(path.begin(), firefox_folder);
    253         found_path = true;
    254         break;
    255       } else if (livemark_id.find(parent->id) != livemark_id.end()) {
    256         // If the entry is under a livemark folder, we don't import it.
    257         break;
    258       }
    259       path.insert(path.begin(), parent->title);
    260       child = parent;
    261     }
    262 
    263     if (!found_path)
    264       continue;
    265 
    266     ProfileWriter::BookmarkEntry entry;
    267     entry.creation_time = item->date_added;
    268     entry.title = item->title;
    269     entry.url = item->url;
    270     entry.path = path;
    271     entry.in_toolbar = is_in_toolbar;
    272     entry.is_folder = item->type == TYPE_FOLDER;
    273 
    274     bookmarks.push_back(entry);
    275 
    276     if (item->type == TYPE_BOOKMARK) {
    277       if (item->favicon)
    278         favicon_map[item->favicon].insert(item->url);
    279 
    280       // This bookmark has a keyword, we import it to our TemplateURL model.
    281       TemplateURL* t_url = Firefox2Importer::CreateTemplateURL(
    282           item->title, UTF8ToUTF16(item->keyword), item->url);
    283       if (t_url)
    284         template_urls.push_back(t_url);
    285     }
    286   }
    287 
    288   STLDeleteContainerPointers(list.begin(), list.end());
    289 
    290   // Write into profile.
    291   if (!bookmarks.empty() && !cancelled()) {
    292     const string16& first_folder_name =
    293         bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
    294     int options = 0;
    295     if (import_to_bookmark_bar())
    296       options = ProfileWriter::IMPORT_TO_BOOKMARK_BAR;
    297     bridge_->AddBookmarkEntries(bookmarks, first_folder_name, options);
    298   }
    299   if (!template_urls.empty() && !cancelled()) {
    300     bridge_->SetKeywords(template_urls, -1, false);
    301   } else {
    302     STLDeleteContainerPointers(template_urls.begin(), template_urls.end());
    303   }
    304   if (!favicon_map.empty() && !cancelled()) {
    305     std::vector<history::ImportedFaviconUsage> favicons;
    306     LoadFavicons(&db, favicon_map, &favicons);
    307     bridge_->SetFavicons(favicons);
    308   }
    309 }
    310 
    311 void Firefox3Importer::ImportPasswords() {
    312   // Initializes NSS3.
    313   NSSDecryptor decryptor;
    314   if (!decryptor.Init(source_path_, source_path_) &&
    315       !decryptor.Init(app_path_, source_path_)) {
    316     return;
    317   }
    318 
    319   std::vector<webkit_glue::PasswordForm> forms;
    320   FilePath source_path = source_path_;
    321   FilePath file = source_path.AppendASCII("signons.sqlite");
    322   if (file_util::PathExists(file)) {
    323     // Since Firefox 3.1, passwords are in signons.sqlite db.
    324     decryptor.ReadAndParseSignons(file, &forms);
    325   } else {
    326     // Firefox 3.0 uses signons3.txt to store the passwords.
    327     file = source_path.AppendASCII("signons3.txt");
    328     if (!file_util::PathExists(file))
    329       file = source_path.AppendASCII("signons2.txt");
    330 
    331     std::string content;
    332     file_util::ReadFileToString(file, &content);
    333     decryptor.ParseSignons(content, &forms);
    334   }
    335 
    336   if (!cancelled()) {
    337     for (size_t i = 0; i < forms.size(); ++i) {
    338       bridge_->SetPasswordForm(forms[i]);
    339     }
    340   }
    341 }
    342 
    343 void Firefox3Importer::ImportSearchEngines() {
    344   std::vector<FilePath> files;
    345   GetSearchEnginesXMLFiles(&files);
    346 
    347   std::vector<TemplateURL*> search_engines;
    348   ParseSearchEnginesFromXMLFiles(files, &search_engines);
    349   int default_index =
    350       GetFirefoxDefaultSearchEngineIndex(search_engines, source_path_);
    351   bridge_->SetKeywords(search_engines, default_index, true);
    352 }
    353 
    354 void Firefox3Importer::ImportHomepage() {
    355   GURL home_page = GetHomepage(source_path_);
    356   if (home_page.is_valid() && !IsDefaultHomepage(home_page, app_path_)) {
    357     bridge_->AddHomePage(home_page);
    358   }
    359 }
    360 
    361 void Firefox3Importer::GetSearchEnginesXMLFiles(
    362     std::vector<FilePath>* files) {
    363   FilePath file = source_path_.AppendASCII("search.sqlite");
    364   if (!file_util::PathExists(file))
    365     return;
    366 
    367   sql::Connection db;
    368   if (!db.Open(file))
    369     return;
    370 
    371   const char* query = "SELECT engineid FROM engine_data "
    372                       "WHERE engineid NOT IN "
    373                       "(SELECT engineid FROM engine_data "
    374                       "WHERE name='hidden') "
    375                       "ORDER BY value ASC";
    376 
    377   sql::Statement s(db.GetUniqueStatement(query));
    378   if (!s)
    379     return;
    380 
    381   FilePath app_path = app_path_.AppendASCII("searchplugins");
    382   FilePath profile_path = source_path_.AppendASCII("searchplugins");
    383 
    384   // Firefox doesn't store a search engine in its sqlite database unless the
    385   // user has added a engine. So we get search engines from sqlite db as well
    386   // as from the file system.
    387   if (s.Step()) {
    388     const std::wstring kAppPrefix = L"[app]/";
    389     const std::wstring kProfilePrefix = L"[profile]/";
    390     do {
    391       FilePath file;
    392       std::wstring engine = UTF8ToWide(s.ColumnString(0));
    393 
    394       // The string contains [app]/<name>.xml or [profile]/<name>.xml where
    395       // the [app] and [profile] need to be replaced with the actual app or
    396       // profile path.
    397       size_t index = engine.find(kAppPrefix);
    398       if (index != std::wstring::npos) {
    399         // Remove '[app]/'.
    400         file = app_path.Append(FilePath::FromWStringHack(
    401                                    engine.substr(index + kAppPrefix.length())));
    402       } else if ((index = engine.find(kProfilePrefix)) != std::wstring::npos) {
    403         // Remove '[profile]/'.
    404           file = profile_path.Append(
    405               FilePath::FromWStringHack(
    406                   engine.substr(index + kProfilePrefix.length())));
    407       } else {
    408         // Looks like absolute path to the file.
    409         file = FilePath::FromWStringHack(engine);
    410       }
    411       files->push_back(file);
    412     } while (s.Step() && !cancelled());
    413   }
    414 
    415 #if defined(OS_LINUX)
    416   // Ubuntu-flavored Firefox3 supports locale-specific search engines via
    417   // locale-named subdirectories. They fall back to en-US.
    418   // See http://crbug.com/53899
    419   // TODO(jshin): we need to make sure our locale code matches that of
    420   // Firefox.
    421   FilePath locale_app_path = app_path.AppendASCII(locale_);
    422   FilePath default_locale_app_path = app_path.AppendASCII("en-US");
    423   if (file_util::DirectoryExists(locale_app_path))
    424     app_path = locale_app_path;
    425   else if (file_util::DirectoryExists(default_locale_app_path))
    426     app_path = default_locale_app_path;
    427 #endif
    428 
    429   // Get search engine definition from file system.
    430   file_util::FileEnumerator engines(app_path, false,
    431                                     file_util::FileEnumerator::FILES);
    432   for (FilePath engine_path = engines.Next(); !engine_path.value().empty();
    433        engine_path = engines.Next()) {
    434     files->push_back(engine_path);
    435   }
    436 }
    437 
    438 void Firefox3Importer::LoadRootNodeID(sql::Connection* db,
    439                                       int* toolbar_folder_id,
    440                                       int* menu_folder_id,
    441                                       int* unsorted_folder_id) {
    442   static const char* kToolbarFolderName = "toolbar";
    443   static const char* kMenuFolderName = "menu";
    444   static const char* kUnsortedFolderName = "unfiled";
    445 
    446   const char* query = "SELECT root_name, folder_id FROM moz_bookmarks_roots";
    447   sql::Statement s(db->GetUniqueStatement(query));
    448   if (!s)
    449     return;
    450 
    451   while (s.Step()) {
    452     std::string folder = s.ColumnString(0);
    453     int id = s.ColumnInt(1);
    454     if (folder == kToolbarFolderName)
    455       *toolbar_folder_id = id;
    456     else if (folder == kMenuFolderName)
    457       *menu_folder_id = id;
    458     else if (folder == kUnsortedFolderName)
    459       *unsorted_folder_id = id;
    460   }
    461 }
    462 
    463 void Firefox3Importer::LoadLivemarkIDs(sql::Connection* db,
    464                                        std::set<int>* livemark) {
    465   static const char* kFeedAnnotation = "livemark/feedURI";
    466   livemark->clear();
    467 
    468   const char* query = "SELECT b.item_id "
    469                       "FROM moz_anno_attributes a "
    470                       "JOIN moz_items_annos b ON a.id = b.anno_attribute_id "
    471                       "WHERE a.name = ? ";
    472   sql::Statement s(db->GetUniqueStatement(query));
    473   if (!s)
    474     return;
    475 
    476   s.BindString(0, kFeedAnnotation);
    477   while (s.Step() && !cancelled())
    478     livemark->insert(s.ColumnInt(0));
    479 }
    480 
    481 void Firefox3Importer::GetTopBookmarkFolder(sql::Connection* db,
    482                                             int folder_id,
    483                                             BookmarkList* list) {
    484   const char* query = "SELECT b.title "
    485                      "FROM moz_bookmarks b "
    486                      "WHERE b.type = 2 AND b.id = ? "
    487                      "ORDER BY b.position";
    488   sql::Statement s(db->GetUniqueStatement(query));
    489   if (!s)
    490     return;
    491 
    492   s.BindInt(0, folder_id);
    493   if (s.Step()) {
    494     BookmarkItem* item = new BookmarkItem;
    495     item->parent = -1;  // The top level folder has no parent.
    496     item->id = folder_id;
    497     item->title = s.ColumnString16(0);
    498     item->type = TYPE_FOLDER;
    499     item->favicon = 0;
    500     item->empty_folder = true;
    501     list->push_back(item);
    502   }
    503 }
    504 
    505 void Firefox3Importer::GetWholeBookmarkFolder(sql::Connection* db,
    506                                               BookmarkList* list,
    507                                               size_t position,
    508                                               bool* empty_folder) {
    509   if (position >= list->size()) {
    510     NOTREACHED();
    511     return;
    512   }
    513 
    514   const char* query = "SELECT b.id, h.url, COALESCE(b.title, h.title), "
    515          "b.type, k.keyword, b.dateAdded, h.favicon_id "
    516          "FROM moz_bookmarks b "
    517          "LEFT JOIN moz_places h ON b.fk = h.id "
    518          "LEFT JOIN moz_keywords k ON k.id = b.keyword_id "
    519          "WHERE b.type IN (1,2) AND b.parent = ? "
    520          "ORDER BY b.position";
    521   sql::Statement s(db->GetUniqueStatement(query));
    522   if (!s)
    523     return;
    524 
    525   s.BindInt(0, (*list)[position]->id);
    526   BookmarkList temp_list;
    527   while (s.Step()) {
    528     BookmarkItem* item = new BookmarkItem;
    529     item->parent = static_cast<int>(position);
    530     item->id = s.ColumnInt(0);
    531     item->url = GURL(s.ColumnString(1));
    532     item->title = s.ColumnString16(2);
    533     item->type = static_cast<BookmarkItemType>(s.ColumnInt(3));
    534     item->keyword = s.ColumnString(4);
    535     item->date_added = base::Time::FromTimeT(s.ColumnInt64(5)/1000000);
    536     item->favicon = s.ColumnInt64(6);
    537     item->empty_folder = true;
    538 
    539     temp_list.push_back(item);
    540     if (empty_folder != NULL)
    541       *empty_folder = false;
    542   }
    543 
    544   // Appends all items to the list.
    545   for (BookmarkList::iterator i = temp_list.begin();
    546        i != temp_list.end(); ++i) {
    547     list->push_back(*i);
    548     // Recursive add bookmarks in sub-folders.
    549     if ((*i)->type == TYPE_FOLDER)
    550       GetWholeBookmarkFolder(db, list, list->size() - 1, &(*i)->empty_folder);
    551   }
    552 }
    553 
    554 void Firefox3Importer::LoadFavicons(
    555     sql::Connection* db,
    556     const FaviconMap& favicon_map,
    557     std::vector<history::ImportedFaviconUsage>* favicons) {
    558   const char* query = "SELECT url, data FROM moz_favicons WHERE id=?";
    559   sql::Statement s(db->GetUniqueStatement(query));
    560   if (!s)
    561     return;
    562 
    563   for (FaviconMap::const_iterator i = favicon_map.begin();
    564        i != favicon_map.end(); ++i) {
    565     s.BindInt64(0, i->first);
    566     if (s.Step()) {
    567       history::ImportedFaviconUsage usage;
    568 
    569       usage.favicon_url = GURL(s.ColumnString(0));
    570       if (!usage.favicon_url.is_valid())
    571         continue;  // Don't bother importing favicons with invalid URLs.
    572 
    573       std::vector<unsigned char> data;
    574       s.ColumnBlobAsVector(1, &data);
    575       if (data.empty())
    576         continue;  // Data definitely invalid.
    577 
    578       if (!ReencodeFavicon(&data[0], data.size(), &usage.png_data))
    579         continue;  // Unable to decode.
    580 
    581       usage.urls = i->second;
    582       favicons->push_back(usage);
    583     }
    584     s.Reset();
    585   }
    586 }
    587