Home | History | Annotate | Download | only in convert_dict 
      1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 // This tool converts Hunspell .aff/.dic pairs to a combined binary dictionary
      6 // format (.bdic). This format is more compact, and can be more efficiently
      7 // read by the client application.
      8 //
      9 // We do this conversion manually before publishing dictionary files. It is not
     10 // part of any build process.
     11 //
     12 // See PrintHelp() below for usage.
     13 
     14 #include <stdio.h>
     15 
     16 #include "base/at_exit.h"
     17 #include "base/files/file_path.h"
     18 #include "base/files/file_util.h"
     19 #include "base/i18n/icu_util.h"
     20 #include "base/logging.h"
     21 #include "base/process/memory.h"
     22 #include "base/strings/string_util.h"
     23 #include "chrome/tools/convert_dict/aff_reader.h"
     24 #include "chrome/tools/convert_dict/dic_reader.h"
     25 #include "third_party/hunspell/google/bdict_reader.h"
     26 #include "third_party/hunspell/google/bdict_writer.h"
     27 
     28 namespace {
     29 
     30 // Compares the given word list with the serialized trie to make sure they
     31 // are the same.
     32 bool VerifyWords(const convert_dict::DicReader::WordList& org_words,
     33                  const std::string& serialized) {
     34   hunspell::BDictReader reader;
     35   if (!reader.Init(reinterpret_cast<const unsigned char*>(serialized.data()),
     36                    serialized.size())) {
     37     printf("BDict is invalid\n");
     38     return false;
     39   }
     40   hunspell::WordIterator iter = reader.GetAllWordIterator();
     41 
     42   int affix_ids[hunspell::BDict::MAX_AFFIXES_PER_WORD];
     43 
     44   static const int buf_size = 128;
     45   char buf[buf_size];
     46   for (size_t i = 0; i < org_words.size(); i++) {
     47     int affix_matches = iter.Advance(buf, buf_size, affix_ids);
     48     if (affix_matches == 0) {
     49       printf("Found the end before we expected\n");
     50       return false;
     51     }
     52 
     53     if (org_words[i].first != buf) {
     54       printf("Word doesn't match, word #%s\n", buf);
     55       return false;
     56     }
     57 
     58     if (affix_matches != static_cast<int>(org_words[i].second.size())) {
     59       printf("Different number of affix indices, word #%s\n", buf);
     60       return false;
     61     }
     62 
     63     // Check the individual affix indices.
     64     for (size_t affix_index = 0; affix_index < org_words[i].second.size();
     65          affix_index++) {
     66       if (affix_ids[affix_index] != org_words[i].second[affix_index]) {
     67         printf("Index doesn't match, word #%s\n", buf);
     68         return false;
     69       }
     70     }
     71   }
     72 
     73   return true;
     74 }
     75 
     76 int PrintHelp() {
     77   printf("Usage: convert_dict <dicfile base name>\n\n");
     78   printf("Example:\n");
     79   printf("  convert_dict en-US\nwill read en-US.dic, en-US.dic_delta, and "
     80          "en-US.aff from the current directory and generate en-US.bdic\n\n");
     81   return 1;
     82 }
     83 
     84 }  // namespace
     85 
     86 #if defined(OS_WIN)
     87 int wmain(int argc, wchar_t* argv[]) {
     88 #else
     89 int main(int argc, char* argv[]) {
     90 #endif
     91   base::EnableTerminationOnHeapCorruption();
     92   if (argc != 2)
     93     return PrintHelp();
     94 
     95   base::AtExitManager exit_manager;
     96   base::i18n::InitializeICU();
     97 
     98   base::FilePath file_base = base::FilePath(argv[1]);
     99 
    100   base::FilePath aff_path =
    101       file_base.ReplaceExtension(FILE_PATH_LITERAL(".aff"));
    102   printf("Reading %" PRFilePath " ...\n", aff_path.value().c_str());
    103   convert_dict::AffReader aff_reader(aff_path);
    104   if (!aff_reader.Read()) {
    105     printf("Unable to read the aff file.\n");
    106     return 1;
    107   }
    108 
    109   base::FilePath dic_path =
    110       file_base.ReplaceExtension(FILE_PATH_LITERAL(".dic"));
    111   printf("Reading %" PRFilePath " ...\n", dic_path.value().c_str());
    112   // DicReader will also read the .dic_delta file.
    113   convert_dict::DicReader dic_reader(dic_path);
    114   if (!dic_reader.Read(&aff_reader)) {
    115     printf("Unable to read the dic file.\n");
    116     return 1;
    117   }
    118 
    119   hunspell::BDictWriter writer;
    120   writer.SetComment(aff_reader.comments());
    121   writer.SetAffixRules(aff_reader.affix_rules());
    122   writer.SetAffixGroups(aff_reader.GetAffixGroups());
    123   writer.SetReplacements(aff_reader.replacements());
    124   writer.SetOtherCommands(aff_reader.other_commands());
    125   writer.SetWords(dic_reader.words());
    126 
    127   printf("Serializing...\n");
    128   std::string serialized = writer.GetBDict();
    129 
    130   printf("Verifying...\n");
    131   if (!VerifyWords(dic_reader.words(), serialized)) {
    132     printf("ERROR converting, the dictionary does not check out OK.");
    133     return 1;
    134   }
    135 
    136   base::FilePath out_path =
    137       file_base.ReplaceExtension(FILE_PATH_LITERAL(".bdic"));
    138   printf("Writing %" PRFilePath " ...\n", out_path.value().c_str());
    139   FILE* out_file = base::OpenFile(out_path, "wb");
    140   if (!out_file) {
    141     printf("ERROR writing file\n");
    142     return 1;
    143   }
    144   size_t written = fwrite(&serialized[0], 1, serialized.size(), out_file);
    145   CHECK(written == serialized.size());
    146   base::CloseFile(out_file);
    147 
    148   return 0;
    149 }
    150