1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // This tool converts Hunspell .aff/.dic pairs to a combined binary dictionary 6 // format (.bdic). This format is more compact, and can be more efficiently 7 // read by the client application. 8 // 9 // We do this conversion manually before publishing dictionary files. It is not 10 // part of any build process. 11 // 12 // See PrintHelp() below for usage. 13 14 #include <stdio.h> 15 16 #include "base/at_exit.h" 17 #include "base/files/file_path.h" 18 #include "base/files/file_util.h" 19 #include "base/i18n/icu_util.h" 20 #include "base/logging.h" 21 #include "base/process/memory.h" 22 #include "base/strings/string_util.h" 23 #include "chrome/tools/convert_dict/aff_reader.h" 24 #include "chrome/tools/convert_dict/dic_reader.h" 25 #include "third_party/hunspell/google/bdict_reader.h" 26 #include "third_party/hunspell/google/bdict_writer.h" 27 28 namespace { 29 30 // Compares the given word list with the serialized trie to make sure they 31 // are the same. 32 bool VerifyWords(const convert_dict::DicReader::WordList& org_words, 33 const std::string& serialized) { 34 hunspell::BDictReader reader; 35 if (!reader.Init(reinterpret_cast<const unsigned char*>(serialized.data()), 36 serialized.size())) { 37 printf("BDict is invalid\n"); 38 return false; 39 } 40 hunspell::WordIterator iter = reader.GetAllWordIterator(); 41 42 int affix_ids[hunspell::BDict::MAX_AFFIXES_PER_WORD]; 43 44 static const int buf_size = 128; 45 char buf[buf_size]; 46 for (size_t i = 0; i < org_words.size(); i++) { 47 int affix_matches = iter.Advance(buf, buf_size, affix_ids); 48 if (affix_matches == 0) { 49 printf("Found the end before we expected\n"); 50 return false; 51 } 52 53 if (org_words[i].first != buf) { 54 printf("Word doesn't match, word #%s\n", buf); 55 return false; 56 } 57 58 if (affix_matches != static_cast<int>(org_words[i].second.size())) { 59 printf("Different number of affix indices, word #%s\n", buf); 60 return false; 61 } 62 63 // Check the individual affix indices. 64 for (size_t affix_index = 0; affix_index < org_words[i].second.size(); 65 affix_index++) { 66 if (affix_ids[affix_index] != org_words[i].second[affix_index]) { 67 printf("Index doesn't match, word #%s\n", buf); 68 return false; 69 } 70 } 71 } 72 73 return true; 74 } 75 76 int PrintHelp() { 77 printf("Usage: convert_dict <dicfile base name>\n\n"); 78 printf("Example:\n"); 79 printf(" convert_dict en-US\nwill read en-US.dic, en-US.dic_delta, and " 80 "en-US.aff from the current directory and generate en-US.bdic\n\n"); 81 return 1; 82 } 83 84 } // namespace 85 86 #if defined(OS_WIN) 87 int wmain(int argc, wchar_t* argv[]) { 88 #else 89 int main(int argc, char* argv[]) { 90 #endif 91 base::EnableTerminationOnHeapCorruption(); 92 if (argc != 2) 93 return PrintHelp(); 94 95 base::AtExitManager exit_manager; 96 base::i18n::InitializeICU(); 97 98 base::FilePath file_base = base::FilePath(argv[1]); 99 100 base::FilePath aff_path = 101 file_base.ReplaceExtension(FILE_PATH_LITERAL(".aff")); 102 printf("Reading %" PRFilePath " ...\n", aff_path.value().c_str()); 103 convert_dict::AffReader aff_reader(aff_path); 104 if (!aff_reader.Read()) { 105 printf("Unable to read the aff file.\n"); 106 return 1; 107 } 108 109 base::FilePath dic_path = 110 file_base.ReplaceExtension(FILE_PATH_LITERAL(".dic")); 111 printf("Reading %" PRFilePath " ...\n", dic_path.value().c_str()); 112 // DicReader will also read the .dic_delta file. 113 convert_dict::DicReader dic_reader(dic_path); 114 if (!dic_reader.Read(&aff_reader)) { 115 printf("Unable to read the dic file.\n"); 116 return 1; 117 } 118 119 hunspell::BDictWriter writer; 120 writer.SetComment(aff_reader.comments()); 121 writer.SetAffixRules(aff_reader.affix_rules()); 122 writer.SetAffixGroups(aff_reader.GetAffixGroups()); 123 writer.SetReplacements(aff_reader.replacements()); 124 writer.SetOtherCommands(aff_reader.other_commands()); 125 writer.SetWords(dic_reader.words()); 126 127 printf("Serializing...\n"); 128 std::string serialized = writer.GetBDict(); 129 130 printf("Verifying...\n"); 131 if (!VerifyWords(dic_reader.words(), serialized)) { 132 printf("ERROR converting, the dictionary does not check out OK."); 133 return 1; 134 } 135 136 base::FilePath out_path = 137 file_base.ReplaceExtension(FILE_PATH_LITERAL(".bdic")); 138 printf("Writing %" PRFilePath " ...\n", out_path.value().c_str()); 139 FILE* out_file = base::OpenFile(out_path, "wb"); 140 if (!out_file) { 141 printf("ERROR writing file\n"); 142 return 1; 143 } 144 size_t written = fwrite(&serialized[0], 1, serialized.size(), out_file); 145 CHECK(written == serialized.size()); 146 base::CloseFile(out_file); 147 148 return 0; 149 } 150