1 /* 2 ********************************************************************** 3 * Copyright (C) 2002-2015, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 * 7 * File gendict.cpp 8 */ 9 10 #include "unicode/utypes.h" 11 #include "unicode/uchar.h" 12 #include "unicode/ucnv.h" 13 #include "unicode/uniset.h" 14 #include "unicode/unistr.h" 15 #include "unicode/uclean.h" 16 #include "unicode/udata.h" 17 #include "unicode/putil.h" 18 #include "unicode/ucharstriebuilder.h" 19 #include "unicode/bytestriebuilder.h" 20 #include "unicode/ucharstrie.h" 21 #include "unicode/bytestrie.h" 22 #include "unicode/ucnv.h" 23 #include "unicode/utf16.h" 24 25 #include "charstr.h" 26 #include "dictionarydata.h" 27 #include "uoptions.h" 28 #include "unewdata.h" 29 #include "cmemory.h" 30 #include "uassert.h" 31 #include "ucbuf.h" 32 #include "toolutil.h" 33 #include "cstring.h" 34 35 #include <stdio.h> 36 #include <stdlib.h> 37 #include <string.h> 38 39 #include "putilimp.h" 40 UDate startTime; 41 42 static int elapsedTime() { 43 return (int)uprv_floor((uprv_getRawUTCtime()-startTime)/1000.0); 44 } 45 46 U_NAMESPACE_USE 47 48 static char *progName; 49 static UOption options[]={ 50 UOPTION_HELP_H, /* 0 */ 51 UOPTION_HELP_QUESTION_MARK, /* 1 */ 52 UOPTION_VERBOSE, /* 2 */ 53 UOPTION_ICUDATADIR, /* 4 */ 54 UOPTION_COPYRIGHT, /* 5 */ 55 { "uchars", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 6 */ 56 { "bytes", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0}, /* 7 */ 57 { "transform", NULL, NULL, NULL, '\1', UOPT_REQUIRES_ARG, 0}, /* 8 */ 58 UOPTION_QUIET, /* 9 */ 59 }; 60 61 enum arguments { 62 ARG_HELP = 0, 63 ARG_QMARK, 64 ARG_VERBOSE, 65 ARG_ICUDATADIR, 66 ARG_COPYRIGHT, 67 ARG_UCHARS, 68 ARG_BYTES, 69 ARG_TRANSFORM, 70 ARG_QUIET 71 }; 72 73 // prints out the standard usage method describing command line arguments, 74 // then bails out with the desired exit code 75 static void usageAndDie(UErrorCode retCode) { 76 fprintf((U_SUCCESS(retCode) ? stdout : stderr), "Usage: %s -trietype [-options] input-dictionary-file output-file\n", progName); 77 fprintf((U_SUCCESS(retCode) ? stdout : stderr), 78 "\tRead in a word list and write out a string trie dictionary\n" 79 "options:\n" 80 "\t-h or -? or --help this usage text\n" 81 "\t-V or --version show a version message\n" 82 "\t-c or --copyright include a copyright notice\n" 83 "\t-v or --verbose turn on verbose output\n" 84 "\t-q or --quiet do not display warnings and progress\n" 85 "\t-i or --icudatadir directory for locating any needed intermediate data files,\n" // TODO: figure out if we need this option 86 "\t followed by path, defaults to %s\n" 87 "\t--uchars output a UCharsTrie (mutually exclusive with -b!)\n" 88 "\t--bytes output a BytesTrie (mutually exclusive with -u!)\n" 89 "\t--transform the kind of transform to use (eg --transform offset-40A3,\n" 90 "\t which specifies an offset transform with constant 0x40A3)\n", 91 u_getDataDirectory()); 92 exit(retCode); 93 } 94 95 96 /* UDataInfo cf. udata.h */ 97 static UDataInfo dataInfo = { 98 sizeof(UDataInfo), 99 0, 100 101 U_IS_BIG_ENDIAN, 102 U_CHARSET_FAMILY, 103 U_SIZEOF_UCHAR, 104 0, 105 106 { 0x44, 0x69, 0x63, 0x74 }, /* "Dict" */ 107 { 1, 0, 0, 0 }, /* format version */ 108 { 0, 0, 0, 0 } /* data version */ 109 }; 110 111 #if !UCONFIG_NO_BREAK_ITERATION 112 113 // A wrapper for both BytesTrieBuilder and UCharsTrieBuilder. 114 // may want to put this somewhere in ICU, as it could be useful outside 115 // of this tool? 116 class DataDict { 117 private: 118 BytesTrieBuilder *bt; 119 UCharsTrieBuilder *ut; 120 UChar32 transformConstant; 121 int32_t transformType; 122 public: 123 // constructs a new data dictionary. if there is an error, 124 // it will be returned in status 125 // isBytesTrie != 0 will produce a BytesTrieBuilder, 126 // isBytesTrie == 0 will produce a UCharsTrieBuilder 127 DataDict(UBool isBytesTrie, UErrorCode &status) : bt(NULL), ut(NULL), 128 transformConstant(0), transformType(DictionaryData::TRANSFORM_NONE) { 129 if (isBytesTrie) { 130 bt = new BytesTrieBuilder(status); 131 } else { 132 ut = new UCharsTrieBuilder(status); 133 } 134 } 135 136 ~DataDict() { 137 delete bt; 138 delete ut; 139 } 140 141 private: 142 char transform(UChar32 c, UErrorCode &status) { 143 if (transformType == DictionaryData::TRANSFORM_TYPE_OFFSET) { 144 if (c == 0x200D) { return (char)0xFF; } 145 else if (c == 0x200C) { return (char)0xFE; } 146 int32_t delta = c - transformConstant; 147 if (delta < 0 || 0xFD < delta) { 148 fprintf(stderr, "Codepoint U+%04lx out of range for --transform offset-%04lx!\n", 149 (long)c, (long)transformConstant); 150 exit(U_ILLEGAL_ARGUMENT_ERROR); // TODO: should return and print the line number 151 } 152 return (char)delta; 153 } else { // no such transform type 154 status = U_INTERNAL_PROGRAM_ERROR; 155 return (char)c; // it should be noted this transform type will not generally work 156 } 157 } 158 159 void transform(const UnicodeString &word, CharString &buf, UErrorCode &errorCode) { 160 UChar32 c = 0; 161 int32_t len = word.length(); 162 for (int32_t i = 0; i < len; i += U16_LENGTH(c)) { 163 c = word.char32At(i); 164 buf.append(transform(c, errorCode), errorCode); 165 } 166 } 167 168 public: 169 // sets the desired transformation data. 170 // should be populated from a command line argument 171 // so far the only acceptable format is offset-<hex constant> 172 // eventually others (mask-<hex constant>?) may be enabled 173 // more complex functions may be more difficult 174 void setTransform(const char *t) { 175 if (strncmp(t, "offset-", 7) == 0) { 176 char *end; 177 unsigned long base = uprv_strtoul(t + 7, &end, 16); 178 if (end == (t + 7) || *end != 0 || base > 0x10FF80) { 179 fprintf(stderr, "Syntax for offset value in --transform offset-%s invalid!\n", t + 7); 180 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 181 } 182 transformType = DictionaryData::TRANSFORM_TYPE_OFFSET; 183 transformConstant = (UChar32)base; 184 } 185 else { 186 fprintf(stderr, "Invalid transform specified: %s\n", t); 187 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 188 } 189 } 190 191 // add a word to the trie 192 void addWord(const UnicodeString &word, int32_t value, UErrorCode &status) { 193 if (bt) { 194 CharString buf; 195 transform(word, buf, status); 196 bt->add(buf.toStringPiece(), value, status); 197 } 198 if (ut) { ut->add(word, value, status); } 199 } 200 201 // if we are a bytestrie, give back the StringPiece representing the serialized version of us 202 StringPiece serializeBytes(UErrorCode &status) { 203 return bt->buildStringPiece(USTRINGTRIE_BUILD_SMALL, status); 204 } 205 206 // if we are a ucharstrie, produce the UnicodeString representing the serialized version of us 207 void serializeUChars(UnicodeString &s, UErrorCode &status) { 208 ut->buildUnicodeString(USTRINGTRIE_BUILD_SMALL, s, status); 209 } 210 211 int32_t getTransform() { 212 return (int32_t)(transformType | transformConstant); 213 } 214 }; 215 #endif 216 217 static const UChar LINEFEED_CHARACTER = 0x000A; 218 static const UChar CARRIAGE_RETURN_CHARACTER = 0x000D; 219 220 static UBool readLine(UCHARBUF *f, UnicodeString &fileLine, IcuToolErrorCode &errorCode) { 221 int32_t lineLength; 222 const UChar *line = ucbuf_readline(f, &lineLength, errorCode); 223 if(line == NULL || errorCode.isFailure()) { return FALSE; } 224 // Strip trailing CR/LF, comments, and spaces. 225 const UChar *comment = u_memchr(line, 0x23, lineLength); // '#' 226 if(comment != NULL) { 227 lineLength = (int32_t)(comment - line); 228 } else { 229 while(lineLength > 0 && (line[lineLength - 1] == CARRIAGE_RETURN_CHARACTER || line[lineLength - 1] == LINEFEED_CHARACTER)) { --lineLength; } 230 } 231 while(lineLength > 0 && u_isspace(line[lineLength - 1])) { --lineLength; } 232 fileLine.setTo(FALSE, line, lineLength); 233 return TRUE; 234 } 235 236 //---------------------------------------------------------------------------- 237 // 238 // main for gendict 239 // 240 //---------------------------------------------------------------------------- 241 int main(int argc, char **argv) { 242 // 243 // Pick up and check the command line arguments, 244 // using the standard ICU tool utils option handling. 245 // 246 U_MAIN_INIT_ARGS(argc, argv); 247 progName = argv[0]; 248 argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options); 249 if(argc<0) { 250 // Unrecognized option 251 fprintf(stderr, "error in command line argument \"%s\"\n", argv[-argc]); 252 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 253 } 254 255 if(options[ARG_HELP].doesOccur || options[ARG_QMARK].doesOccur) { 256 // -? or -h for help. 257 usageAndDie(U_ZERO_ERROR); 258 } 259 260 UBool verbose = options[ARG_VERBOSE].doesOccur; 261 UBool quiet = options[ARG_QUIET].doesOccur; 262 263 if (argc < 3) { 264 fprintf(stderr, "input and output file must both be specified.\n"); 265 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 266 } 267 const char *outFileName = argv[2]; 268 const char *wordFileName = argv[1]; 269 270 startTime = uprv_getRawUTCtime(); // initialize start timer 271 272 if (options[ARG_ICUDATADIR].doesOccur) { 273 u_setDataDirectory(options[ARG_ICUDATADIR].value); 274 } 275 276 const char *copyright = NULL; 277 if (options[ARG_COPYRIGHT].doesOccur) { 278 copyright = U_COPYRIGHT_STRING; 279 } 280 281 if (options[ARG_UCHARS].doesOccur == options[ARG_BYTES].doesOccur) { 282 fprintf(stderr, "you must specify exactly one type of trie to output!\n"); 283 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 284 } 285 UBool isBytesTrie = options[ARG_BYTES].doesOccur; 286 if (isBytesTrie != options[ARG_TRANSFORM].doesOccur) { 287 fprintf(stderr, "you must provide a transformation for a bytes trie, and must not provide one for a uchars trie!\n"); 288 usageAndDie(U_ILLEGAL_ARGUMENT_ERROR); 289 } 290 291 IcuToolErrorCode status("gendict/main()"); 292 293 #if UCONFIG_NO_BREAK_ITERATION || UCONFIG_NO_FILE_IO 294 const char* outDir=NULL; 295 296 UNewDataMemory *pData; 297 char msg[1024]; 298 UErrorCode tempstatus = U_ZERO_ERROR; 299 300 /* write message with just the name */ // potential for a buffer overflow here... 301 sprintf(msg, "gendict writes dummy %s because of UCONFIG_NO_BREAK_ITERATION and/or UCONFIG_NO_FILE_IO, see uconfig.h", outFileName); 302 fprintf(stderr, "%s\n", msg); 303 304 /* write the dummy data file */ 305 pData = udata_create(outDir, NULL, outFileName, &dataInfo, NULL, &tempstatus); 306 udata_writeBlock(pData, msg, strlen(msg)); 307 udata_finish(pData, &tempstatus); 308 return (int)tempstatus; 309 310 #else 311 // Read in the dictionary source file 312 if (verbose) { printf("Opening file %s...\n", wordFileName); } 313 const char *codepage = "UTF-8"; 314 UCHARBUF *f = ucbuf_open(wordFileName, &codepage, TRUE, FALSE, status); 315 if (status.isFailure()) { 316 fprintf(stderr, "error opening input file: ICU Error \"%s\"\n", status.errorName()); 317 exit(status.reset()); 318 } 319 if (verbose) { printf("Initializing dictionary builder of type %s...\n", (isBytesTrie ? "BytesTrie" : "UCharsTrie")); } 320 DataDict dict(isBytesTrie, status); 321 if (status.isFailure()) { 322 fprintf(stderr, "new DataDict: ICU Error \"%s\"\n", status.errorName()); 323 exit(status.reset()); 324 } 325 if (options[ARG_TRANSFORM].doesOccur) { 326 dict.setTransform(options[ARG_TRANSFORM].value); 327 } 328 329 UnicodeString fileLine; 330 if (verbose) { puts("Adding words to dictionary..."); } 331 UBool hasValues = FALSE; 332 UBool hasValuelessContents = FALSE; 333 int lineCount = 0; 334 int wordCount = 0; 335 int minlen = 255; 336 int maxlen = 0; 337 UBool isOk = TRUE; 338 while (readLine(f, fileLine, status)) { 339 lineCount++; 340 if (fileLine.isEmpty()) continue; 341 342 // Parse word [spaces value]. 343 int32_t keyLen; 344 for (keyLen = 0; keyLen < fileLine.length() && !u_isspace(fileLine[keyLen]); ++keyLen) {} 345 if (keyLen == 0) { 346 fprintf(stderr, "Error: no word on line %i!\n", lineCount); 347 isOk = FALSE; 348 continue; 349 } 350 int32_t valueStart; 351 for (valueStart = keyLen; 352 valueStart < fileLine.length() && u_isspace(fileLine[valueStart]); 353 ++valueStart) {} 354 355 if (keyLen < valueStart) { 356 int32_t valueLength = fileLine.length() - valueStart; 357 if (valueLength > 15) { 358 fprintf(stderr, "Error: value too long on line %i!\n", lineCount); 359 isOk = FALSE; 360 continue; 361 } 362 char s[16]; 363 fileLine.extract(valueStart, valueLength, s, 16, US_INV); 364 char *end; 365 unsigned long value = uprv_strtoul(s, &end, 0); 366 if (end == s || *end != 0 || (int32_t)uprv_strlen(s) != valueLength || value > 0xffffffff) { 367 fprintf(stderr, "Error: value syntax error or value too large on line %i!\n", lineCount); 368 isOk = FALSE; 369 continue; 370 } 371 dict.addWord(fileLine.tempSubString(0, keyLen), (int32_t)value, status); 372 hasValues = TRUE; 373 wordCount++; 374 if (keyLen < minlen) minlen = keyLen; 375 if (keyLen > maxlen) maxlen = keyLen; 376 } else { 377 dict.addWord(fileLine.tempSubString(0, keyLen), 0, status); 378 hasValuelessContents = TRUE; 379 wordCount++; 380 if (keyLen < minlen) minlen = keyLen; 381 if (keyLen > maxlen) maxlen = keyLen; 382 } 383 384 if (status.isFailure()) { 385 fprintf(stderr, "ICU Error \"%s\": Failed to add word to trie at input line %d in input file\n", 386 status.errorName(), lineCount); 387 exit(status.reset()); 388 } 389 } 390 if (verbose) { printf("Processed %d lines, added %d words, minlen %d, maxlen %d\n", lineCount, wordCount, minlen, maxlen); } 391 392 if (!isOk && status.isSuccess()) { 393 status.set(U_ILLEGAL_ARGUMENT_ERROR); 394 } 395 if (hasValues && hasValuelessContents) { 396 fprintf(stderr, "warning: file contained both valued and unvalued strings!\n"); 397 } 398 399 if (verbose) { printf("Serializing data...isBytesTrie? %d\n", isBytesTrie); } 400 int32_t outDataSize; 401 const void *outData; 402 UnicodeString usp; 403 if (isBytesTrie) { 404 StringPiece sp = dict.serializeBytes(status); 405 outDataSize = sp.size(); 406 outData = sp.data(); 407 } else { 408 dict.serializeUChars(usp, status); 409 outDataSize = usp.length() * U_SIZEOF_UCHAR; 410 outData = usp.getBuffer(); 411 } 412 if (status.isFailure()) { 413 fprintf(stderr, "gendict: got failure of type %s while serializing, if U_ILLEGAL_ARGUMENT_ERROR possibly due to duplicate dictionary entries\n", status.errorName()); 414 exit(status.reset()); 415 } 416 if (verbose) { puts("Opening output file..."); } 417 UNewDataMemory *pData = udata_create(NULL, NULL, outFileName, &dataInfo, copyright, status); 418 if (status.isFailure()) { 419 fprintf(stderr, "gendict: could not open output file \"%s\", \"%s\"\n", outFileName, status.errorName()); 420 exit(status.reset()); 421 } 422 423 if (verbose) { puts("Writing to output file..."); } 424 int32_t indexes[DictionaryData::IX_COUNT] = { 425 DictionaryData::IX_COUNT * sizeof(int32_t), 0, 0, 0, 0, 0, 0, 0 426 }; 427 int32_t size = outDataSize + indexes[DictionaryData::IX_STRING_TRIE_OFFSET]; 428 indexes[DictionaryData::IX_RESERVED1_OFFSET] = size; 429 indexes[DictionaryData::IX_RESERVED2_OFFSET] = size; 430 indexes[DictionaryData::IX_TOTAL_SIZE] = size; 431 432 indexes[DictionaryData::IX_TRIE_TYPE] = isBytesTrie ? DictionaryData::TRIE_TYPE_BYTES : DictionaryData::TRIE_TYPE_UCHARS; 433 if (hasValues) { 434 indexes[DictionaryData::IX_TRIE_TYPE] |= DictionaryData::TRIE_HAS_VALUES; 435 } 436 437 indexes[DictionaryData::IX_TRANSFORM] = dict.getTransform(); 438 udata_writeBlock(pData, indexes, sizeof(indexes)); 439 udata_writeBlock(pData, outData, outDataSize); 440 size_t bytesWritten = udata_finish(pData, status); 441 if (status.isFailure()) { 442 fprintf(stderr, "gendict: error \"%s\" writing the output file\n", status.errorName()); 443 exit(status.reset()); 444 } 445 446 if (bytesWritten != (size_t)size) { 447 fprintf(stderr, "Error writing to output file \"%s\"\n", outFileName); 448 exit(U_INTERNAL_PROGRAM_ERROR); 449 } 450 451 if (!quiet) { printf("%s: done writing\t%s (%ds).\n", progName, outFileName, elapsedTime()); } 452 453 #ifdef TEST_GENDICT 454 if (isBytesTrie) { 455 BytesTrie::Iterator it(outData, outDataSize, status); 456 while (it.hasNext()) { 457 it.next(status); 458 const StringPiece s = it.getString(); 459 int32_t val = it.getValue(); 460 printf("%s -> %i\n", s.data(), val); 461 } 462 } else { 463 UCharsTrie::Iterator it((const UChar *)outData, outDataSize, status); 464 while (it.hasNext()) { 465 it.next(status); 466 const UnicodeString s = it.getString(); 467 int32_t val = it.getValue(); 468 char tmp[1024]; 469 s.extract(0, s.length(), tmp, 1024); 470 printf("%s -> %i\n", tmp, val); 471 } 472 } 473 #endif 474 475 return 0; 476 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ 477 } 478
