1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 1998-2012, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * 9 * File parse.cpp 10 * 11 * Modification History: 12 * 13 * Date Name Description 14 * 05/26/99 stephen Creation. 15 * 02/25/00 weiv Overhaul to write udata 16 * 5/10/01 Ram removed ustdio dependency 17 * 06/10/2001 Dominic Ludlam <dom (at) recoil.org> Rewritten 18 ******************************************************************************* 19 */ 20 21 #include "ucol_imp.h" 22 #include "parse.h" 23 #include "errmsg.h" 24 #include "uhash.h" 25 #include "cmemory.h" 26 #include "cstring.h" 27 #include "uinvchar.h" 28 #include "read.h" 29 #include "ustr.h" 30 #include "reslist.h" 31 #include "rbt_pars.h" 32 #include "genrb.h" 33 #include "unicode/ustring.h" 34 #include "unicode/uscript.h" 35 #include "unicode/putil.h" 36 #include <stdio.h> 37 38 /* Number of tokens to read ahead of the current stream position */ 39 #define MAX_LOOKAHEAD 3 40 41 #define CR 0x000D 42 #define LF 0x000A 43 #define SPACE 0x0020 44 #define TAB 0x0009 45 #define ESCAPE 0x005C 46 #define HASH 0x0023 47 #define QUOTE 0x0027 48 #define ZERO 0x0030 49 #define STARTCOMMAND 0x005B 50 #define ENDCOMMAND 0x005D 51 #define OPENSQBRACKET 0x005B 52 #define CLOSESQBRACKET 0x005D 53 54 struct Lookahead 55 { 56 enum ETokenType type; 57 struct UString value; 58 struct UString comment; 59 uint32_t line; 60 }; 61 62 /* keep in sync with token defines in read.h */ 63 const char *tokenNames[TOK_TOKEN_COUNT] = 64 { 65 "string", /* A string token, such as "MonthNames" */ 66 "'{'", /* An opening brace character */ 67 "'}'", /* A closing brace character */ 68 "','", /* A comma */ 69 "':'", /* A colon */ 70 71 "<end of file>", /* End of the file has been reached successfully */ 72 "<end of line>" 73 }; 74 75 /* Just to store "TRUE" */ 76 //static const UChar trueValue[] = {0x0054, 0x0052, 0x0055, 0x0045, 0x0000}; 77 78 typedef struct { 79 struct Lookahead lookahead[MAX_LOOKAHEAD + 1]; 80 uint32_t lookaheadPosition; 81 UCHARBUF *buffer; 82 struct SRBRoot *bundle; 83 const char *inputdir; 84 uint32_t inputdirLength; 85 const char *outputdir; 86 uint32_t outputdirLength; 87 UBool makeBinaryCollation; 88 } ParseState; 89 90 static UBool gOmitCollationRules = FALSE; 91 92 typedef struct SResource * 93 ParseResourceFunction(ParseState* state, char *tag, uint32_t startline, const struct UString* comment, UErrorCode *status); 94 95 static struct SResource *parseResource(ParseState* state, char *tag, const struct UString *comment, UErrorCode *status); 96 97 /* The nature of the lookahead buffer: 98 There are MAX_LOOKAHEAD + 1 slots, used as a circular buffer. This provides 99 MAX_LOOKAHEAD lookahead tokens and a slot for the current token and value. 100 When getToken is called, the current pointer is moved to the next slot and the 101 old slot is filled with the next token from the reader by calling getNextToken. 102 The token values are stored in the slot, which means that token values don't 103 survive a call to getToken, ie. 104 105 UString *value; 106 107 getToken(&value, NULL, status); 108 getToken(NULL, NULL, status); bad - value is now a different string 109 */ 110 static void 111 initLookahead(ParseState* state, UCHARBUF *buf, UErrorCode *status) 112 { 113 static uint32_t initTypeStrings = 0; 114 uint32_t i; 115 116 if (!initTypeStrings) 117 { 118 initTypeStrings = 1; 119 } 120 121 state->lookaheadPosition = 0; 122 state->buffer = buf; 123 124 resetLineNumber(); 125 126 for (i = 0; i < MAX_LOOKAHEAD; i++) 127 { 128 state->lookahead[i].type = getNextToken(state->buffer, &state->lookahead[i].value, &state->lookahead[i].line, &state->lookahead[i].comment, status); 129 if (U_FAILURE(*status)) 130 { 131 return; 132 } 133 } 134 135 *status = U_ZERO_ERROR; 136 } 137 138 static void 139 cleanupLookahead(ParseState* state) 140 { 141 uint32_t i; 142 for (i = 0; i <= MAX_LOOKAHEAD; i++) 143 { 144 ustr_deinit(&state->lookahead[i].value); 145 ustr_deinit(&state->lookahead[i].comment); 146 } 147 148 } 149 150 static enum ETokenType 151 getToken(ParseState* state, struct UString **tokenValue, struct UString* comment, uint32_t *linenumber, UErrorCode *status) 152 { 153 enum ETokenType result; 154 uint32_t i; 155 156 result = state->lookahead[state->lookaheadPosition].type; 157 158 if (tokenValue != NULL) 159 { 160 *tokenValue = &state->lookahead[state->lookaheadPosition].value; 161 } 162 163 if (linenumber != NULL) 164 { 165 *linenumber = state->lookahead[state->lookaheadPosition].line; 166 } 167 168 if (comment != NULL) 169 { 170 ustr_cpy(comment, &(state->lookahead[state->lookaheadPosition].comment), status); 171 } 172 173 i = (state->lookaheadPosition + MAX_LOOKAHEAD) % (MAX_LOOKAHEAD + 1); 174 state->lookaheadPosition = (state->lookaheadPosition + 1) % (MAX_LOOKAHEAD + 1); 175 ustr_setlen(&state->lookahead[i].comment, 0, status); 176 ustr_setlen(&state->lookahead[i].value, 0, status); 177 state->lookahead[i].type = getNextToken(state->buffer, &state->lookahead[i].value, &state->lookahead[i].line, &state->lookahead[i].comment, status); 178 179 /* printf("getToken, returning %s\n", tokenNames[result]); */ 180 181 return result; 182 } 183 184 static enum ETokenType 185 peekToken(ParseState* state, uint32_t lookaheadCount, struct UString **tokenValue, uint32_t *linenumber, struct UString *comment, UErrorCode *status) 186 { 187 uint32_t i = (state->lookaheadPosition + lookaheadCount) % (MAX_LOOKAHEAD + 1); 188 189 if (U_FAILURE(*status)) 190 { 191 return TOK_ERROR; 192 } 193 194 if (lookaheadCount >= MAX_LOOKAHEAD) 195 { 196 *status = U_INTERNAL_PROGRAM_ERROR; 197 return TOK_ERROR; 198 } 199 200 if (tokenValue != NULL) 201 { 202 *tokenValue = &state->lookahead[i].value; 203 } 204 205 if (linenumber != NULL) 206 { 207 *linenumber = state->lookahead[i].line; 208 } 209 210 if(comment != NULL){ 211 ustr_cpy(comment, &(state->lookahead[state->lookaheadPosition].comment), status); 212 } 213 214 return state->lookahead[i].type; 215 } 216 217 static void 218 expect(ParseState* state, enum ETokenType expectedToken, struct UString **tokenValue, struct UString *comment, uint32_t *linenumber, UErrorCode *status) 219 { 220 uint32_t line; 221 222 enum ETokenType token = getToken(state, tokenValue, comment, &line, status); 223 224 if (linenumber != NULL) 225 { 226 *linenumber = line; 227 } 228 229 if (U_FAILURE(*status)) 230 { 231 return; 232 } 233 234 if (token != expectedToken) 235 { 236 *status = U_INVALID_FORMAT_ERROR; 237 error(line, "expecting %s, got %s", tokenNames[expectedToken], tokenNames[token]); 238 } 239 else 240 { 241 *status = U_ZERO_ERROR; 242 } 243 } 244 245 static char *getInvariantString(ParseState* state, uint32_t *line, struct UString *comment, UErrorCode *status) 246 { 247 struct UString *tokenValue; 248 char *result; 249 uint32_t count; 250 251 expect(state, TOK_STRING, &tokenValue, comment, line, status); 252 253 if (U_FAILURE(*status)) 254 { 255 return NULL; 256 } 257 258 count = u_strlen(tokenValue->fChars); 259 if(!uprv_isInvariantUString(tokenValue->fChars, count)) { 260 *status = U_INVALID_FORMAT_ERROR; 261 error(*line, "invariant characters required for table keys, binary data, etc."); 262 return NULL; 263 } 264 265 result = static_cast<char *>(uprv_malloc(count+1)); 266 267 if (result == NULL) 268 { 269 *status = U_MEMORY_ALLOCATION_ERROR; 270 return NULL; 271 } 272 273 u_UCharsToChars(tokenValue->fChars, result, count+1); 274 return result; 275 } 276 277 static struct SResource * 278 parseUCARules(ParseState* state, char *tag, uint32_t startline, const struct UString* /*comment*/, UErrorCode *status) 279 { 280 struct SResource *result = NULL; 281 struct UString *tokenValue; 282 FileStream *file = NULL; 283 char filename[256] = { '\0' }; 284 char cs[128] = { '\0' }; 285 uint32_t line; 286 UBool quoted = FALSE; 287 UCHARBUF *ucbuf=NULL; 288 UChar32 c = 0; 289 const char* cp = NULL; 290 UChar *pTarget = NULL; 291 UChar *target = NULL; 292 UChar *targetLimit = NULL; 293 int32_t size = 0; 294 295 expect(state, TOK_STRING, &tokenValue, NULL, &line, status); 296 297 if(isVerbose()){ 298 printf(" %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 299 } 300 301 if (U_FAILURE(*status)) 302 { 303 return NULL; 304 } 305 /* make the filename including the directory */ 306 if (state->inputdir != NULL) 307 { 308 uprv_strcat(filename, state->inputdir); 309 310 if (state->inputdir[state->inputdirLength - 1] != U_FILE_SEP_CHAR) 311 { 312 uprv_strcat(filename, U_FILE_SEP_STRING); 313 } 314 } 315 316 u_UCharsToChars(tokenValue->fChars, cs, tokenValue->fLength); 317 318 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 319 320 if (U_FAILURE(*status)) 321 { 322 return NULL; 323 } 324 uprv_strcat(filename, cs); 325 326 if(gOmitCollationRules) { 327 return res_none(); 328 } 329 330 ucbuf = ucbuf_open(filename, &cp, getShowWarning(),FALSE, status); 331 332 if (U_FAILURE(*status)) { 333 error(line, "An error occured while opening the input file %s\n", filename); 334 return NULL; 335 } 336 337 /* We allocate more space than actually required 338 * since the actual size needed for storing UChars 339 * is not known in UTF-8 byte stream 340 */ 341 size = ucbuf_size(ucbuf) + 1; 342 pTarget = (UChar*) uprv_malloc(U_SIZEOF_UCHAR * size); 343 uprv_memset(pTarget, 0, size*U_SIZEOF_UCHAR); 344 target = pTarget; 345 targetLimit = pTarget+size; 346 347 /* read the rules into the buffer */ 348 while (target < targetLimit) 349 { 350 c = ucbuf_getc(ucbuf, status); 351 if(c == QUOTE) { 352 quoted = (UBool)!quoted; 353 } 354 /* weiv (06/26/2002): adding the following: 355 * - preserving spaces in commands [...] 356 * - # comments until the end of line 357 */ 358 if (c == STARTCOMMAND && !quoted) 359 { 360 /* preserve commands 361 * closing bracket will be handled by the 362 * append at the end of the loop 363 */ 364 while(c != ENDCOMMAND) { 365 U_APPEND_CHAR32_ONLY(c, target); 366 c = ucbuf_getc(ucbuf, status); 367 } 368 } 369 else if (c == HASH && !quoted) { 370 /* skip comments */ 371 while(c != CR && c != LF) { 372 c = ucbuf_getc(ucbuf, status); 373 } 374 continue; 375 } 376 else if (c == ESCAPE) 377 { 378 c = unescape(ucbuf, status); 379 380 if (c == (UChar32)U_ERR) 381 { 382 uprv_free(pTarget); 383 T_FileStream_close(file); 384 return NULL; 385 } 386 } 387 else if (!quoted && (c == SPACE || c == TAB || c == CR || c == LF)) 388 { 389 /* ignore spaces carriage returns 390 * and line feed unless in the form \uXXXX 391 */ 392 continue; 393 } 394 395 /* Append UChar * after dissembling if c > 0xffff*/ 396 if (c != (UChar32)U_EOF) 397 { 398 U_APPEND_CHAR32_ONLY(c, target); 399 } 400 else 401 { 402 break; 403 } 404 } 405 406 /* terminate the string */ 407 if(target < targetLimit){ 408 *target = 0x0000; 409 } 410 411 result = string_open(state->bundle, tag, pTarget, (int32_t)(target - pTarget), NULL, status); 412 413 414 ucbuf_close(ucbuf); 415 uprv_free(pTarget); 416 T_FileStream_close(file); 417 418 return result; 419 } 420 421 static struct SResource * 422 parseTransliterator(ParseState* state, char *tag, uint32_t startline, const struct UString* /*comment*/, UErrorCode *status) 423 { 424 struct SResource *result = NULL; 425 struct UString *tokenValue; 426 FileStream *file = NULL; 427 char filename[256] = { '\0' }; 428 char cs[128] = { '\0' }; 429 uint32_t line; 430 UCHARBUF *ucbuf=NULL; 431 const char* cp = NULL; 432 UChar *pTarget = NULL; 433 const UChar *pSource = NULL; 434 int32_t size = 0; 435 436 expect(state, TOK_STRING, &tokenValue, NULL, &line, status); 437 438 if(isVerbose()){ 439 printf(" %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 440 } 441 442 if (U_FAILURE(*status)) 443 { 444 return NULL; 445 } 446 /* make the filename including the directory */ 447 if (state->inputdir != NULL) 448 { 449 uprv_strcat(filename, state->inputdir); 450 451 if (state->inputdir[state->inputdirLength - 1] != U_FILE_SEP_CHAR) 452 { 453 uprv_strcat(filename, U_FILE_SEP_STRING); 454 } 455 } 456 457 u_UCharsToChars(tokenValue->fChars, cs, tokenValue->fLength); 458 459 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 460 461 if (U_FAILURE(*status)) 462 { 463 return NULL; 464 } 465 uprv_strcat(filename, cs); 466 467 468 ucbuf = ucbuf_open(filename, &cp, getShowWarning(),FALSE, status); 469 470 if (U_FAILURE(*status)) { 471 error(line, "An error occured while opening the input file %s\n", filename); 472 return NULL; 473 } 474 475 /* We allocate more space than actually required 476 * since the actual size needed for storing UChars 477 * is not known in UTF-8 byte stream 478 */ 479 pSource = ucbuf_getBuffer(ucbuf, &size, status); 480 pTarget = (UChar*) uprv_malloc(U_SIZEOF_UCHAR * (size + 1)); 481 uprv_memset(pTarget, 0, size*U_SIZEOF_UCHAR); 482 483 #if !UCONFIG_NO_TRANSLITERATION 484 size = utrans_stripRules(pSource, size, pTarget, status); 485 #else 486 size = 0; 487 fprintf(stderr, " Warning: writing empty transliteration data ( UCONFIG_NO_TRANSLITERATION ) \n"); 488 #endif 489 result = string_open(state->bundle, tag, pTarget, size, NULL, status); 490 491 ucbuf_close(ucbuf); 492 uprv_free(pTarget); 493 T_FileStream_close(file); 494 495 return result; 496 } 497 static struct SResource* dependencyArray = NULL; 498 499 static struct SResource * 500 parseDependency(ParseState* state, char *tag, uint32_t startline, const struct UString* comment, UErrorCode *status) 501 { 502 struct SResource *result = NULL; 503 struct SResource *elem = NULL; 504 struct UString *tokenValue; 505 uint32_t line; 506 char filename[256] = { '\0' }; 507 char cs[128] = { '\0' }; 508 509 expect(state, TOK_STRING, &tokenValue, NULL, &line, status); 510 511 if(isVerbose()){ 512 printf(" %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 513 } 514 515 if (U_FAILURE(*status)) 516 { 517 return NULL; 518 } 519 /* make the filename including the directory */ 520 if (state->outputdir != NULL) 521 { 522 uprv_strcat(filename, state->outputdir); 523 524 if (state->outputdir[state->outputdirLength - 1] != U_FILE_SEP_CHAR) 525 { 526 uprv_strcat(filename, U_FILE_SEP_STRING); 527 } 528 } 529 530 u_UCharsToChars(tokenValue->fChars, cs, tokenValue->fLength); 531 532 if (U_FAILURE(*status)) 533 { 534 return NULL; 535 } 536 uprv_strcat(filename, cs); 537 if(!T_FileStream_file_exists(filename)){ 538 if(isStrict()){ 539 error(line, "The dependency file %s does not exist. Please make sure it exists.\n",filename); 540 }else{ 541 warning(line, "The dependency file %s does not exist. Please make sure it exists.\n",filename); 542 } 543 } 544 if(dependencyArray==NULL){ 545 dependencyArray = array_open(state->bundle, "%%DEPENDENCY", NULL, status); 546 } 547 if(tag!=NULL){ 548 result = string_open(state->bundle, tag, tokenValue->fChars, tokenValue->fLength, comment, status); 549 } 550 elem = string_open(state->bundle, NULL, tokenValue->fChars, tokenValue->fLength, comment, status); 551 552 array_add(dependencyArray, elem, status); 553 554 if (U_FAILURE(*status)) 555 { 556 return NULL; 557 } 558 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 559 return result; 560 } 561 static struct SResource * 562 parseString(ParseState* state, char *tag, uint32_t startline, const struct UString* comment, UErrorCode *status) 563 { 564 struct UString *tokenValue; 565 struct SResource *result = NULL; 566 567 /* if (tag != NULL && uprv_strcmp(tag, "%%UCARULES") == 0) 568 { 569 return parseUCARules(tag, startline, status); 570 }*/ 571 if(isVerbose()){ 572 printf(" string %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 573 } 574 expect(state, TOK_STRING, &tokenValue, NULL, NULL, status); 575 576 if (U_SUCCESS(*status)) 577 { 578 /* create the string now - tokenValue doesn't survive a call to getToken (and therefore 579 doesn't survive expect either) */ 580 581 result = string_open(state->bundle, tag, tokenValue->fChars, tokenValue->fLength, comment, status); 582 if(U_SUCCESS(*status) && result) { 583 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 584 585 if (U_FAILURE(*status)) 586 { 587 res_close(result); 588 return NULL; 589 } 590 } 591 } 592 593 return result; 594 } 595 596 static struct SResource * 597 parseAlias(ParseState* state, char *tag, uint32_t startline, const struct UString *comment, UErrorCode *status) 598 { 599 struct UString *tokenValue; 600 struct SResource *result = NULL; 601 602 expect(state, TOK_STRING, &tokenValue, NULL, NULL, status); 603 604 if(isVerbose()){ 605 printf(" alias %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 606 } 607 608 if (U_SUCCESS(*status)) 609 { 610 /* create the string now - tokenValue doesn't survive a call to getToken (and therefore 611 doesn't survive expect either) */ 612 613 result = alias_open(state->bundle, tag, tokenValue->fChars, tokenValue->fLength, comment, status); 614 615 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 616 617 if (U_FAILURE(*status)) 618 { 619 res_close(result); 620 return NULL; 621 } 622 } 623 624 return result; 625 } 626 627 typedef struct{ 628 const char* inputDir; 629 const char* outputDir; 630 } GenrbData; 631 632 static struct SResource* resLookup(struct SResource* res, const char* key){ 633 struct SResource *current = NULL; 634 struct SResTable *list; 635 if (res == res_none()) { 636 return NULL; 637 } 638 639 list = &(res->u.fTable); 640 641 current = list->fFirst; 642 while (current != NULL) { 643 if (uprv_strcmp(((list->fRoot->fKeys) + (current->fKey)), key) == 0) { 644 return current; 645 } 646 current = current->fNext; 647 } 648 return NULL; 649 } 650 651 static const UChar* importFromDataFile(void* context, const char* locale, const char* type, int32_t* pLength, UErrorCode* status){ 652 struct SRBRoot *data = NULL; 653 UCHARBUF *ucbuf = NULL; 654 GenrbData* genrbdata = (GenrbData*) context; 655 int localeLength = strlen(locale); 656 char* filename = (char*)uprv_malloc(localeLength+5); 657 char *inputDirBuf = NULL; 658 char *openFileName = NULL; 659 const char* cp = ""; 660 UChar* urules = NULL; 661 int32_t urulesLength = 0; 662 int32_t i = 0; 663 int32_t dirlen = 0; 664 int32_t filelen = 0; 665 struct SResource* root; 666 struct SResource* collations; 667 struct SResource* collation; 668 struct SResource* sequence; 669 670 memcpy(filename, locale, localeLength); 671 for(i = 0; i < localeLength; i++){ 672 if(filename[i] == '-'){ 673 filename[i] = '_'; 674 } 675 } 676 filename[localeLength] = '.'; 677 filename[localeLength+1] = 't'; 678 filename[localeLength+2] = 'x'; 679 filename[localeLength+3] = 't'; 680 filename[localeLength+4] = 0; 681 682 683 if (status==NULL || U_FAILURE(*status)) { 684 return NULL; 685 } 686 if(filename==NULL){ 687 *status=U_ILLEGAL_ARGUMENT_ERROR; 688 return NULL; 689 }else{ 690 filelen = (int32_t)uprv_strlen(filename); 691 } 692 if(genrbdata->inputDir == NULL) { 693 const char *filenameBegin = uprv_strrchr(filename, U_FILE_SEP_CHAR); 694 openFileName = (char *) uprv_malloc(dirlen + filelen + 2); 695 openFileName[0] = '\0'; 696 if (filenameBegin != NULL) { 697 /* 698 * When a filename ../../../data/root.txt is specified, 699 * we presume that the input directory is ../../../data 700 * This is very important when the resource file includes 701 * another file, like UCARules.txt or thaidict.brk. 702 */ 703 int32_t filenameSize = (int32_t)(filenameBegin - filename + 1); 704 inputDirBuf = uprv_strncpy((char *)uprv_malloc(filenameSize), filename, filenameSize); 705 706 /* test for NULL */ 707 if(inputDirBuf == NULL) { 708 *status = U_MEMORY_ALLOCATION_ERROR; 709 goto finish; 710 } 711 712 inputDirBuf[filenameSize - 1] = 0; 713 genrbdata->inputDir = inputDirBuf; 714 dirlen = (int32_t)uprv_strlen(genrbdata->inputDir); 715 } 716 }else{ 717 dirlen = (int32_t)uprv_strlen(genrbdata->inputDir); 718 719 if(genrbdata->inputDir[dirlen-1] != U_FILE_SEP_CHAR) { 720 openFileName = (char *) uprv_malloc(dirlen + filelen + 2); 721 722 /* test for NULL */ 723 if(openFileName == NULL) { 724 *status = U_MEMORY_ALLOCATION_ERROR; 725 goto finish; 726 } 727 728 openFileName[0] = '\0'; 729 /* 730 * append the input dir to openFileName if the first char in 731 * filename is not file seperation char and the last char input directory is not '.'. 732 * This is to support : 733 * genrb -s. /home/icu/data 734 * genrb -s. icu/data 735 * The user cannot mix notations like 736 * genrb -s. /icu/data --- the absolute path specified. -s redundant 737 * user should use 738 * genrb -s. icu/data --- start from CWD and look in icu/data dir 739 */ 740 if( (filename[0] != U_FILE_SEP_CHAR) && (genrbdata->inputDir[dirlen-1] !='.')){ 741 uprv_strcpy(openFileName, genrbdata->inputDir); 742 openFileName[dirlen] = U_FILE_SEP_CHAR; 743 } 744 openFileName[dirlen + 1] = '\0'; 745 } else { 746 openFileName = (char *) uprv_malloc(dirlen + filelen + 1); 747 748 /* test for NULL */ 749 if(openFileName == NULL) { 750 *status = U_MEMORY_ALLOCATION_ERROR; 751 goto finish; 752 } 753 754 uprv_strcpy(openFileName, genrbdata->inputDir); 755 756 } 757 } 758 uprv_strcat(openFileName, filename); 759 /* printf("%s\n", openFileName); */ 760 *status = U_ZERO_ERROR; 761 ucbuf = ucbuf_open(openFileName, &cp,getShowWarning(),TRUE, status); 762 763 if(*status == U_FILE_ACCESS_ERROR) { 764 765 fprintf(stderr, "couldn't open file %s\n", openFileName == NULL ? filename : openFileName); 766 goto finish; 767 } 768 if (ucbuf == NULL || U_FAILURE(*status)) { 769 fprintf(stderr, "An error occured processing file %s. Error: %s\n", openFileName == NULL ? filename : openFileName,u_errorName(*status)); 770 goto finish; 771 } 772 773 /* Parse the data into an SRBRoot */ 774 data = parse(ucbuf, genrbdata->inputDir, genrbdata->outputDir, FALSE, status); 775 776 root = data->fRoot; 777 collations = resLookup(root, "collations"); 778 if (collations != NULL) { 779 collation = resLookup(collations, type); 780 if (collation != NULL) { 781 sequence = resLookup(collation, "Sequence"); 782 if (sequence != NULL) { 783 urules = sequence->u.fString.fChars; 784 urulesLength = sequence->u.fString.fLength; 785 *pLength = urulesLength; 786 } 787 } 788 } 789 790 finish: 791 if (inputDirBuf != NULL) { 792 uprv_free(inputDirBuf); 793 } 794 795 if (openFileName != NULL) { 796 uprv_free(openFileName); 797 } 798 799 if(ucbuf) { 800 ucbuf_close(ucbuf); 801 } 802 803 return urules; 804 } 805 806 // Quick-and-dirty escaping function. 807 // Assumes that we are on an ASCII-based platform. 808 static void 809 escape(const UChar *s, char *buffer) { 810 int32_t length = u_strlen(s); 811 int32_t i = 0; 812 for (;;) { 813 UChar32 c; 814 U16_NEXT(s, i, length, c); 815 if (c == 0) { 816 *buffer = 0; 817 return; 818 } else if (0x20 <= c && c <= 0x7e) { 819 // printable ASCII 820 *buffer++ = (char)c; // assumes ASCII-based platform 821 } else { 822 buffer += sprintf(buffer, "\\u%04X", (int)c); 823 } 824 } 825 } 826 827 static struct SResource * 828 addCollation(ParseState* state, struct SResource *result, uint32_t startline, UErrorCode *status) 829 { 830 struct SResource *member = NULL; 831 struct UString *tokenValue; 832 struct UString comment; 833 enum ETokenType token; 834 char subtag[1024]; 835 UVersionInfo version; 836 uint32_t line; 837 GenrbData genrbdata; 838 /* '{' . (name resource)* '}' */ 839 version[0]=0; version[1]=0; version[2]=0; version[3]=0; 840 841 for (;;) 842 { 843 ustr_init(&comment); 844 token = getToken(state, &tokenValue, &comment, &line, status); 845 846 if (token == TOK_CLOSE_BRACE) 847 { 848 return result; 849 } 850 851 if (token != TOK_STRING) 852 { 853 res_close(result); 854 *status = U_INVALID_FORMAT_ERROR; 855 856 if (token == TOK_EOF) 857 { 858 error(startline, "unterminated table"); 859 } 860 else 861 { 862 error(line, "Unexpected token %s", tokenNames[token]); 863 } 864 865 return NULL; 866 } 867 868 u_UCharsToChars(tokenValue->fChars, subtag, u_strlen(tokenValue->fChars) + 1); 869 870 if (U_FAILURE(*status)) 871 { 872 res_close(result); 873 return NULL; 874 } 875 876 member = parseResource(state, subtag, NULL, status); 877 878 if (U_FAILURE(*status)) 879 { 880 res_close(result); 881 return NULL; 882 } 883 884 if (uprv_strcmp(subtag, "Version") == 0) 885 { 886 char ver[40]; 887 int32_t length = member->u.fString.fLength; 888 889 if (length >= (int32_t) sizeof(ver)) 890 { 891 length = (int32_t) sizeof(ver) - 1; 892 } 893 894 u_UCharsToChars(member->u.fString.fChars, ver, length + 1); /* +1 for copying NULL */ 895 u_versionFromString(version, ver); 896 897 table_add(result, member, line, status); 898 899 } 900 else if (uprv_strcmp(subtag, "Override") == 0) 901 { 902 // UBool override = (u_strncmp(member->u.fString.fChars, trueValue, u_strlen(trueValue)) == 0); 903 table_add(result, member, line, status); 904 905 } 906 else if(uprv_strcmp(subtag, "%%CollationBin")==0) 907 { 908 /* discard duplicate %%CollationBin if any*/ 909 } 910 else if (uprv_strcmp(subtag, "Sequence") == 0) 911 { 912 #if UCONFIG_NO_COLLATION || UCONFIG_NO_FILE_IO 913 warning(line, "Not building collation elements because of UCONFIG_NO_COLLATION and/or UCONFIG_NO_FILE_IO, see uconfig.h"); 914 #else 915 if(state->makeBinaryCollation) { 916 917 /* do the collation elements */ 918 int32_t len = 0; 919 uint8_t *data = NULL; 920 UCollator *coll = NULL; 921 int32_t reorderCodes[USCRIPT_CODE_LIMIT + (UCOL_REORDER_CODE_LIMIT - UCOL_REORDER_CODE_FIRST)]; 922 int32_t reorderCodeCount; 923 int32_t reorderCodeIndex; 924 UParseError parseError; 925 926 genrbdata.inputDir = state->inputdir; 927 genrbdata.outputDir = state->outputdir; 928 929 UErrorCode intStatus = U_ZERO_ERROR; 930 uprv_memset(&parseError, 0, sizeof(parseError)); 931 coll = ucol_openRulesForImport(member->u.fString.fChars, member->u.fString.fLength, 932 UCOL_OFF, UCOL_DEFAULT_STRENGTH,&parseError, importFromDataFile, &genrbdata, &intStatus); 933 934 if (U_SUCCESS(intStatus) && coll != NULL) 935 { 936 len = ucol_cloneBinary(coll, NULL, 0, &intStatus); 937 data = (uint8_t *)uprv_malloc(len); 938 intStatus = U_ZERO_ERROR; 939 len = ucol_cloneBinary(coll, data, len, &intStatus); 940 /*data = ucol_cloneRuleData(coll, &len, &intStatus);*/ 941 942 /* tailoring rules version */ 943 /* This is wrong! */ 944 /*coll->dataInfo.dataVersion[1] = version[0];*/ 945 /* Copy tailoring version. Builder version already */ 946 /* set in ucol_openRules */ 947 ((UCATableHeader *)data)->version[1] = version[0]; 948 ((UCATableHeader *)data)->version[2] = version[1]; 949 ((UCATableHeader *)data)->version[3] = version[2]; 950 951 if (U_SUCCESS(intStatus) && data != NULL) 952 { 953 struct SResource *collationBin = bin_open(state->bundle, "%%CollationBin", len, data, NULL, NULL, status); 954 table_add(result, collationBin, line, status); 955 uprv_free(data); 956 957 reorderCodeCount = ucol_getReorderCodes( 958 coll, reorderCodes, USCRIPT_CODE_LIMIT + (UCOL_REORDER_CODE_LIMIT - UCOL_REORDER_CODE_FIRST), &intStatus); 959 if (U_SUCCESS(intStatus) && reorderCodeCount > 0) { 960 struct SResource *reorderCodeRes = intvector_open(state->bundle, "%%ReorderCodes", NULL, status); 961 for (reorderCodeIndex = 0; reorderCodeIndex < reorderCodeCount; reorderCodeIndex++) { 962 intvector_add(reorderCodeRes, reorderCodes[reorderCodeIndex], status); 963 } 964 table_add(result, reorderCodeRes, line, status); 965 } 966 } 967 else 968 { 969 warning(line, "could not obtain rules from collator"); 970 if(isStrict()){ 971 *status = U_INVALID_FORMAT_ERROR; 972 return NULL; 973 } 974 } 975 976 ucol_close(coll); 977 } 978 else 979 { 980 if(intStatus == U_FILE_ACCESS_ERROR) { 981 error(startline, "Collation could not be built- U_FILE_ACCESS_ERROR. Make sure ICU's data has been built and is loading properly."); 982 *status = intStatus; 983 return NULL; 984 } 985 char preBuffer[100], postBuffer[100]; 986 escape(parseError.preContext, preBuffer); 987 escape(parseError.postContext, postBuffer); 988 warning(line, 989 "%%%%CollationBin could not be constructed from CollationElements\n" 990 " check context, check that the FractionalUCA.txt UCA version " 991 "matches the current UCD version\n" 992 " UErrorCode=%s UParseError={ line=%d offset=%d pre=<> post=<> }", 993 u_errorName(intStatus), 994 parseError.line, 995 parseError.offset, 996 preBuffer, 997 postBuffer); 998 if(isStrict()){ 999 *status = intStatus; 1000 return NULL; 1001 } 1002 } 1003 } else { 1004 if(isVerbose()) { 1005 printf("Not building Collation binary\n"); 1006 } 1007 } 1008 #endif 1009 /* in order to achieve smaller data files, we can direct genrb */ 1010 /* to omit collation rules */ 1011 if(gOmitCollationRules) { 1012 bundle_closeString(state->bundle, member); 1013 } else { 1014 table_add(result, member, line, status); 1015 } 1016 } 1017 if (U_FAILURE(*status)) 1018 { 1019 res_close(result); 1020 return NULL; 1021 } 1022 } 1023 1024 // Reached the end without a TOK_CLOSE_BRACE. Should be an error. 1025 *status = U_INTERNAL_PROGRAM_ERROR; 1026 return NULL; 1027 } 1028 1029 static struct SResource * 1030 parseCollationElements(ParseState* state, char *tag, uint32_t startline, UBool newCollation, UErrorCode *status) 1031 { 1032 struct SResource *result = NULL; 1033 struct SResource *member = NULL; 1034 struct SResource *collationRes = NULL; 1035 struct UString *tokenValue; 1036 struct UString comment; 1037 enum ETokenType token; 1038 char subtag[1024], typeKeyword[1024]; 1039 uint32_t line; 1040 1041 result = table_open(state->bundle, tag, NULL, status); 1042 1043 if (result == NULL || U_FAILURE(*status)) 1044 { 1045 return NULL; 1046 } 1047 if(isVerbose()){ 1048 printf(" collation elements %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1049 } 1050 if(!newCollation) { 1051 return addCollation(state, result, startline, status); 1052 } 1053 else { 1054 for(;;) { 1055 ustr_init(&comment); 1056 token = getToken(state, &tokenValue, &comment, &line, status); 1057 1058 if (token == TOK_CLOSE_BRACE) 1059 { 1060 return result; 1061 } 1062 1063 if (token != TOK_STRING) 1064 { 1065 res_close(result); 1066 *status = U_INVALID_FORMAT_ERROR; 1067 1068 if (token == TOK_EOF) 1069 { 1070 error(startline, "unterminated table"); 1071 } 1072 else 1073 { 1074 error(line, "Unexpected token %s", tokenNames[token]); 1075 } 1076 1077 return NULL; 1078 } 1079 1080 u_UCharsToChars(tokenValue->fChars, subtag, u_strlen(tokenValue->fChars) + 1); 1081 1082 if (U_FAILURE(*status)) 1083 { 1084 res_close(result); 1085 return NULL; 1086 } 1087 1088 if (uprv_strcmp(subtag, "default") == 0) 1089 { 1090 member = parseResource(state, subtag, NULL, status); 1091 1092 if (U_FAILURE(*status)) 1093 { 1094 res_close(result); 1095 return NULL; 1096 } 1097 1098 table_add(result, member, line, status); 1099 } 1100 else 1101 { 1102 token = peekToken(state, 0, &tokenValue, &line, &comment, status); 1103 /* this probably needs to be refactored or recursively use the parser */ 1104 /* first we assume that our collation table won't have the explicit type */ 1105 /* then, we cannot handle aliases */ 1106 if(token == TOK_OPEN_BRACE) { 1107 token = getToken(state, &tokenValue, &comment, &line, status); 1108 collationRes = table_open(state->bundle, subtag, NULL, status); 1109 collationRes = addCollation(state, collationRes, startline, status); /* need to parse the collation data regardless */ 1110 if (gIncludeUnihanColl || uprv_strcmp(subtag, "unihan") != 0) { 1111 table_add(result, collationRes, startline, status); 1112 } 1113 } else if(token == TOK_COLON) { /* right now, we'll just try to see if we have aliases */ 1114 /* we could have a table too */ 1115 token = peekToken(state, 1, &tokenValue, &line, &comment, status); 1116 u_UCharsToChars(tokenValue->fChars, typeKeyword, u_strlen(tokenValue->fChars) + 1); 1117 if(uprv_strcmp(typeKeyword, "alias") == 0) { 1118 member = parseResource(state, subtag, NULL, status); 1119 if (U_FAILURE(*status)) 1120 { 1121 res_close(result); 1122 return NULL; 1123 } 1124 1125 table_add(result, member, line, status); 1126 } else { 1127 res_close(result); 1128 *status = U_INVALID_FORMAT_ERROR; 1129 return NULL; 1130 } 1131 } else { 1132 res_close(result); 1133 *status = U_INVALID_FORMAT_ERROR; 1134 return NULL; 1135 } 1136 } 1137 1138 /*member = string_open(bundle, subtag, tokenValue->fChars, tokenValue->fLength, status);*/ 1139 1140 /*expect(TOK_CLOSE_BRACE, NULL, NULL, status);*/ 1141 1142 if (U_FAILURE(*status)) 1143 { 1144 res_close(result); 1145 return NULL; 1146 } 1147 } 1148 } 1149 } 1150 1151 /* Necessary, because CollationElements requires the bundle->fRoot member to be present which, 1152 if this weren't special-cased, wouldn't be set until the entire file had been processed. */ 1153 static struct SResource * 1154 realParseTable(ParseState* state, struct SResource *table, char *tag, uint32_t startline, UErrorCode *status) 1155 { 1156 struct SResource *member = NULL; 1157 struct UString *tokenValue=NULL; 1158 struct UString comment; 1159 enum ETokenType token; 1160 char subtag[1024]; 1161 uint32_t line; 1162 UBool readToken = FALSE; 1163 1164 /* '{' . (name resource)* '}' */ 1165 1166 if(isVerbose()){ 1167 printf(" parsing table %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1168 } 1169 for (;;) 1170 { 1171 ustr_init(&comment); 1172 token = getToken(state, &tokenValue, &comment, &line, status); 1173 1174 if (token == TOK_CLOSE_BRACE) 1175 { 1176 if (!readToken) { 1177 warning(startline, "Encountered empty table"); 1178 } 1179 return table; 1180 } 1181 1182 if (token != TOK_STRING) 1183 { 1184 *status = U_INVALID_FORMAT_ERROR; 1185 1186 if (token == TOK_EOF) 1187 { 1188 error(startline, "unterminated table"); 1189 } 1190 else 1191 { 1192 error(line, "unexpected token %s", tokenNames[token]); 1193 } 1194 1195 return NULL; 1196 } 1197 1198 if(uprv_isInvariantUString(tokenValue->fChars, -1)) { 1199 u_UCharsToChars(tokenValue->fChars, subtag, u_strlen(tokenValue->fChars) + 1); 1200 } else { 1201 *status = U_INVALID_FORMAT_ERROR; 1202 error(line, "invariant characters required for table keys"); 1203 return NULL; 1204 } 1205 1206 if (U_FAILURE(*status)) 1207 { 1208 error(line, "parse error. Stopped parsing tokens with %s", u_errorName(*status)); 1209 return NULL; 1210 } 1211 1212 member = parseResource(state, subtag, &comment, status); 1213 1214 if (member == NULL || U_FAILURE(*status)) 1215 { 1216 error(line, "parse error. Stopped parsing resource with %s", u_errorName(*status)); 1217 return NULL; 1218 } 1219 1220 table_add(table, member, line, status); 1221 1222 if (U_FAILURE(*status)) 1223 { 1224 error(line, "parse error. Stopped parsing table with %s", u_errorName(*status)); 1225 return NULL; 1226 } 1227 readToken = TRUE; 1228 ustr_deinit(&comment); 1229 } 1230 1231 /* not reached */ 1232 /* A compiler warning will appear if all paths don't contain a return statement. */ 1233 /* *status = U_INTERNAL_PROGRAM_ERROR; 1234 return NULL;*/ 1235 } 1236 1237 static struct SResource * 1238 parseTable(ParseState* state, char *tag, uint32_t startline, const struct UString *comment, UErrorCode *status) 1239 { 1240 struct SResource *result; 1241 1242 if (tag != NULL && uprv_strcmp(tag, "CollationElements") == 0) 1243 { 1244 return parseCollationElements(state, tag, startline, FALSE, status); 1245 } 1246 if (tag != NULL && uprv_strcmp(tag, "collations") == 0) 1247 { 1248 return parseCollationElements(state, tag, startline, TRUE, status); 1249 } 1250 if(isVerbose()){ 1251 printf(" table %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1252 } 1253 1254 result = table_open(state->bundle, tag, comment, status); 1255 1256 if (result == NULL || U_FAILURE(*status)) 1257 { 1258 return NULL; 1259 } 1260 return realParseTable(state, result, tag, startline, status); 1261 } 1262 1263 static struct SResource * 1264 parseArray(ParseState* state, char *tag, uint32_t startline, const struct UString *comment, UErrorCode *status) 1265 { 1266 struct SResource *result = NULL; 1267 struct SResource *member = NULL; 1268 struct UString *tokenValue; 1269 struct UString memberComments; 1270 enum ETokenType token; 1271 UBool readToken = FALSE; 1272 1273 result = array_open(state->bundle, tag, comment, status); 1274 1275 if (result == NULL || U_FAILURE(*status)) 1276 { 1277 return NULL; 1278 } 1279 if(isVerbose()){ 1280 printf(" array %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1281 } 1282 1283 ustr_init(&memberComments); 1284 1285 /* '{' . resource [','] '}' */ 1286 for (;;) 1287 { 1288 /* reset length */ 1289 ustr_setlen(&memberComments, 0, status); 1290 1291 /* check for end of array, but don't consume next token unless it really is the end */ 1292 token = peekToken(state, 0, &tokenValue, NULL, &memberComments, status); 1293 1294 1295 if (token == TOK_CLOSE_BRACE) 1296 { 1297 getToken(state, NULL, NULL, NULL, status); 1298 if (!readToken) { 1299 warning(startline, "Encountered empty array"); 1300 } 1301 break; 1302 } 1303 1304 if (token == TOK_EOF) 1305 { 1306 res_close(result); 1307 *status = U_INVALID_FORMAT_ERROR; 1308 error(startline, "unterminated array"); 1309 return NULL; 1310 } 1311 1312 /* string arrays are a special case */ 1313 if (token == TOK_STRING) 1314 { 1315 getToken(state, &tokenValue, &memberComments, NULL, status); 1316 member = string_open(state->bundle, NULL, tokenValue->fChars, tokenValue->fLength, &memberComments, status); 1317 } 1318 else 1319 { 1320 member = parseResource(state, NULL, &memberComments, status); 1321 } 1322 1323 if (member == NULL || U_FAILURE(*status)) 1324 { 1325 res_close(result); 1326 return NULL; 1327 } 1328 1329 array_add(result, member, status); 1330 1331 if (U_FAILURE(*status)) 1332 { 1333 res_close(result); 1334 return NULL; 1335 } 1336 1337 /* eat optional comma if present */ 1338 token = peekToken(state, 0, NULL, NULL, NULL, status); 1339 1340 if (token == TOK_COMMA) 1341 { 1342 getToken(state, NULL, NULL, NULL, status); 1343 } 1344 1345 if (U_FAILURE(*status)) 1346 { 1347 res_close(result); 1348 return NULL; 1349 } 1350 readToken = TRUE; 1351 } 1352 1353 ustr_deinit(&memberComments); 1354 return result; 1355 } 1356 1357 static struct SResource * 1358 parseIntVector(ParseState* state, char *tag, uint32_t startline, const struct UString *comment, UErrorCode *status) 1359 { 1360 struct SResource *result = NULL; 1361 enum ETokenType token; 1362 char *string; 1363 int32_t value; 1364 UBool readToken = FALSE; 1365 char *stopstring; 1366 uint32_t len; 1367 struct UString memberComments; 1368 1369 result = intvector_open(state->bundle, tag, comment, status); 1370 1371 if (result == NULL || U_FAILURE(*status)) 1372 { 1373 return NULL; 1374 } 1375 1376 if(isVerbose()){ 1377 printf(" vector %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1378 } 1379 ustr_init(&memberComments); 1380 /* '{' . string [','] '}' */ 1381 for (;;) 1382 { 1383 ustr_setlen(&memberComments, 0, status); 1384 1385 /* check for end of array, but don't consume next token unless it really is the end */ 1386 token = peekToken(state, 0, NULL, NULL,&memberComments, status); 1387 1388 if (token == TOK_CLOSE_BRACE) 1389 { 1390 /* it's the end, consume the close brace */ 1391 getToken(state, NULL, NULL, NULL, status); 1392 if (!readToken) { 1393 warning(startline, "Encountered empty int vector"); 1394 } 1395 ustr_deinit(&memberComments); 1396 return result; 1397 } 1398 1399 string = getInvariantString(state, NULL, NULL, status); 1400 1401 if (U_FAILURE(*status)) 1402 { 1403 res_close(result); 1404 return NULL; 1405 } 1406 1407 /* For handling illegal char in the Intvector */ 1408 value = uprv_strtoul(string, &stopstring, 0);/* make intvector support decimal,hexdigit,octal digit ranging from -2^31-2^32-1*/ 1409 len=(uint32_t)(stopstring-string); 1410 1411 if(len==uprv_strlen(string)) 1412 { 1413 intvector_add(result, value, status); 1414 uprv_free(string); 1415 token = peekToken(state, 0, NULL, NULL, NULL, status); 1416 } 1417 else 1418 { 1419 uprv_free(string); 1420 *status=U_INVALID_CHAR_FOUND; 1421 } 1422 1423 if (U_FAILURE(*status)) 1424 { 1425 res_close(result); 1426 return NULL; 1427 } 1428 1429 /* the comma is optional (even though it is required to prevent the reader from concatenating 1430 consecutive entries) so that a missing comma on the last entry isn't an error */ 1431 if (token == TOK_COMMA) 1432 { 1433 getToken(state, NULL, NULL, NULL, status); 1434 } 1435 readToken = TRUE; 1436 } 1437 1438 /* not reached */ 1439 /* A compiler warning will appear if all paths don't contain a return statement. */ 1440 /* intvector_close(result, status); 1441 *status = U_INTERNAL_PROGRAM_ERROR; 1442 return NULL;*/ 1443 } 1444 1445 static struct SResource * 1446 parseBinary(ParseState* state, char *tag, uint32_t startline, const struct UString *comment, UErrorCode *status) 1447 { 1448 struct SResource *result = NULL; 1449 uint8_t *value; 1450 char *string; 1451 char toConv[3] = {'\0', '\0', '\0'}; 1452 uint32_t count; 1453 uint32_t i; 1454 uint32_t line; 1455 char *stopstring; 1456 uint32_t len; 1457 1458 string = getInvariantString(state, &line, NULL, status); 1459 1460 if (string == NULL || U_FAILURE(*status)) 1461 { 1462 return NULL; 1463 } 1464 1465 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 1466 1467 if (U_FAILURE(*status)) 1468 { 1469 uprv_free(string); 1470 return NULL; 1471 } 1472 1473 if(isVerbose()){ 1474 printf(" binary %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1475 } 1476 1477 count = (uint32_t)uprv_strlen(string); 1478 if (count > 0){ 1479 if((count % 2)==0){ 1480 value = static_cast<uint8_t *>(uprv_malloc(sizeof(uint8_t) * count)); 1481 1482 if (value == NULL) 1483 { 1484 uprv_free(string); 1485 *status = U_MEMORY_ALLOCATION_ERROR; 1486 return NULL; 1487 } 1488 1489 for (i = 0; i < count; i += 2) 1490 { 1491 toConv[0] = string[i]; 1492 toConv[1] = string[i + 1]; 1493 1494 value[i >> 1] = (uint8_t) uprv_strtoul(toConv, &stopstring, 16); 1495 len=(uint32_t)(stopstring-toConv); 1496 1497 if(len!=uprv_strlen(toConv)) 1498 { 1499 uprv_free(string); 1500 *status=U_INVALID_CHAR_FOUND; 1501 return NULL; 1502 } 1503 } 1504 1505 result = bin_open(state->bundle, tag, (i >> 1), value,NULL, comment, status); 1506 1507 uprv_free(value); 1508 } 1509 else 1510 { 1511 *status = U_INVALID_CHAR_FOUND; 1512 uprv_free(string); 1513 error(line, "Encountered invalid binary string"); 1514 return NULL; 1515 } 1516 } 1517 else 1518 { 1519 result = bin_open(state->bundle, tag, 0, NULL, "",comment,status); 1520 warning(startline, "Encountered empty binary tag"); 1521 } 1522 uprv_free(string); 1523 1524 return result; 1525 } 1526 1527 static struct SResource * 1528 parseInteger(ParseState* state, char *tag, uint32_t startline, const struct UString *comment, UErrorCode *status) 1529 { 1530 struct SResource *result = NULL; 1531 int32_t value; 1532 char *string; 1533 char *stopstring; 1534 uint32_t len; 1535 1536 string = getInvariantString(state, NULL, NULL, status); 1537 1538 if (string == NULL || U_FAILURE(*status)) 1539 { 1540 return NULL; 1541 } 1542 1543 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 1544 1545 if (U_FAILURE(*status)) 1546 { 1547 uprv_free(string); 1548 return NULL; 1549 } 1550 1551 if(isVerbose()){ 1552 printf(" integer %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1553 } 1554 1555 if (uprv_strlen(string) <= 0) 1556 { 1557 warning(startline, "Encountered empty integer. Default value is 0."); 1558 } 1559 1560 /* Allow integer support for hexdecimal, octal digit and decimal*/ 1561 /* and handle illegal char in the integer*/ 1562 value = uprv_strtoul(string, &stopstring, 0); 1563 len=(uint32_t)(stopstring-string); 1564 if(len==uprv_strlen(string)) 1565 { 1566 result = int_open(state->bundle, tag, value, comment, status); 1567 } 1568 else 1569 { 1570 *status=U_INVALID_CHAR_FOUND; 1571 } 1572 uprv_free(string); 1573 1574 return result; 1575 } 1576 1577 static struct SResource * 1578 parseImport(ParseState* state, char *tag, uint32_t startline, const struct UString* comment, UErrorCode *status) 1579 { 1580 struct SResource *result; 1581 FileStream *file; 1582 int32_t len; 1583 uint8_t *data; 1584 char *filename; 1585 uint32_t line; 1586 char *fullname = NULL; 1587 filename = getInvariantString(state, &line, NULL, status); 1588 1589 if (U_FAILURE(*status)) 1590 { 1591 return NULL; 1592 } 1593 1594 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 1595 1596 if (U_FAILURE(*status)) 1597 { 1598 uprv_free(filename); 1599 return NULL; 1600 } 1601 1602 if(isVerbose()){ 1603 printf(" import %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1604 } 1605 1606 /* Open the input file for reading */ 1607 if (state->inputdir == NULL) 1608 { 1609 #if 1 1610 /* 1611 * Always save file file name, even if there's 1612 * no input directory specified. MIGHT BREAK SOMETHING 1613 */ 1614 int32_t filenameLength = uprv_strlen(filename); 1615 1616 fullname = (char *) uprv_malloc(filenameLength + 1); 1617 uprv_strcpy(fullname, filename); 1618 #endif 1619 1620 file = T_FileStream_open(filename, "rb"); 1621 } 1622 else 1623 { 1624 1625 int32_t count = (int32_t)uprv_strlen(filename); 1626 1627 if (state->inputdir[state->inputdirLength - 1] != U_FILE_SEP_CHAR) 1628 { 1629 fullname = (char *) uprv_malloc(state->inputdirLength + count + 2); 1630 1631 /* test for NULL */ 1632 if(fullname == NULL) 1633 { 1634 *status = U_MEMORY_ALLOCATION_ERROR; 1635 return NULL; 1636 } 1637 1638 uprv_strcpy(fullname, state->inputdir); 1639 1640 fullname[state->inputdirLength] = U_FILE_SEP_CHAR; 1641 fullname[state->inputdirLength + 1] = '\0'; 1642 1643 uprv_strcat(fullname, filename); 1644 } 1645 else 1646 { 1647 fullname = (char *) uprv_malloc(state->inputdirLength + count + 1); 1648 1649 /* test for NULL */ 1650 if(fullname == NULL) 1651 { 1652 *status = U_MEMORY_ALLOCATION_ERROR; 1653 return NULL; 1654 } 1655 1656 uprv_strcpy(fullname, state->inputdir); 1657 uprv_strcat(fullname, filename); 1658 } 1659 1660 file = T_FileStream_open(fullname, "rb"); 1661 1662 } 1663 1664 if (file == NULL) 1665 { 1666 error(line, "couldn't open input file %s", filename); 1667 *status = U_FILE_ACCESS_ERROR; 1668 return NULL; 1669 } 1670 1671 len = T_FileStream_size(file); 1672 data = (uint8_t*)uprv_malloc(len * sizeof(uint8_t)); 1673 /* test for NULL */ 1674 if(data == NULL) 1675 { 1676 *status = U_MEMORY_ALLOCATION_ERROR; 1677 T_FileStream_close (file); 1678 return NULL; 1679 } 1680 1681 /* int32_t numRead = */ T_FileStream_read (file, data, len); 1682 T_FileStream_close (file); 1683 1684 result = bin_open(state->bundle, tag, len, data, fullname, comment, status); 1685 1686 uprv_free(data); 1687 uprv_free(filename); 1688 uprv_free(fullname); 1689 1690 return result; 1691 } 1692 1693 static struct SResource * 1694 parseInclude(ParseState* state, char *tag, uint32_t startline, const struct UString* comment, UErrorCode *status) 1695 { 1696 struct SResource *result; 1697 int32_t len=0; 1698 char *filename; 1699 uint32_t line; 1700 UChar *pTarget = NULL; 1701 1702 UCHARBUF *ucbuf; 1703 char *fullname = NULL; 1704 int32_t count = 0; 1705 const char* cp = NULL; 1706 const UChar* uBuffer = NULL; 1707 1708 filename = getInvariantString(state, &line, NULL, status); 1709 count = (int32_t)uprv_strlen(filename); 1710 1711 if (U_FAILURE(*status)) 1712 { 1713 return NULL; 1714 } 1715 1716 expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status); 1717 1718 if (U_FAILURE(*status)) 1719 { 1720 uprv_free(filename); 1721 return NULL; 1722 } 1723 1724 if(isVerbose()){ 1725 printf(" include %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1726 } 1727 1728 fullname = (char *) uprv_malloc(state->inputdirLength + count + 2); 1729 /* test for NULL */ 1730 if(fullname == NULL) 1731 { 1732 *status = U_MEMORY_ALLOCATION_ERROR; 1733 uprv_free(filename); 1734 return NULL; 1735 } 1736 1737 if(state->inputdir!=NULL){ 1738 if (state->inputdir[state->inputdirLength - 1] != U_FILE_SEP_CHAR) 1739 { 1740 1741 uprv_strcpy(fullname, state->inputdir); 1742 1743 fullname[state->inputdirLength] = U_FILE_SEP_CHAR; 1744 fullname[state->inputdirLength + 1] = '\0'; 1745 1746 uprv_strcat(fullname, filename); 1747 } 1748 else 1749 { 1750 uprv_strcpy(fullname, state->inputdir); 1751 uprv_strcat(fullname, filename); 1752 } 1753 }else{ 1754 uprv_strcpy(fullname,filename); 1755 } 1756 1757 ucbuf = ucbuf_open(fullname, &cp,getShowWarning(),FALSE,status); 1758 1759 if (U_FAILURE(*status)) { 1760 error(line, "couldn't open input file %s\n", filename); 1761 return NULL; 1762 } 1763 1764 uBuffer = ucbuf_getBuffer(ucbuf,&len,status); 1765 result = string_open(state->bundle, tag, uBuffer, len, comment, status); 1766 1767 ucbuf_close(ucbuf); 1768 1769 uprv_free(pTarget); 1770 1771 uprv_free(filename); 1772 uprv_free(fullname); 1773 1774 return result; 1775 } 1776 1777 1778 1779 1780 1781 U_STRING_DECL(k_type_string, "string", 6); 1782 U_STRING_DECL(k_type_binary, "binary", 6); 1783 U_STRING_DECL(k_type_bin, "bin", 3); 1784 U_STRING_DECL(k_type_table, "table", 5); 1785 U_STRING_DECL(k_type_table_no_fallback, "table(nofallback)", 17); 1786 U_STRING_DECL(k_type_int, "int", 3); 1787 U_STRING_DECL(k_type_integer, "integer", 7); 1788 U_STRING_DECL(k_type_array, "array", 5); 1789 U_STRING_DECL(k_type_alias, "alias", 5); 1790 U_STRING_DECL(k_type_intvector, "intvector", 9); 1791 U_STRING_DECL(k_type_import, "import", 6); 1792 U_STRING_DECL(k_type_include, "include", 7); 1793 1794 /* Various non-standard processing plugins that create one or more special resources. */ 1795 U_STRING_DECL(k_type_plugin_uca_rules, "process(uca_rules)", 18); 1796 U_STRING_DECL(k_type_plugin_collation, "process(collation)", 18); 1797 U_STRING_DECL(k_type_plugin_transliterator, "process(transliterator)", 23); 1798 U_STRING_DECL(k_type_plugin_dependency, "process(dependency)", 19); 1799 1800 typedef enum EResourceType 1801 { 1802 RT_UNKNOWN, 1803 RT_STRING, 1804 RT_BINARY, 1805 RT_TABLE, 1806 RT_TABLE_NO_FALLBACK, 1807 RT_INTEGER, 1808 RT_ARRAY, 1809 RT_ALIAS, 1810 RT_INTVECTOR, 1811 RT_IMPORT, 1812 RT_INCLUDE, 1813 RT_PROCESS_UCA_RULES, 1814 RT_PROCESS_COLLATION, 1815 RT_PROCESS_TRANSLITERATOR, 1816 RT_PROCESS_DEPENDENCY, 1817 RT_RESERVED 1818 } EResourceType; 1819 1820 static struct { 1821 const char *nameChars; /* only used for debugging */ 1822 const UChar *nameUChars; 1823 ParseResourceFunction *parseFunction; 1824 } gResourceTypes[] = { 1825 {"Unknown", NULL, NULL}, 1826 {"string", k_type_string, parseString}, 1827 {"binary", k_type_binary, parseBinary}, 1828 {"table", k_type_table, parseTable}, 1829 {"table(nofallback)", k_type_table_no_fallback, NULL}, /* parseFunction will never be called */ 1830 {"integer", k_type_integer, parseInteger}, 1831 {"array", k_type_array, parseArray}, 1832 {"alias", k_type_alias, parseAlias}, 1833 {"intvector", k_type_intvector, parseIntVector}, 1834 {"import", k_type_import, parseImport}, 1835 {"include", k_type_include, parseInclude}, 1836 {"process(uca_rules)", k_type_plugin_uca_rules, parseUCARules}, 1837 {"process(collation)", k_type_plugin_collation, NULL /* not implemented yet */}, 1838 {"process(transliterator)", k_type_plugin_transliterator, parseTransliterator}, 1839 {"process(dependency)", k_type_plugin_dependency, parseDependency}, 1840 {"reserved", NULL, NULL} 1841 }; 1842 1843 void initParser(UBool omitCollationRules) 1844 { 1845 U_STRING_INIT(k_type_string, "string", 6); 1846 U_STRING_INIT(k_type_binary, "binary", 6); 1847 U_STRING_INIT(k_type_bin, "bin", 3); 1848 U_STRING_INIT(k_type_table, "table", 5); 1849 U_STRING_INIT(k_type_table_no_fallback, "table(nofallback)", 17); 1850 U_STRING_INIT(k_type_int, "int", 3); 1851 U_STRING_INIT(k_type_integer, "integer", 7); 1852 U_STRING_INIT(k_type_array, "array", 5); 1853 U_STRING_INIT(k_type_alias, "alias", 5); 1854 U_STRING_INIT(k_type_intvector, "intvector", 9); 1855 U_STRING_INIT(k_type_import, "import", 6); 1856 U_STRING_INIT(k_type_include, "include", 7); 1857 1858 U_STRING_INIT(k_type_plugin_uca_rules, "process(uca_rules)", 18); 1859 U_STRING_INIT(k_type_plugin_collation, "process(collation)", 18); 1860 U_STRING_INIT(k_type_plugin_transliterator, "process(transliterator)", 23); 1861 U_STRING_INIT(k_type_plugin_dependency, "process(dependency)", 19); 1862 1863 gOmitCollationRules = omitCollationRules; 1864 } 1865 1866 static inline UBool isTable(enum EResourceType type) { 1867 return (UBool)(type==RT_TABLE || type==RT_TABLE_NO_FALLBACK); 1868 } 1869 1870 static enum EResourceType 1871 parseResourceType(ParseState* state, UErrorCode *status) 1872 { 1873 struct UString *tokenValue; 1874 struct UString comment; 1875 enum EResourceType result = RT_UNKNOWN; 1876 uint32_t line=0; 1877 ustr_init(&comment); 1878 expect(state, TOK_STRING, &tokenValue, &comment, &line, status); 1879 1880 if (U_FAILURE(*status)) 1881 { 1882 return RT_UNKNOWN; 1883 } 1884 1885 *status = U_ZERO_ERROR; 1886 1887 /* Search for normal types */ 1888 result=RT_UNKNOWN; 1889 while ((result=(EResourceType)(result+1)) < RT_RESERVED) { 1890 if (u_strcmp(tokenValue->fChars, gResourceTypes[result].nameUChars) == 0) { 1891 break; 1892 } 1893 } 1894 /* Now search for the aliases */ 1895 if (u_strcmp(tokenValue->fChars, k_type_int) == 0) { 1896 result = RT_INTEGER; 1897 } 1898 else if (u_strcmp(tokenValue->fChars, k_type_bin) == 0) { 1899 result = RT_BINARY; 1900 } 1901 else if (result == RT_RESERVED) { 1902 char tokenBuffer[1024]; 1903 u_austrncpy(tokenBuffer, tokenValue->fChars, sizeof(tokenBuffer)); 1904 tokenBuffer[sizeof(tokenBuffer) - 1] = 0; 1905 *status = U_INVALID_FORMAT_ERROR; 1906 error(line, "unknown resource type '%s'", tokenBuffer); 1907 } 1908 1909 return result; 1910 } 1911 1912 /* parse a non-top-level resource */ 1913 static struct SResource * 1914 parseResource(ParseState* state, char *tag, const struct UString *comment, UErrorCode *status) 1915 { 1916 enum ETokenType token; 1917 enum EResourceType resType = RT_UNKNOWN; 1918 ParseResourceFunction *parseFunction = NULL; 1919 struct UString *tokenValue; 1920 uint32_t startline; 1921 uint32_t line; 1922 1923 1924 token = getToken(state, &tokenValue, NULL, &startline, status); 1925 1926 if(isVerbose()){ 1927 printf(" resource %s at line %i \n", (tag == NULL) ? "(null)" : tag, (int)startline); 1928 } 1929 1930 /* name . [ ':' type ] '{' resource '}' */ 1931 /* This function parses from the colon onwards. If the colon is present, parse the 1932 type then try to parse a resource of that type. If there is no explicit type, 1933 work it out using the lookahead tokens. */ 1934 switch (token) 1935 { 1936 case TOK_EOF: 1937 *status = U_INVALID_FORMAT_ERROR; 1938 error(startline, "Unexpected EOF encountered"); 1939 return NULL; 1940 1941 case TOK_ERROR: 1942 *status = U_INVALID_FORMAT_ERROR; 1943 return NULL; 1944 1945 case TOK_COLON: 1946 resType = parseResourceType(state, status); 1947 expect(state, TOK_OPEN_BRACE, &tokenValue, NULL, &startline, status); 1948 1949 if (U_FAILURE(*status)) 1950 { 1951 return NULL; 1952 } 1953 1954 break; 1955 1956 case TOK_OPEN_BRACE: 1957 break; 1958 1959 default: 1960 *status = U_INVALID_FORMAT_ERROR; 1961 error(startline, "syntax error while reading a resource, expected '{' or ':'"); 1962 return NULL; 1963 } 1964 1965 1966 if (resType == RT_UNKNOWN) 1967 { 1968 /* No explicit type, so try to work it out. At this point, we've read the first '{'. 1969 We could have any of the following: 1970 { { => array (nested) 1971 { :/} => array 1972 { string , => string array 1973 1974 { string { => table 1975 1976 { string :/{ => table 1977 { string } => string 1978 */ 1979 1980 token = peekToken(state, 0, NULL, &line, NULL,status); 1981 1982 if (U_FAILURE(*status)) 1983 { 1984 return NULL; 1985 } 1986 1987 if (token == TOK_OPEN_BRACE || token == TOK_COLON ||token ==TOK_CLOSE_BRACE ) 1988 { 1989 resType = RT_ARRAY; 1990 } 1991 else if (token == TOK_STRING) 1992 { 1993 token = peekToken(state, 1, NULL, &line, NULL, status); 1994 1995 if (U_FAILURE(*status)) 1996 { 1997 return NULL; 1998 } 1999 2000 switch (token) 2001 { 2002 case TOK_COMMA: resType = RT_ARRAY; break; 2003 case TOK_OPEN_BRACE: resType = RT_TABLE; break; 2004 case TOK_CLOSE_BRACE: resType = RT_STRING; break; 2005 case TOK_COLON: resType = RT_TABLE; break; 2006 default: 2007 *status = U_INVALID_FORMAT_ERROR; 2008 error(line, "Unexpected token after string, expected ',', '{' or '}'"); 2009 return NULL; 2010 } 2011 } 2012 else 2013 { 2014 *status = U_INVALID_FORMAT_ERROR; 2015 error(line, "Unexpected token after '{'"); 2016 return NULL; 2017 } 2018 2019 /* printf("Type guessed as %s\n", resourceNames[resType]); */ 2020 } else if(resType == RT_TABLE_NO_FALLBACK) { 2021 *status = U_INVALID_FORMAT_ERROR; 2022 error(startline, "error: %s resource type not valid except on top bundle level", gResourceTypes[resType].nameChars); 2023 return NULL; 2024 } 2025 2026 2027 /* We should now know what we need to parse next, so call the appropriate parser 2028 function and return. */ 2029 parseFunction = gResourceTypes[resType].parseFunction; 2030 if (parseFunction != NULL) { 2031 return parseFunction(state, tag, startline, comment, status); 2032 } 2033 else { 2034 *status = U_INTERNAL_PROGRAM_ERROR; 2035 error(startline, "internal error: %s resource type found and not handled", gResourceTypes[resType].nameChars); 2036 } 2037 2038 return NULL; 2039 } 2040 2041 /* parse the top-level resource */ 2042 struct SRBRoot * 2043 parse(UCHARBUF *buf, const char *inputDir, const char *outputDir, UBool makeBinaryCollation, 2044 UErrorCode *status) 2045 { 2046 struct UString *tokenValue; 2047 struct UString comment; 2048 uint32_t line; 2049 enum EResourceType bundleType; 2050 enum ETokenType token; 2051 ParseState state; 2052 uint32_t i; 2053 2054 2055 for (i = 0; i < MAX_LOOKAHEAD + 1; i++) 2056 { 2057 ustr_init(&state.lookahead[i].value); 2058 ustr_init(&state.lookahead[i].comment); 2059 } 2060 2061 initLookahead(&state, buf, status); 2062 2063 state.inputdir = inputDir; 2064 state.inputdirLength = (state.inputdir != NULL) ? (uint32_t)uprv_strlen(state.inputdir) : 0; 2065 state.outputdir = outputDir; 2066 state.outputdirLength = (state.outputdir != NULL) ? (uint32_t)uprv_strlen(state.outputdir) : 0; 2067 state.makeBinaryCollation = makeBinaryCollation; 2068 2069 ustr_init(&comment); 2070 expect(&state, TOK_STRING, &tokenValue, &comment, NULL, status); 2071 2072 state.bundle = bundle_open(&comment, FALSE, status); 2073 2074 if (state.bundle == NULL || U_FAILURE(*status)) 2075 { 2076 return NULL; 2077 } 2078 2079 2080 bundle_setlocale(state.bundle, tokenValue->fChars, status); 2081 2082 /* The following code is to make Empty bundle work no matter with :table specifer or not */ 2083 token = getToken(&state, NULL, NULL, &line, status); 2084 if(token==TOK_COLON) { 2085 *status=U_ZERO_ERROR; 2086 bundleType=parseResourceType(&state, status); 2087 2088 if(isTable(bundleType)) 2089 { 2090 expect(&state, TOK_OPEN_BRACE, NULL, NULL, &line, status); 2091 } 2092 else 2093 { 2094 *status=U_PARSE_ERROR; 2095 error(line, "parse error. Stopped parsing with %s", u_errorName(*status)); 2096 } 2097 } 2098 else 2099 { 2100 /* not a colon */ 2101 if(token==TOK_OPEN_BRACE) 2102 { 2103 *status=U_ZERO_ERROR; 2104 bundleType=RT_TABLE; 2105 } 2106 else 2107 { 2108 /* neither colon nor open brace */ 2109 *status=U_PARSE_ERROR; 2110 bundleType=RT_UNKNOWN; 2111 error(line, "parse error, did not find open-brace '{' or colon ':', stopped with %s", u_errorName(*status)); 2112 } 2113 } 2114 2115 if (U_FAILURE(*status)) 2116 { 2117 bundle_close(state.bundle, status); 2118 return NULL; 2119 } 2120 2121 if(bundleType==RT_TABLE_NO_FALLBACK) { 2122 /* 2123 * Parse a top-level table with the table(nofallback) declaration. 2124 * This is the same as a regular table, but also sets the 2125 * URES_ATT_NO_FALLBACK flag in indexes[URES_INDEX_ATTRIBUTES] . 2126 */ 2127 state.bundle->noFallback=TRUE; 2128 } 2129 /* top-level tables need not handle special table names like "collations" */ 2130 realParseTable(&state, state.bundle->fRoot, NULL, line, status); 2131 if(dependencyArray!=NULL){ 2132 table_add(state.bundle->fRoot, dependencyArray, 0, status); 2133 dependencyArray = NULL; 2134 } 2135 if (U_FAILURE(*status)) 2136 { 2137 bundle_close(state.bundle, status); 2138 res_close(dependencyArray); 2139 return NULL; 2140 } 2141 2142 if (getToken(&state, NULL, NULL, &line, status) != TOK_EOF) 2143 { 2144 warning(line, "extraneous text after resource bundle (perhaps unmatched braces)"); 2145 if(isStrict()){ 2146 *status = U_INVALID_FORMAT_ERROR; 2147 return NULL; 2148 } 2149 } 2150 2151 cleanupLookahead(&state); 2152 ustr_deinit(&comment); 2153 return state.bundle; 2154 } 2155
