1 /* 2 ******************************************************************************** 3 * 4 * Copyright (C) 1998-2014, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************** 8 * 9 * 10 * makeconv.c: 11 * tool creating a binary (compressed) representation of the conversion mapping 12 * table (IBM NLTC ucmap format). 13 * 14 * 05/04/2000 helena Added fallback mapping into the picture... 15 * 06/29/2000 helena Major rewrite of the callback APIs. 16 */ 17 18 #include <stdio.h> 19 #include "unicode/putil.h" 20 #include "unicode/ucnv_err.h" 21 #include "ucnv_bld.h" 22 #include "ucnv_imp.h" 23 #include "ucnv_cnv.h" 24 #include "cstring.h" 25 #include "cmemory.h" 26 #include "uinvchar.h" 27 #include "filestrm.h" 28 #include "toolutil.h" 29 #include "uoptions.h" 30 #include "unicode/udata.h" 31 #include "unewdata.h" 32 #include "uparse.h" 33 #include "ucm.h" 34 #include "makeconv.h" 35 #include "genmbcs.h" 36 37 #define DEBUG 0 38 39 typedef struct ConvData { 40 UCMFile *ucm; 41 NewConverter *cnvData, *extData; 42 UConverterSharedData sharedData; 43 UConverterStaticData staticData; 44 } ConvData; 45 46 static void 47 initConvData(ConvData *data) { 48 uprv_memset(data, 0, sizeof(ConvData)); 49 data->sharedData.structSize=sizeof(UConverterSharedData); 50 data->staticData.structSize=sizeof(UConverterStaticData); 51 data->sharedData.staticData=&data->staticData; 52 } 53 54 static void 55 cleanupConvData(ConvData *data) { 56 if(data!=NULL) { 57 if(data->cnvData!=NULL) { 58 data->cnvData->close(data->cnvData); 59 data->cnvData=NULL; 60 } 61 if(data->extData!=NULL) { 62 data->extData->close(data->extData); 63 data->extData=NULL; 64 } 65 ucm_close(data->ucm); 66 data->ucm=NULL; 67 } 68 } 69 70 /* 71 * from ucnvstat.c - static prototypes of data-based converters 72 */ 73 extern const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES]; 74 75 /* 76 * Global - verbosity 77 */ 78 UBool VERBOSE = FALSE; 79 UBool SMALL = FALSE; 80 UBool IGNORE_SISO_CHECK = FALSE; 81 82 static void 83 createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode); 84 85 /* 86 * Set up the UNewData and write the converter.. 87 */ 88 static void 89 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status); 90 91 UBool haveCopyright=TRUE; 92 93 static UDataInfo dataInfo={ 94 sizeof(UDataInfo), 95 0, 96 97 U_IS_BIG_ENDIAN, 98 U_CHARSET_FAMILY, 99 sizeof(UChar), 100 0, 101 102 {0x63, 0x6e, 0x76, 0x74}, /* dataFormat="cnvt" */ 103 {6, 2, 0, 0}, /* formatVersion */ 104 {0, 0, 0, 0} /* dataVersion (calculated at runtime) */ 105 }; 106 107 static void 108 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status) 109 { 110 UNewDataMemory *mem = NULL; 111 uint32_t sz2; 112 uint32_t size = 0; 113 int32_t tableType; 114 115 if(U_FAILURE(*status)) 116 { 117 return; 118 } 119 120 tableType=TABLE_NONE; 121 if(data->cnvData!=NULL) { 122 tableType|=TABLE_BASE; 123 } 124 if(data->extData!=NULL) { 125 tableType|=TABLE_EXT; 126 } 127 128 mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status); 129 130 if(U_FAILURE(*status)) 131 { 132 fprintf(stderr, "Couldn't create the udata %s.%s: %s\n", 133 cnvName, 134 "cnv", 135 u_errorName(*status)); 136 return; 137 } 138 139 if(VERBOSE) 140 { 141 printf("- Opened udata %s.%s\n", cnvName, "cnv"); 142 } 143 144 145 /* all read only, clean, platform independent data. Mmmm. :) */ 146 udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData)); 147 size += sizeof(UConverterStaticData); /* Is 4-aligned - by size */ 148 /* Now, write the table */ 149 if(tableType&TABLE_BASE) { 150 size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType); 151 } 152 if(tableType&TABLE_EXT) { 153 size += data->extData->write(data->extData, &data->staticData, mem, tableType); 154 } 155 156 sz2 = udata_finish(mem, status); 157 if(size != sz2) 158 { 159 fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size); 160 *status=U_INTERNAL_PROGRAM_ERROR; 161 } 162 if(VERBOSE) 163 { 164 printf("- Wrote %u bytes to the udata.\n", (int)sz2); 165 } 166 } 167 168 enum { 169 OPT_HELP_H, 170 OPT_HELP_QUESTION_MARK, 171 OPT_COPYRIGHT, 172 OPT_VERSION, 173 OPT_DESTDIR, 174 OPT_VERBOSE, 175 OPT_SMALL, 176 OPT_IGNORE_SISO_CHECK, 177 OPT_COUNT 178 }; 179 180 static UOption options[]={ 181 UOPTION_HELP_H, 182 UOPTION_HELP_QUESTION_MARK, 183 UOPTION_COPYRIGHT, 184 UOPTION_VERSION, 185 UOPTION_DESTDIR, 186 UOPTION_VERBOSE, 187 { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }, 188 { "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 } 189 }; 190 191 int main(int argc, char* argv[]) 192 { 193 ConvData data; 194 UErrorCode err = U_ZERO_ERROR, localError; 195 char outFileName[UCNV_MAX_FULL_FILE_NAME_LENGTH]; 196 const char* destdir, *arg; 197 size_t destdirlen; 198 char* dot = NULL, *outBasename; 199 char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH]; 200 char cnvNameWithPkg[UCNV_MAX_FULL_FILE_NAME_LENGTH]; 201 UVersionInfo icuVersion; 202 UBool printFilename; 203 204 err = U_ZERO_ERROR; 205 206 U_MAIN_INIT_ARGS(argc, argv); 207 208 /* Set up the ICU version number */ 209 u_getVersion(icuVersion); 210 uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo)); 211 212 /* preset then read command line options */ 213 options[OPT_DESTDIR].value=u_getDataDirectory(); 214 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); 215 216 /* error handling, printing usage message */ 217 if(argc<0) { 218 fprintf(stderr, 219 "error in command line argument \"%s\"\n", 220 argv[-argc]); 221 } else if(argc<2) { 222 argc=-1; 223 } 224 if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) { 225 FILE *stdfile=argc<0 ? stderr : stdout; 226 fprintf(stdfile, 227 "usage: %s [-options] files...\n" 228 "\tread .ucm codepage mapping files and write .cnv files\n" 229 "options:\n" 230 "\t-h or -? or --help this usage text\n" 231 "\t-V or --version show a version message\n" 232 "\t-c or --copyright include a copyright notice\n" 233 "\t-d or --destdir destination directory, followed by the path\n" 234 "\t-v or --verbose Turn on verbose output\n", 235 argv[0]); 236 fprintf(stdfile, 237 "\t --small Generate smaller .cnv files. They will be\n" 238 "\t significantly smaller but may not be compatible with\n" 239 "\t older versions of ICU and will require heap memory\n" 240 "\t allocation when loaded.\n" 241 "\t --ignore-siso-check Use SI/SO other than 0xf/0xe.\n"); 242 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; 243 } 244 245 if(options[OPT_VERSION].doesOccur) { 246 printf("makeconv version %u.%u, ICU tool to read .ucm codepage mapping files and write .cnv files\n", 247 dataInfo.formatVersion[0], dataInfo.formatVersion[1]); 248 printf("%s\n", U_COPYRIGHT_STRING); 249 exit(0); 250 } 251 252 /* get the options values */ 253 haveCopyright = options[OPT_COPYRIGHT].doesOccur; 254 destdir = options[OPT_DESTDIR].value; 255 VERBOSE = options[OPT_VERBOSE].doesOccur; 256 SMALL = options[OPT_SMALL].doesOccur; 257 258 if (options[OPT_IGNORE_SISO_CHECK].doesOccur) { 259 IGNORE_SISO_CHECK = TRUE; 260 } 261 262 if (destdir != NULL && *destdir != 0) { 263 uprv_strcpy(outFileName, destdir); 264 destdirlen = uprv_strlen(destdir); 265 outBasename = outFileName + destdirlen; 266 if (*(outBasename - 1) != U_FILE_SEP_CHAR) { 267 *outBasename++ = U_FILE_SEP_CHAR; 268 ++destdirlen; 269 } 270 } else { 271 destdirlen = 0; 272 outBasename = outFileName; 273 } 274 275 #if DEBUG 276 { 277 int i; 278 printf("makeconv: processing %d files...\n", argc - 1); 279 for(i=1; i<argc; ++i) { 280 printf("%s ", argv[i]); 281 } 282 printf("\n"); 283 fflush(stdout); 284 } 285 #endif 286 287 err = U_ZERO_ERROR; 288 printFilename = (UBool) (argc > 2 || VERBOSE); 289 for (++argv; --argc; ++argv) 290 { 291 arg = getLongPathname(*argv); 292 293 /* Check for potential buffer overflow */ 294 if(strlen(arg) >= UCNV_MAX_FULL_FILE_NAME_LENGTH) 295 { 296 fprintf(stderr, "%s\n", u_errorName(U_BUFFER_OVERFLOW_ERROR)); 297 return U_BUFFER_OVERFLOW_ERROR; 298 } 299 300 /*produces the right destination path for display*/ 301 if (destdirlen != 0) 302 { 303 const char *basename; 304 305 /* find the last file sepator */ 306 basename = findBasename(arg); 307 uprv_strcpy(outBasename, basename); 308 } 309 else 310 { 311 uprv_strcpy(outFileName, arg); 312 } 313 314 /*removes the extension if any is found*/ 315 dot = uprv_strrchr(outBasename, '.'); 316 if (dot) 317 { 318 *dot = '\0'; 319 } 320 321 /* the basename without extension is the converter name */ 322 uprv_strcpy(cnvName, outBasename); 323 324 /*Adds the target extension*/ 325 uprv_strcat(outBasename, CONVERTER_FILE_EXTENSION); 326 327 #if DEBUG 328 printf("makeconv: processing %s ...\n", arg); 329 fflush(stdout); 330 #endif 331 localError = U_ZERO_ERROR; 332 initConvData(&data); 333 createConverter(&data, arg, &localError); 334 335 if (U_FAILURE(localError)) 336 { 337 /* if an error is found, print out an error msg and keep going */ 338 fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n", outFileName, arg, 339 u_errorName(localError)); 340 if(U_SUCCESS(err)) { 341 err = localError; 342 } 343 } 344 else 345 { 346 /* Insure the static data name matches the file name */ 347 /* Changed to ignore directory and only compare base name 348 LDH 1/2/08*/ 349 char *p; 350 p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */ 351 352 if(p == NULL) /* OK, try alternate */ 353 { 354 p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR); 355 if(p == NULL) 356 { 357 p=cnvName; /* If no separators, no problem */ 358 } 359 } 360 else 361 { 362 p++; /* If found separtor, don't include it in compare */ 363 } 364 if(uprv_stricmp(p,data.staticData.name)) 365 { 366 fprintf(stderr, "Warning: %s%s claims to be '%s'\n", 367 cnvName, CONVERTER_FILE_EXTENSION, 368 data.staticData.name); 369 } 370 371 uprv_strcpy((char*)data.staticData.name, cnvName); 372 373 if(!uprv_isInvariantString((char*)data.staticData.name, -1)) { 374 fprintf(stderr, 375 "Error: A converter name must contain only invariant characters.\n" 376 "%s is not a valid converter name.\n", 377 data.staticData.name); 378 if(U_SUCCESS(err)) { 379 err = U_INVALID_TABLE_FORMAT; 380 } 381 } 382 383 uprv_strcpy(cnvNameWithPkg, cnvName); 384 385 localError = U_ZERO_ERROR; 386 writeConverterData(&data, cnvNameWithPkg, destdir, &localError); 387 388 if(U_FAILURE(localError)) 389 { 390 /* if an error is found, print out an error msg and keep going*/ 391 fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName, arg, 392 u_errorName(localError)); 393 if(U_SUCCESS(err)) { 394 err = localError; 395 } 396 } 397 else if (printFilename) 398 { 399 puts(outBasename); 400 } 401 } 402 fflush(stdout); 403 fflush(stderr); 404 405 cleanupConvData(&data); 406 } 407 408 return err; 409 } 410 411 static void 412 getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) { 413 if( (name[0]=='i' || name[0]=='I') && 414 (name[1]=='b' || name[1]=='B') && 415 (name[2]=='m' || name[2]=='M') 416 ) { 417 name+=3; 418 if(*name=='-') { 419 ++name; 420 } 421 *pPlatform=UCNV_IBM; 422 *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10); 423 } else { 424 *pPlatform=UCNV_UNKNOWN; 425 *pCCSID=0; 426 } 427 } 428 429 static void 430 readHeader(ConvData *data, 431 FileStream* convFile, 432 const char* converterName, 433 UErrorCode *pErrorCode) { 434 char line[1024]; 435 char *s, *key, *value; 436 const UConverterStaticData *prototype; 437 UConverterStaticData *staticData; 438 439 if(U_FAILURE(*pErrorCode)) { 440 return; 441 } 442 443 staticData=&data->staticData; 444 staticData->platform=UCNV_IBM; 445 staticData->subCharLen=0; 446 447 while(T_FileStream_readLine(convFile, line, sizeof(line))) { 448 /* basic parsing and handling of state-related items */ 449 if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) { 450 continue; 451 } 452 453 /* stop at the beginning of the mapping section */ 454 if(uprv_strcmp(line, "CHARMAP")==0) { 455 break; 456 } 457 458 /* collect the information from the header field, ignore unknown keys */ 459 if(uprv_strcmp(key, "code_set_name")==0) { 460 if(*value!=0) { 461 uprv_strcpy((char *)staticData->name, value); 462 getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage); 463 } 464 } else if(uprv_strcmp(key, "subchar")==0) { 465 uint8_t bytes[UCNV_EXT_MAX_BYTES]; 466 int8_t length; 467 468 s=value; 469 length=ucm_parseBytes(bytes, line, (const char **)&s); 470 if(1<=length && length<=4 && *s==0) { 471 staticData->subCharLen=length; 472 uprv_memcpy(staticData->subChar, bytes, length); 473 } else { 474 fprintf(stderr, "error: illegal <subchar> %s\n", value); 475 *pErrorCode=U_INVALID_TABLE_FORMAT; 476 return; 477 } 478 } else if(uprv_strcmp(key, "subchar1")==0) { 479 uint8_t bytes[UCNV_EXT_MAX_BYTES]; 480 481 s=value; 482 if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) { 483 staticData->subChar1=bytes[0]; 484 } else { 485 fprintf(stderr, "error: illegal <subchar1> %s\n", value); 486 *pErrorCode=U_INVALID_TABLE_FORMAT; 487 return; 488 } 489 } 490 } 491 492 /* copy values from the UCMFile to the static data */ 493 staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength; 494 staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength; 495 staticData->conversionType=data->ucm->states.conversionType; 496 497 if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) { 498 fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n"); 499 *pErrorCode=U_INVALID_TABLE_FORMAT; 500 return; 501 } 502 503 /* 504 * Now that we know the type, copy any 'default' values from the table. 505 * We need not check the type any further because the parser only 506 * recognizes what we have prototypes for. 507 * 508 * For delta (extension-only) tables, copy values from the base file 509 * instead, see createConverter(). 510 */ 511 if(data->ucm->baseName[0]==0) { 512 prototype=ucnv_converterStaticData[staticData->conversionType]; 513 if(prototype!=NULL) { 514 if(staticData->name[0]==0) { 515 uprv_strcpy((char *)staticData->name, prototype->name); 516 } 517 518 if(staticData->codepage==0) { 519 staticData->codepage=prototype->codepage; 520 } 521 522 if(staticData->platform==0) { 523 staticData->platform=prototype->platform; 524 } 525 526 if(staticData->minBytesPerChar==0) { 527 staticData->minBytesPerChar=prototype->minBytesPerChar; 528 } 529 530 if(staticData->maxBytesPerChar==0) { 531 staticData->maxBytesPerChar=prototype->maxBytesPerChar; 532 } 533 534 if(staticData->subCharLen==0) { 535 staticData->subCharLen=prototype->subCharLen; 536 if(prototype->subCharLen>0) { 537 uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen); 538 } 539 } 540 } 541 } 542 543 if(data->ucm->states.outputType<0) { 544 data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1; 545 } 546 547 if( staticData->subChar1!=0 && 548 (staticData->minBytesPerChar>1 || 549 (staticData->conversionType!=UCNV_MBCS && 550 staticData->conversionType!=UCNV_EBCDIC_STATEFUL)) 551 ) { 552 fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n"); 553 *pErrorCode=U_INVALID_TABLE_FORMAT; 554 } 555 } 556 557 /* return TRUE if a base table was read, FALSE for an extension table */ 558 static UBool 559 readFile(ConvData *data, const char* converterName, 560 UErrorCode *pErrorCode) { 561 char line[1024]; 562 char *end; 563 FileStream *convFile; 564 565 UCMStates *baseStates; 566 UBool dataIsBase; 567 568 if(U_FAILURE(*pErrorCode)) { 569 return FALSE; 570 } 571 572 data->ucm=ucm_open(); 573 574 convFile=T_FileStream_open(converterName, "r"); 575 if(convFile==NULL) { 576 *pErrorCode=U_FILE_ACCESS_ERROR; 577 return FALSE; 578 } 579 580 readHeader(data, convFile, converterName, pErrorCode); 581 if(U_FAILURE(*pErrorCode)) { 582 return FALSE; 583 } 584 585 if(data->ucm->baseName[0]==0) { 586 dataIsBase=TRUE; 587 baseStates=&data->ucm->states; 588 ucm_processStates(baseStates, IGNORE_SISO_CHECK); 589 } else { 590 dataIsBase=FALSE; 591 baseStates=NULL; 592 } 593 594 /* read the base table */ 595 ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode); 596 if(U_FAILURE(*pErrorCode)) { 597 return FALSE; 598 } 599 600 /* read an extension table if there is one */ 601 while(T_FileStream_readLine(convFile, line, sizeof(line))) { 602 end=uprv_strchr(line, 0); 603 while(line<end && 604 (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) { 605 --end; 606 } 607 *end=0; 608 609 if(line[0]=='#' || u_skipWhitespace(line)==end) { 610 continue; /* ignore empty and comment lines */ 611 } 612 613 if(0==uprv_strcmp(line, "CHARMAP")) { 614 /* read the extension table */ 615 ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode); 616 } else { 617 fprintf(stderr, "unexpected text after the base mapping table\n"); 618 } 619 break; 620 } 621 622 T_FileStream_close(convFile); 623 624 if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) { 625 fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n"); 626 *pErrorCode=U_INVALID_TABLE_FORMAT; 627 } 628 629 return dataIsBase; 630 } 631 632 static void 633 createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) { 634 ConvData baseData; 635 UBool dataIsBase; 636 637 UConverterStaticData *staticData; 638 UCMStates *states, *baseStates; 639 640 if(U_FAILURE(*pErrorCode)) { 641 return; 642 } 643 644 initConvData(data); 645 646 dataIsBase=readFile(data, converterName, pErrorCode); 647 if(U_FAILURE(*pErrorCode)) { 648 return; 649 } 650 651 staticData=&data->staticData; 652 states=&data->ucm->states; 653 654 if(dataIsBase) { 655 /* 656 * Build a normal .cnv file with a base table 657 * and an optional extension table. 658 */ 659 data->cnvData=MBCSOpen(data->ucm); 660 if(data->cnvData==NULL) { 661 *pErrorCode=U_MEMORY_ALLOCATION_ERROR; 662 663 } else if(!data->cnvData->isValid(data->cnvData, 664 staticData->subChar, staticData->subCharLen) 665 ) { 666 fprintf(stderr, " the substitution character byte sequence is illegal in this codepage structure!\n"); 667 *pErrorCode=U_INVALID_TABLE_FORMAT; 668 669 } else if(staticData->subChar1!=0 && 670 !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1) 671 ) { 672 fprintf(stderr, " the subchar1 byte is illegal in this codepage structure!\n"); 673 *pErrorCode=U_INVALID_TABLE_FORMAT; 674 675 } else if( 676 data->ucm->ext->mappingsLength>0 && 677 !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, FALSE) 678 ) { 679 *pErrorCode=U_INVALID_TABLE_FORMAT; 680 } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) { 681 /* sort the table so that it can be turned into UTF-8-friendly data */ 682 ucm_sortTable(data->ucm->base); 683 } 684 685 if(U_SUCCESS(*pErrorCode)) { 686 if( 687 /* add the base table after ucm_checkBaseExt()! */ 688 !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData) 689 ) { 690 *pErrorCode=U_INVALID_TABLE_FORMAT; 691 } else { 692 /* 693 * addTable() may have requested moving more mappings to the extension table 694 * if they fit into the base toUnicode table but not into the 695 * base fromUnicode table. 696 * (Especially for UTF-8-friendly fromUnicode tables.) 697 * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them 698 * to be excluded from the extension toUnicode data. 699 * See MBCSOkForBaseFromUnicode() for which mappings do not fit into 700 * the base fromUnicode table. 701 */ 702 ucm_moveMappings(data->ucm->base, data->ucm->ext); 703 ucm_sortTable(data->ucm->ext); 704 if(data->ucm->ext->mappingsLength>0) { 705 /* prepare the extension table, if there is one */ 706 data->extData=CnvExtOpen(data->ucm); 707 if(data->extData==NULL) { 708 *pErrorCode=U_MEMORY_ALLOCATION_ERROR; 709 } else if( 710 !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData) 711 ) { 712 *pErrorCode=U_INVALID_TABLE_FORMAT; 713 } 714 } 715 } 716 } 717 } else { 718 /* Build an extension-only .cnv file. */ 719 char baseFilename[500]; 720 char *basename; 721 722 initConvData(&baseData); 723 724 /* assemble a path/filename for data->ucm->baseName */ 725 uprv_strcpy(baseFilename, converterName); 726 basename=(char *)findBasename(baseFilename); 727 uprv_strcpy(basename, data->ucm->baseName); 728 uprv_strcat(basename, ".ucm"); 729 730 /* read the base table */ 731 dataIsBase=readFile(&baseData, baseFilename, pErrorCode); 732 if(U_FAILURE(*pErrorCode)) { 733 return; 734 } else if(!dataIsBase) { 735 fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename); 736 *pErrorCode=U_INVALID_TABLE_FORMAT; 737 } else { 738 /* prepare the extension table */ 739 data->extData=CnvExtOpen(data->ucm); 740 if(data->extData==NULL) { 741 *pErrorCode=U_MEMORY_ALLOCATION_ERROR; 742 } else { 743 /* fill in gaps in extension file header fields */ 744 UCMapping *m, *mLimit; 745 uint8_t fallbackFlags; 746 747 baseStates=&baseData.ucm->states; 748 if(states->conversionType==UCNV_DBCS) { 749 staticData->minBytesPerChar=(int8_t)(states->minCharLength=2); 750 } else if(states->minCharLength==0) { 751 staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength); 752 } 753 if(states->maxCharLength<states->minCharLength) { 754 staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength); 755 } 756 757 if(staticData->subCharLen==0) { 758 uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4); 759 staticData->subCharLen=baseData.staticData.subCharLen; 760 } 761 /* 762 * do not copy subChar1 - 763 * only use what is explicitly specified 764 * because it cannot be unset in the extension file header 765 */ 766 767 /* get the fallback flags */ 768 fallbackFlags=0; 769 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength; 770 m<mLimit && fallbackFlags!=3; 771 ++m 772 ) { 773 if(m->f==1) { 774 fallbackFlags|=1; 775 } else if(m->f==3) { 776 fallbackFlags|=2; 777 } 778 } 779 780 if(fallbackFlags&1) { 781 staticData->hasFromUnicodeFallback=TRUE; 782 } 783 if(fallbackFlags&2) { 784 staticData->hasToUnicodeFallback=TRUE; 785 } 786 787 if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) { 788 fprintf(stderr, " the substitution character byte sequence is illegal in this codepage structure!\n"); 789 *pErrorCode=U_INVALID_TABLE_FORMAT; 790 791 } else if(staticData->subChar1!=0 && 1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) { 792 fprintf(stderr, " the subchar1 byte is illegal in this codepage structure!\n"); 793 *pErrorCode=U_INVALID_TABLE_FORMAT; 794 795 } else if( 796 !ucm_checkValidity(data->ucm->ext, baseStates) || 797 !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, FALSE) 798 ) { 799 *pErrorCode=U_INVALID_TABLE_FORMAT; 800 } else { 801 if(states->maxCharLength>1) { 802 /* 803 * When building a normal .cnv file with a base table 804 * for an MBCS (not SBCS) table with explicit precision flags, 805 * the MBCSAddTable() function marks some mappings for moving 806 * to the extension table. 807 * They fit into the base toUnicode table but not into the 808 * base fromUnicode table. 809 * (Note: We do have explicit precision flags because they are 810 * required for extension table generation, and 811 * ucm_checkBaseExt() verified it.) 812 * 813 * We do not call MBCSAddTable() here (we probably could) 814 * so we need to do the analysis before building the extension table. 815 * We assume that MBCSAddTable() will build a UTF-8-friendly table. 816 * Redundant mappings in the extension table are ok except they cost some size. 817 * 818 * Do this after ucm_checkBaseExt(). 819 */ 820 const MBCSData *mbcsData=MBCSGetDummy(); 821 int32_t needsMove=0; 822 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength; 823 m<mLimit; 824 ++m 825 ) { 826 if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) { 827 m->f|=MBCS_FROM_U_EXT_FLAG; 828 m->moveFlag=UCM_MOVE_TO_EXT; 829 ++needsMove; 830 } 831 } 832 833 if(needsMove!=0) { 834 ucm_moveMappings(baseData.ucm->base, data->ucm->ext); 835 ucm_sortTable(data->ucm->ext); 836 } 837 } 838 if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) { 839 *pErrorCode=U_INVALID_TABLE_FORMAT; 840 } 841 } 842 } 843 } 844 845 cleanupConvData(&baseData); 846 } 847 } 848 849 /* 850 * Hey, Emacs, please set the following: 851 * 852 * Local Variables: 853 * indent-tabs-mode: nil 854 * End: 855 * 856 */ 857
