1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************** 5 * 6 * Copyright (C) 1998-2015, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************** 10 * 11 * 12 * makeconv.cpp: 13 * tool creating a binary (compressed) representation of the conversion mapping 14 * table (IBM NLTC ucmap format). 15 * 16 * 05/04/2000 helena Added fallback mapping into the picture... 17 * 06/29/2000 helena Major rewrite of the callback APIs. 18 */ 19 20 #include <stdio.h> 21 #include "unicode/putil.h" 22 #include "unicode/ucnv_err.h" 23 #include "charstr.h" 24 #include "ucnv_bld.h" 25 #include "ucnv_imp.h" 26 #include "ucnv_cnv.h" 27 #include "cstring.h" 28 #include "cmemory.h" 29 #include "uinvchar.h" 30 #include "filestrm.h" 31 #include "toolutil.h" 32 #include "uoptions.h" 33 #include "unicode/udata.h" 34 #include "unewdata.h" 35 #include "uparse.h" 36 #include "ucm.h" 37 #include "makeconv.h" 38 #include "genmbcs.h" 39 40 #define DEBUG 0 41 42 typedef struct ConvData { 43 UCMFile *ucm; 44 NewConverter *cnvData, *extData; 45 UConverterSharedData sharedData; 46 UConverterStaticData staticData; 47 } ConvData; 48 49 static void 50 initConvData(ConvData *data) { 51 uprv_memset(data, 0, sizeof(ConvData)); 52 data->sharedData.structSize=sizeof(UConverterSharedData); 53 data->staticData.structSize=sizeof(UConverterStaticData); 54 data->sharedData.staticData=&data->staticData; 55 } 56 57 static void 58 cleanupConvData(ConvData *data) { 59 if(data!=NULL) { 60 if(data->cnvData!=NULL) { 61 data->cnvData->close(data->cnvData); 62 data->cnvData=NULL; 63 } 64 if(data->extData!=NULL) { 65 data->extData->close(data->extData); 66 data->extData=NULL; 67 } 68 ucm_close(data->ucm); 69 data->ucm=NULL; 70 } 71 } 72 73 /* 74 * from ucnvstat.c - static prototypes of data-based converters 75 */ 76 U_CAPI const UConverterStaticData * ucnv_converterStaticData[UCNV_NUMBER_OF_SUPPORTED_CONVERTER_TYPES]; 77 78 /* 79 * Global - verbosity 80 */ 81 UBool VERBOSE = FALSE; 82 UBool QUIET = FALSE; 83 UBool SMALL = FALSE; 84 UBool IGNORE_SISO_CHECK = FALSE; 85 86 static void 87 createConverter(ConvData *data, const char* converterName, UErrorCode *pErrorCode); 88 89 /* 90 * Set up the UNewData and write the converter.. 91 */ 92 static void 93 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status); 94 95 UBool haveCopyright=TRUE; 96 97 static UDataInfo dataInfo={ 98 sizeof(UDataInfo), 99 0, 100 101 U_IS_BIG_ENDIAN, 102 U_CHARSET_FAMILY, 103 sizeof(UChar), 104 0, 105 106 {0x63, 0x6e, 0x76, 0x74}, /* dataFormat="cnvt" */ 107 {6, 2, 0, 0}, /* formatVersion */ 108 {0, 0, 0, 0} /* dataVersion (calculated at runtime) */ 109 }; 110 111 static void 112 writeConverterData(ConvData *data, const char *cnvName, const char *cnvDir, UErrorCode *status) 113 { 114 UNewDataMemory *mem = NULL; 115 uint32_t sz2; 116 uint32_t size = 0; 117 int32_t tableType; 118 119 if(U_FAILURE(*status)) 120 { 121 return; 122 } 123 124 tableType=TABLE_NONE; 125 if(data->cnvData!=NULL) { 126 tableType|=TABLE_BASE; 127 } 128 if(data->extData!=NULL) { 129 tableType|=TABLE_EXT; 130 } 131 132 mem = udata_create(cnvDir, "cnv", cnvName, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, status); 133 134 if(U_FAILURE(*status)) 135 { 136 fprintf(stderr, "Couldn't create the udata %s.%s: %s\n", 137 cnvName, 138 "cnv", 139 u_errorName(*status)); 140 return; 141 } 142 143 if(VERBOSE) 144 { 145 printf("- Opened udata %s.%s\n", cnvName, "cnv"); 146 } 147 148 149 /* all read only, clean, platform independent data. Mmmm. :) */ 150 udata_writeBlock(mem, &data->staticData, sizeof(UConverterStaticData)); 151 size += sizeof(UConverterStaticData); /* Is 4-aligned - by size */ 152 /* Now, write the table */ 153 if(tableType&TABLE_BASE) { 154 size += data->cnvData->write(data->cnvData, &data->staticData, mem, tableType); 155 } 156 if(tableType&TABLE_EXT) { 157 size += data->extData->write(data->extData, &data->staticData, mem, tableType); 158 } 159 160 sz2 = udata_finish(mem, status); 161 if(size != sz2) 162 { 163 fprintf(stderr, "error: wrote %u bytes to the .cnv file but counted %u bytes\n", (int)sz2, (int)size); 164 *status=U_INTERNAL_PROGRAM_ERROR; 165 } 166 if(VERBOSE) 167 { 168 printf("- Wrote %u bytes to the udata.\n", (int)sz2); 169 } 170 } 171 172 enum { 173 OPT_HELP_H, 174 OPT_HELP_QUESTION_MARK, 175 OPT_COPYRIGHT, 176 OPT_VERSION, 177 OPT_DESTDIR, 178 OPT_VERBOSE, 179 OPT_SMALL, 180 OPT_IGNORE_SISO_CHECK, 181 OPT_QUIET, 182 183 OPT_COUNT 184 }; 185 186 static UOption options[]={ 187 UOPTION_HELP_H, 188 UOPTION_HELP_QUESTION_MARK, 189 UOPTION_COPYRIGHT, 190 UOPTION_VERSION, 191 UOPTION_DESTDIR, 192 UOPTION_VERBOSE, 193 { "small", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }, 194 { "ignore-siso-check", NULL, NULL, NULL, '\1', UOPT_NO_ARG, 0 }, 195 UOPTION_QUIET, 196 }; 197 198 int main(int argc, char* argv[]) 199 { 200 ConvData data; 201 char cnvName[UCNV_MAX_FULL_FILE_NAME_LENGTH]; 202 203 U_MAIN_INIT_ARGS(argc, argv); 204 205 /* Set up the ICU version number */ 206 UVersionInfo icuVersion; 207 u_getVersion(icuVersion); 208 uprv_memcpy(&dataInfo.dataVersion, &icuVersion, sizeof(UVersionInfo)); 209 210 /* preset then read command line options */ 211 options[OPT_DESTDIR].value=u_getDataDirectory(); 212 argc=u_parseArgs(argc, argv, UPRV_LENGTHOF(options), options); 213 214 /* error handling, printing usage message */ 215 if(argc<0) { 216 fprintf(stderr, 217 "error in command line argument \"%s\"\n", 218 argv[-argc]); 219 } else if(argc<2) { 220 argc=-1; 221 } 222 if(argc<0 || options[OPT_HELP_H].doesOccur || options[OPT_HELP_QUESTION_MARK].doesOccur) { 223 FILE *stdfile=argc<0 ? stderr : stdout; 224 fprintf(stdfile, 225 "usage: %s [-options] files...\n" 226 "\tread .ucm codepage mapping files and write .cnv files\n" 227 "options:\n" 228 "\t-h or -? or --help this usage text\n" 229 "\t-V or --version show a version message\n" 230 "\t-c or --copyright include a copyright notice\n" 231 "\t-d or --destdir destination directory, followed by the path\n" 232 "\t-v or --verbose Turn on verbose output\n" 233 "\t-q or --quiet do not display warnings and progress\n", 234 argv[0]); 235 fprintf(stdfile, 236 "\t --small Generate smaller .cnv files. They will be\n" 237 "\t significantly smaller but may not be compatible with\n" 238 "\t older versions of ICU and will require heap memory\n" 239 "\t allocation when loaded.\n" 240 "\t --ignore-siso-check Use SI/SO other than 0xf/0xe.\n"); 241 return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR; 242 } 243 244 if(options[OPT_VERSION].doesOccur) { 245 printf("makeconv version %u.%u, ICU tool to read .ucm codepage mapping files and write .cnv files\n", 246 dataInfo.formatVersion[0], dataInfo.formatVersion[1]); 247 printf("%s\n", U_COPYRIGHT_STRING); 248 exit(0); 249 } 250 251 /* get the options values */ 252 haveCopyright = options[OPT_COPYRIGHT].doesOccur; 253 const char *destdir = options[OPT_DESTDIR].value; 254 VERBOSE = options[OPT_VERBOSE].doesOccur; 255 QUIET = options[OPT_QUIET].doesOccur; 256 SMALL = options[OPT_SMALL].doesOccur; 257 258 if (options[OPT_IGNORE_SISO_CHECK].doesOccur) { 259 IGNORE_SISO_CHECK = TRUE; 260 } 261 262 icu::CharString outFileName; 263 UErrorCode err = U_ZERO_ERROR; 264 if (destdir != NULL && *destdir != 0) { 265 outFileName.append(destdir, err).ensureEndsWithFileSeparator(err); 266 if (U_FAILURE(err)) { 267 return err; 268 } 269 } 270 int32_t outBasenameStart = outFileName.length(); 271 272 #if DEBUG 273 { 274 int i; 275 printf("makeconv: processing %d files...\n", argc - 1); 276 for(i=1; i<argc; ++i) { 277 printf("%s ", argv[i]); 278 } 279 printf("\n"); 280 fflush(stdout); 281 } 282 #endif 283 284 UBool printFilename = (UBool) (argc > 2 || VERBOSE); 285 for (++argv; --argc; ++argv) 286 { 287 UErrorCode localError = U_ZERO_ERROR; 288 const char *arg = getLongPathname(*argv); 289 290 /*produces the right destination path for display*/ 291 outFileName.truncate(outBasenameStart); 292 if (outBasenameStart != 0) 293 { 294 /* find the last file sepator */ 295 const char *basename = findBasename(arg); 296 outFileName.append(basename, localError); 297 } 298 else 299 { 300 outFileName.append(arg, localError); 301 } 302 if (U_FAILURE(localError)) { 303 return localError; 304 } 305 306 /*removes the extension if any is found*/ 307 int32_t lastDotIndex = outFileName.lastIndexOf('.'); 308 if (lastDotIndex >= outBasenameStart) { 309 outFileName.truncate(lastDotIndex); 310 } 311 312 /* the basename without extension is the converter name */ 313 if ((outFileName.length() - outBasenameStart) >= UPRV_LENGTHOF(cnvName)) { 314 fprintf(stderr, "converter name %s too long\n", outFileName.data() + outBasenameStart); 315 return U_BUFFER_OVERFLOW_ERROR; 316 } 317 uprv_strcpy(cnvName, outFileName.data() + outBasenameStart); 318 319 /*Adds the target extension*/ 320 outFileName.append(CONVERTER_FILE_EXTENSION, localError); 321 if (U_FAILURE(localError)) { 322 return localError; 323 } 324 325 #if DEBUG 326 printf("makeconv: processing %s ...\n", arg); 327 fflush(stdout); 328 #endif 329 initConvData(&data); 330 createConverter(&data, arg, &localError); 331 332 if (U_FAILURE(localError)) 333 { 334 /* if an error is found, print out an error msg and keep going */ 335 fprintf(stderr, "Error creating converter for \"%s\" file for \"%s\" (%s)\n", 336 outFileName.data(), arg, u_errorName(localError)); 337 if(U_SUCCESS(err)) { 338 err = localError; 339 } 340 } 341 else 342 { 343 /* Insure the static data name matches the file name */ 344 /* Changed to ignore directory and only compare base name 345 LDH 1/2/08*/ 346 char *p; 347 p = strrchr(cnvName, U_FILE_SEP_CHAR); /* Find last file separator */ 348 349 if(p == NULL) /* OK, try alternate */ 350 { 351 p = strrchr(cnvName, U_FILE_ALT_SEP_CHAR); 352 if(p == NULL) 353 { 354 p=cnvName; /* If no separators, no problem */ 355 } 356 } 357 else 358 { 359 p++; /* If found separator, don't include it in compare */ 360 } 361 if(uprv_stricmp(p,data.staticData.name) && !QUIET) 362 { 363 fprintf(stderr, "Warning: %s%s claims to be '%s'\n", 364 cnvName, CONVERTER_FILE_EXTENSION, 365 data.staticData.name); 366 } 367 368 uprv_strcpy((char*)data.staticData.name, cnvName); 369 370 if(!uprv_isInvariantString((char*)data.staticData.name, -1)) { 371 fprintf(stderr, 372 "Error: A converter name must contain only invariant characters.\n" 373 "%s is not a valid converter name.\n", 374 data.staticData.name); 375 if(U_SUCCESS(err)) { 376 err = U_INVALID_TABLE_FORMAT; 377 } 378 } 379 380 localError = U_ZERO_ERROR; 381 writeConverterData(&data, cnvName, destdir, &localError); 382 383 if(U_FAILURE(localError)) 384 { 385 /* if an error is found, print out an error msg and keep going*/ 386 fprintf(stderr, "Error writing \"%s\" file for \"%s\" (%s)\n", outFileName.data(), arg, 387 u_errorName(localError)); 388 if(U_SUCCESS(err)) { 389 err = localError; 390 } 391 } 392 else if (printFilename) 393 { 394 puts(outFileName.data() + outBasenameStart); 395 } 396 } 397 fflush(stdout); 398 fflush(stderr); 399 400 cleanupConvData(&data); 401 } 402 403 return err; 404 } 405 406 static void 407 getPlatformAndCCSIDFromName(const char *name, int8_t *pPlatform, int32_t *pCCSID) { 408 if( (name[0]=='i' || name[0]=='I') && 409 (name[1]=='b' || name[1]=='B') && 410 (name[2]=='m' || name[2]=='M') 411 ) { 412 name+=3; 413 if(*name=='-') { 414 ++name; 415 } 416 *pPlatform=UCNV_IBM; 417 *pCCSID=(int32_t)uprv_strtoul(name, NULL, 10); 418 } else { 419 *pPlatform=UCNV_UNKNOWN; 420 *pCCSID=0; 421 } 422 } 423 424 static void 425 readHeader(ConvData *data, 426 FileStream* convFile, 427 UErrorCode *pErrorCode) { 428 char line[1024]; 429 char *s, *key, *value; 430 const UConverterStaticData *prototype; 431 UConverterStaticData *staticData; 432 433 if(U_FAILURE(*pErrorCode)) { 434 return; 435 } 436 437 staticData=&data->staticData; 438 staticData->platform=UCNV_IBM; 439 staticData->subCharLen=0; 440 441 while(T_FileStream_readLine(convFile, line, sizeof(line))) { 442 /* basic parsing and handling of state-related items */ 443 if(ucm_parseHeaderLine(data->ucm, line, &key, &value)) { 444 continue; 445 } 446 447 /* stop at the beginning of the mapping section */ 448 if(uprv_strcmp(line, "CHARMAP")==0) { 449 break; 450 } 451 452 /* collect the information from the header field, ignore unknown keys */ 453 if(uprv_strcmp(key, "code_set_name")==0) { 454 if(*value!=0) { 455 uprv_strcpy((char *)staticData->name, value); 456 getPlatformAndCCSIDFromName(value, &staticData->platform, &staticData->codepage); 457 } 458 } else if(uprv_strcmp(key, "subchar")==0) { 459 uint8_t bytes[UCNV_EXT_MAX_BYTES]; 460 int8_t length; 461 462 s=value; 463 length=ucm_parseBytes(bytes, line, (const char **)&s); 464 if(1<=length && length<=4 && *s==0) { 465 staticData->subCharLen=length; 466 uprv_memcpy(staticData->subChar, bytes, length); 467 } else { 468 fprintf(stderr, "error: illegal <subchar> %s\n", value); 469 *pErrorCode=U_INVALID_TABLE_FORMAT; 470 return; 471 } 472 } else if(uprv_strcmp(key, "subchar1")==0) { 473 uint8_t bytes[UCNV_EXT_MAX_BYTES]; 474 475 s=value; 476 if(1==ucm_parseBytes(bytes, line, (const char **)&s) && *s==0) { 477 staticData->subChar1=bytes[0]; 478 } else { 479 fprintf(stderr, "error: illegal <subchar1> %s\n", value); 480 *pErrorCode=U_INVALID_TABLE_FORMAT; 481 return; 482 } 483 } 484 } 485 486 /* copy values from the UCMFile to the static data */ 487 staticData->maxBytesPerChar=(int8_t)data->ucm->states.maxCharLength; 488 staticData->minBytesPerChar=(int8_t)data->ucm->states.minCharLength; 489 staticData->conversionType=data->ucm->states.conversionType; 490 491 if(staticData->conversionType==UCNV_UNSUPPORTED_CONVERTER) { 492 fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n"); 493 *pErrorCode=U_INVALID_TABLE_FORMAT; 494 return; 495 } 496 497 /* 498 * Now that we know the type, copy any 'default' values from the table. 499 * We need not check the type any further because the parser only 500 * recognizes what we have prototypes for. 501 * 502 * For delta (extension-only) tables, copy values from the base file 503 * instead, see createConverter(). 504 */ 505 if(data->ucm->baseName[0]==0) { 506 prototype=ucnv_converterStaticData[staticData->conversionType]; 507 if(prototype!=NULL) { 508 if(staticData->name[0]==0) { 509 uprv_strcpy((char *)staticData->name, prototype->name); 510 } 511 512 if(staticData->codepage==0) { 513 staticData->codepage=prototype->codepage; 514 } 515 516 if(staticData->platform==0) { 517 staticData->platform=prototype->platform; 518 } 519 520 if(staticData->minBytesPerChar==0) { 521 staticData->minBytesPerChar=prototype->minBytesPerChar; 522 } 523 524 if(staticData->maxBytesPerChar==0) { 525 staticData->maxBytesPerChar=prototype->maxBytesPerChar; 526 } 527 528 if(staticData->subCharLen==0) { 529 staticData->subCharLen=prototype->subCharLen; 530 if(prototype->subCharLen>0) { 531 uprv_memcpy(staticData->subChar, prototype->subChar, prototype->subCharLen); 532 } 533 } 534 } 535 } 536 537 if(data->ucm->states.outputType<0) { 538 data->ucm->states.outputType=(int8_t)data->ucm->states.maxCharLength-1; 539 } 540 541 if( staticData->subChar1!=0 && 542 (staticData->minBytesPerChar>1 || 543 (staticData->conversionType!=UCNV_MBCS && 544 staticData->conversionType!=UCNV_EBCDIC_STATEFUL)) 545 ) { 546 fprintf(stderr, "error: <subchar1> defined for a type other than MBCS or EBCDIC_STATEFUL\n"); 547 *pErrorCode=U_INVALID_TABLE_FORMAT; 548 } 549 } 550 551 /* return TRUE if a base table was read, FALSE for an extension table */ 552 static UBool 553 readFile(ConvData *data, const char* converterName, 554 UErrorCode *pErrorCode) { 555 char line[1024]; 556 char *end; 557 FileStream *convFile; 558 559 UCMStates *baseStates; 560 UBool dataIsBase; 561 562 if(U_FAILURE(*pErrorCode)) { 563 return FALSE; 564 } 565 566 data->ucm=ucm_open(); 567 568 convFile=T_FileStream_open(converterName, "r"); 569 if(convFile==NULL) { 570 *pErrorCode=U_FILE_ACCESS_ERROR; 571 return FALSE; 572 } 573 574 readHeader(data, convFile, pErrorCode); 575 if(U_FAILURE(*pErrorCode)) { 576 return FALSE; 577 } 578 579 if(data->ucm->baseName[0]==0) { 580 dataIsBase=TRUE; 581 baseStates=&data->ucm->states; 582 ucm_processStates(baseStates, IGNORE_SISO_CHECK); 583 } else { 584 dataIsBase=FALSE; 585 baseStates=NULL; 586 } 587 588 /* read the base table */ 589 ucm_readTable(data->ucm, convFile, dataIsBase, baseStates, pErrorCode); 590 if(U_FAILURE(*pErrorCode)) { 591 return FALSE; 592 } 593 594 /* read an extension table if there is one */ 595 while(T_FileStream_readLine(convFile, line, sizeof(line))) { 596 end=uprv_strchr(line, 0); 597 while(line<end && 598 (*(end-1)=='\n' || *(end-1)=='\r' || *(end-1)==' ' || *(end-1)=='\t')) { 599 --end; 600 } 601 *end=0; 602 603 if(line[0]=='#' || u_skipWhitespace(line)==end) { 604 continue; /* ignore empty and comment lines */ 605 } 606 607 if(0==uprv_strcmp(line, "CHARMAP")) { 608 /* read the extension table */ 609 ucm_readTable(data->ucm, convFile, FALSE, baseStates, pErrorCode); 610 } else { 611 fprintf(stderr, "unexpected text after the base mapping table\n"); 612 } 613 break; 614 } 615 616 T_FileStream_close(convFile); 617 618 if(data->ucm->base->flagsType==UCM_FLAGS_MIXED || data->ucm->ext->flagsType==UCM_FLAGS_MIXED) { 619 fprintf(stderr, "error: some entries have the mapping precision (with '|'), some do not\n"); 620 *pErrorCode=U_INVALID_TABLE_FORMAT; 621 } 622 623 return dataIsBase; 624 } 625 626 static void 627 createConverter(ConvData *data, const char *converterName, UErrorCode *pErrorCode) { 628 ConvData baseData; 629 UBool dataIsBase; 630 631 UConverterStaticData *staticData; 632 UCMStates *states, *baseStates; 633 634 if(U_FAILURE(*pErrorCode)) { 635 return; 636 } 637 638 initConvData(data); 639 640 dataIsBase=readFile(data, converterName, pErrorCode); 641 if(U_FAILURE(*pErrorCode)) { 642 return; 643 } 644 645 staticData=&data->staticData; 646 states=&data->ucm->states; 647 648 if(dataIsBase) { 649 /* 650 * Build a normal .cnv file with a base table 651 * and an optional extension table. 652 */ 653 data->cnvData=MBCSOpen(data->ucm); 654 if(data->cnvData==NULL) { 655 *pErrorCode=U_MEMORY_ALLOCATION_ERROR; 656 657 } else if(!data->cnvData->isValid(data->cnvData, 658 staticData->subChar, staticData->subCharLen) 659 ) { 660 fprintf(stderr, " the substitution character byte sequence is illegal in this codepage structure!\n"); 661 *pErrorCode=U_INVALID_TABLE_FORMAT; 662 663 } else if(staticData->subChar1!=0 && 664 !data->cnvData->isValid(data->cnvData, &staticData->subChar1, 1) 665 ) { 666 fprintf(stderr, " the subchar1 byte is illegal in this codepage structure!\n"); 667 *pErrorCode=U_INVALID_TABLE_FORMAT; 668 669 } else if( 670 data->ucm->ext->mappingsLength>0 && 671 !ucm_checkBaseExt(states, data->ucm->base, data->ucm->ext, data->ucm->ext, FALSE) 672 ) { 673 *pErrorCode=U_INVALID_TABLE_FORMAT; 674 } else if(data->ucm->base->flagsType&UCM_FLAGS_EXPLICIT) { 675 /* sort the table so that it can be turned into UTF-8-friendly data */ 676 ucm_sortTable(data->ucm->base); 677 } 678 679 if(U_SUCCESS(*pErrorCode)) { 680 if( 681 /* add the base table after ucm_checkBaseExt()! */ 682 !data->cnvData->addTable(data->cnvData, data->ucm->base, &data->staticData) 683 ) { 684 *pErrorCode=U_INVALID_TABLE_FORMAT; 685 } else { 686 /* 687 * addTable() may have requested moving more mappings to the extension table 688 * if they fit into the base toUnicode table but not into the 689 * base fromUnicode table. 690 * (Especially for UTF-8-friendly fromUnicode tables.) 691 * Such mappings will have the MBCS_FROM_U_EXT_FLAG set, which causes them 692 * to be excluded from the extension toUnicode data. 693 * See MBCSOkForBaseFromUnicode() for which mappings do not fit into 694 * the base fromUnicode table. 695 */ 696 ucm_moveMappings(data->ucm->base, data->ucm->ext); 697 ucm_sortTable(data->ucm->ext); 698 if(data->ucm->ext->mappingsLength>0) { 699 /* prepare the extension table, if there is one */ 700 data->extData=CnvExtOpen(data->ucm); 701 if(data->extData==NULL) { 702 *pErrorCode=U_MEMORY_ALLOCATION_ERROR; 703 } else if( 704 !data->extData->addTable(data->extData, data->ucm->ext, &data->staticData) 705 ) { 706 *pErrorCode=U_INVALID_TABLE_FORMAT; 707 } 708 } 709 } 710 } 711 } else { 712 /* Build an extension-only .cnv file. */ 713 char baseFilename[500]; 714 char *basename; 715 716 initConvData(&baseData); 717 718 /* assemble a path/filename for data->ucm->baseName */ 719 uprv_strcpy(baseFilename, converterName); 720 basename=(char *)findBasename(baseFilename); 721 uprv_strcpy(basename, data->ucm->baseName); 722 uprv_strcat(basename, ".ucm"); 723 724 /* read the base table */ 725 dataIsBase=readFile(&baseData, baseFilename, pErrorCode); 726 if(U_FAILURE(*pErrorCode)) { 727 return; 728 } else if(!dataIsBase) { 729 fprintf(stderr, "error: the <icu:base> file \"%s\" is not a base table file\n", baseFilename); 730 *pErrorCode=U_INVALID_TABLE_FORMAT; 731 } else { 732 /* prepare the extension table */ 733 data->extData=CnvExtOpen(data->ucm); 734 if(data->extData==NULL) { 735 *pErrorCode=U_MEMORY_ALLOCATION_ERROR; 736 } else { 737 /* fill in gaps in extension file header fields */ 738 UCMapping *m, *mLimit; 739 uint8_t fallbackFlags; 740 741 baseStates=&baseData.ucm->states; 742 if(states->conversionType==UCNV_DBCS) { 743 staticData->minBytesPerChar=(int8_t)(states->minCharLength=2); 744 } else if(states->minCharLength==0) { 745 staticData->minBytesPerChar=(int8_t)(states->minCharLength=baseStates->minCharLength); 746 } 747 if(states->maxCharLength<states->minCharLength) { 748 staticData->maxBytesPerChar=(int8_t)(states->maxCharLength=baseStates->maxCharLength); 749 } 750 751 if(staticData->subCharLen==0) { 752 uprv_memcpy(staticData->subChar, baseData.staticData.subChar, 4); 753 staticData->subCharLen=baseData.staticData.subCharLen; 754 } 755 /* 756 * do not copy subChar1 - 757 * only use what is explicitly specified 758 * because it cannot be unset in the extension file header 759 */ 760 761 /* get the fallback flags */ 762 fallbackFlags=0; 763 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength; 764 m<mLimit && fallbackFlags!=3; 765 ++m 766 ) { 767 if(m->f==1) { 768 fallbackFlags|=1; 769 } else if(m->f==3) { 770 fallbackFlags|=2; 771 } 772 } 773 774 if(fallbackFlags&1) { 775 staticData->hasFromUnicodeFallback=TRUE; 776 } 777 if(fallbackFlags&2) { 778 staticData->hasToUnicodeFallback=TRUE; 779 } 780 781 if(1!=ucm_countChars(baseStates, staticData->subChar, staticData->subCharLen)) { 782 fprintf(stderr, " the substitution character byte sequence is illegal in this codepage structure!\n"); 783 *pErrorCode=U_INVALID_TABLE_FORMAT; 784 785 } else if(staticData->subChar1!=0 && 1!=ucm_countChars(baseStates, &staticData->subChar1, 1)) { 786 fprintf(stderr, " the subchar1 byte is illegal in this codepage structure!\n"); 787 *pErrorCode=U_INVALID_TABLE_FORMAT; 788 789 } else if( 790 !ucm_checkValidity(data->ucm->ext, baseStates) || 791 !ucm_checkBaseExt(baseStates, baseData.ucm->base, data->ucm->ext, data->ucm->ext, FALSE) 792 ) { 793 *pErrorCode=U_INVALID_TABLE_FORMAT; 794 } else { 795 if(states->maxCharLength>1) { 796 /* 797 * When building a normal .cnv file with a base table 798 * for an MBCS (not SBCS) table with explicit precision flags, 799 * the MBCSAddTable() function marks some mappings for moving 800 * to the extension table. 801 * They fit into the base toUnicode table but not into the 802 * base fromUnicode table. 803 * (Note: We do have explicit precision flags because they are 804 * required for extension table generation, and 805 * ucm_checkBaseExt() verified it.) 806 * 807 * We do not call MBCSAddTable() here (we probably could) 808 * so we need to do the analysis before building the extension table. 809 * We assume that MBCSAddTable() will build a UTF-8-friendly table. 810 * Redundant mappings in the extension table are ok except they cost some size. 811 * 812 * Do this after ucm_checkBaseExt(). 813 */ 814 const MBCSData *mbcsData=MBCSGetDummy(); 815 int32_t needsMove=0; 816 for(m=baseData.ucm->base->mappings, mLimit=m+baseData.ucm->base->mappingsLength; 817 m<mLimit; 818 ++m 819 ) { 820 if(!MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, m->u, m->f)) { 821 m->f|=MBCS_FROM_U_EXT_FLAG; 822 m->moveFlag=UCM_MOVE_TO_EXT; 823 ++needsMove; 824 } 825 } 826 827 if(needsMove!=0) { 828 ucm_moveMappings(baseData.ucm->base, data->ucm->ext); 829 ucm_sortTable(data->ucm->ext); 830 } 831 } 832 if(!data->extData->addTable(data->extData, data->ucm->ext, &data->staticData)) { 833 *pErrorCode=U_INVALID_TABLE_FORMAT; 834 } 835 } 836 } 837 } 838 839 cleanupConvData(&baseData); 840 } 841 } 842 843 /* 844 * Hey, Emacs, please set the following: 845 * 846 * Local Variables: 847 * indent-tabs-mode: nil 848 * End: 849 * 850 */ 851
