1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2003-2009, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: pkgitems.cpp 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2005sep18 14 * created by: Markus W. Scherer 15 * 16 * Companion file to package.cpp. Deals with details of ICU data item formats. 17 * Used for item dependencies. 18 * Contains adapted code from ucnv_bld.c (swapper code from 2003). 19 */ 20 21 #include "unicode/utypes.h" 22 #include "unicode/ures.h" 23 #include "unicode/putil.h" 24 #include "unicode/udata.h" 25 #include "cstring.h" 26 #include "uinvchar.h" 27 #include "ucmndata.h" 28 #include "udataswp.h" 29 #include "swapimpl.h" 30 #include "toolutil.h" 31 #include "package.h" 32 #include "pkg_imp.h" 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <string.h> 37 38 /* item formats in common */ 39 40 #include "uresdata.h" 41 #include "ucnv_bld.h" 42 #include "ucnv_io.h" 43 44 // general definitions ----------------------------------------------------- *** 45 46 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) 47 48 U_CDECL_BEGIN 49 50 static void U_CALLCONV 51 printError(void *context, const char *fmt, va_list args) { 52 vfprintf((FILE *)context, fmt, args); 53 } 54 55 U_CDECL_END 56 57 // a data item in native-platform form ------------------------------------- *** 58 59 U_NAMESPACE_BEGIN 60 61 class NativeItem { 62 public: 63 NativeItem() : pItem(NULL), pInfo(NULL), bytes(NULL), swapped(NULL), length(0) {} 64 NativeItem(const Item *item, UDataSwapFn *swap) : swapped(NULL) { 65 setItem(item, swap); 66 } 67 ~NativeItem() { 68 delete [] swapped; 69 } 70 const UDataInfo *getDataInfo() const { 71 return pInfo; 72 } 73 const uint8_t *getBytes() const { 74 return bytes; 75 } 76 int32_t getLength() const { 77 return length; 78 } 79 80 void setItem(const Item *item, UDataSwapFn *swap) { 81 pItem=item; 82 int32_t infoLength, itemHeaderLength; 83 UErrorCode errorCode=U_ZERO_ERROR; 84 pInfo=::getDataInfo(pItem->data, pItem->length, infoLength, itemHeaderLength, &errorCode); 85 if(U_FAILURE(errorCode)) { 86 exit(errorCode); // should succeed because readFile() checks headers 87 } 88 length=pItem->length-itemHeaderLength; 89 90 if(pInfo->isBigEndian==U_IS_BIG_ENDIAN && pInfo->charsetFamily==U_CHARSET_FAMILY) { 91 bytes=pItem->data+itemHeaderLength; 92 } else { 93 UDataSwapper *ds=udata_openSwapper((UBool)pInfo->isBigEndian, pInfo->charsetFamily, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &errorCode); 94 if(U_FAILURE(errorCode)) { 95 fprintf(stderr, "icupkg: udata_openSwapper(\"%s\") failed - %s\n", 96 pItem->name, u_errorName(errorCode)); 97 exit(errorCode); 98 } 99 100 ds->printError=printError; 101 ds->printErrorContext=stderr; 102 103 swapped=new uint8_t[pItem->length]; 104 if(swapped==NULL) { 105 fprintf(stderr, "icupkg: unable to allocate memory for swapping \"%s\"\n", pItem->name); 106 exit(U_MEMORY_ALLOCATION_ERROR); 107 } 108 swap(ds, pItem->data, pItem->length, swapped, &errorCode); 109 pInfo=::getDataInfo(swapped, pItem->length, infoLength, itemHeaderLength, &errorCode); 110 bytes=swapped+itemHeaderLength; 111 udata_closeSwapper(ds); 112 } 113 } 114 115 private: 116 const Item *pItem; 117 const UDataInfo *pInfo; 118 const uint8_t *bytes; 119 uint8_t *swapped; 120 int32_t length; 121 }; 122 123 U_NAMESPACE_END 124 125 // check a dependency ------------------------------------------------------ *** 126 127 /* 128 * assemble the target item name from the source item name, an ID 129 * and a suffix 130 */ 131 static void 132 makeTargetName(const char *itemName, const char *id, int32_t idLength, const char *suffix, 133 char *target, int32_t capacity, 134 UErrorCode *pErrorCode) { 135 const char *itemID; 136 int32_t treeLength, suffixLength, targetLength; 137 138 // get the item basename 139 itemID=strrchr(itemName, '/'); 140 if(itemID!=NULL) { 141 ++itemID; 142 } else { 143 itemID=itemName; 144 } 145 146 // build the target string 147 treeLength=(int32_t)(itemID-itemName); 148 if(idLength<0) { 149 idLength=(int32_t)strlen(id); 150 } 151 suffixLength=(int32_t)strlen(suffix); 152 targetLength=treeLength+idLength+suffixLength; 153 if(targetLength>=capacity) { 154 fprintf(stderr, "icupkg/makeTargetName(%s) target item name length %ld too long\n", 155 itemName, (long)targetLength); 156 *pErrorCode=U_BUFFER_OVERFLOW_ERROR; 157 return; 158 } 159 160 memcpy(target, itemName, treeLength); 161 memcpy(target+treeLength, id, idLength); 162 memcpy(target+treeLength+idLength, suffix, suffixLength+1); // +1 includes the terminating NUL 163 } 164 165 static void 166 checkIDSuffix(const char *itemName, const char *id, int32_t idLength, const char *suffix, 167 CheckDependency check, void *context, 168 UErrorCode *pErrorCode) { 169 char target[200]; 170 makeTargetName(itemName, id, idLength, suffix, target, (int32_t)sizeof(target), pErrorCode); 171 if(U_SUCCESS(*pErrorCode)) { 172 check(context, itemName, target); 173 } 174 } 175 176 /* assemble the target item name from the item's parent item name */ 177 static void 178 checkParent(const char *itemName, CheckDependency check, void *context, 179 UErrorCode *pErrorCode) { 180 const char *itemID, *parent, *parentLimit, *suffix; 181 int32_t parentLength; 182 183 // get the item basename 184 itemID=strrchr(itemName, '/'); 185 if(itemID!=NULL) { 186 ++itemID; 187 } else { 188 itemID=itemName; 189 } 190 191 // get the item suffix 192 suffix=strrchr(itemID, '.'); 193 if(suffix==NULL) { 194 // empty suffix, point to the end of the string 195 suffix=strrchr(itemID, 0); 196 } 197 198 // get the position of the last '_' 199 for(parentLimit=suffix; parentLimit>itemID && *--parentLimit!='_';) {} 200 201 if(parentLimit!=itemID) { 202 // get the parent item name by truncating the last part of this item's name */ 203 parent=itemID; 204 parentLength=(int32_t)(parentLimit-itemID); 205 } else { 206 // no '_' in the item name: the parent is the root bundle 207 parent="root"; 208 parentLength=4; 209 if((suffix-itemID)==parentLength && 0==memcmp(itemID, parent, parentLength)) { 210 // the item itself is "root", which does not depend on a parent 211 return; 212 } 213 } 214 checkIDSuffix(itemName, parent, parentLength, suffix, check, context, pErrorCode); 215 } 216 217 // get dependencies from resource bundles ---------------------------------- *** 218 219 static const UChar SLASH=0x2f; 220 221 /* 222 * Check for the alias from the string or alias resource res. 223 */ 224 static void 225 checkAlias(const char *itemName, 226 Resource res, const UChar *alias, int32_t length, UBool useResSuffix, 227 CheckDependency check, void *context, UErrorCode *pErrorCode) { 228 int32_t i; 229 230 if(!uprv_isInvariantUString(alias, length)) { 231 fprintf(stderr, "icupkg/ures_enumDependencies(%s res=%08x) alias string contains non-invariant characters\n", 232 itemName, res); 233 *pErrorCode=U_INVALID_CHAR_FOUND; 234 return; 235 } 236 237 // extract the locale ID from alias strings like 238 // locale_ID/key1/key2/key3 239 // locale_ID 240 241 // search for the first slash 242 for(i=0; i<length && alias[i]!=SLASH; ++i) {} 243 244 if(res_getPublicType(res)==URES_ALIAS) { 245 // ignore aliases with an initial slash: 246 // /ICUDATA/... and /pkgname/... go to a different package 247 // /LOCALE/... are for dynamic sideways fallbacks and don't go to a fixed bundle 248 if(i==0) { 249 return; // initial slash ('/') 250 } 251 252 // ignore the intra-bundle path starting from the first slash ('/') 253 length=i; 254 } else /* URES_STRING */ { 255 // the whole string should only consist of a locale ID 256 if(i!=length) { 257 fprintf(stderr, "icupkg/ures_enumDependencies(%s res=%08x) %%ALIAS contains a '/'\n", 258 itemName, res); 259 *pErrorCode=U_UNSUPPORTED_ERROR; 260 return; 261 } 262 } 263 264 // convert the Unicode string to char * 265 char localeID[32]; 266 if(length>=(int32_t)sizeof(localeID)) { 267 fprintf(stderr, "icupkg/ures_enumDependencies(%s res=%08x) alias locale ID length %ld too long\n", 268 itemName, res, (long)length); 269 *pErrorCode=U_BUFFER_OVERFLOW_ERROR; 270 return; 271 } 272 u_UCharsToChars(alias, localeID, length); 273 localeID[length]=0; 274 275 checkIDSuffix(itemName, localeID, -1, (useResSuffix ? ".res" : ""), check, context, pErrorCode); 276 } 277 278 /* 279 * Enumerate one resource item and its children and extract dependencies from 280 * aliases. 281 */ 282 static void 283 ures_enumDependencies(const char *itemName, 284 const ResourceData *pResData, 285 Resource res, const char *inKey, const char *parentKey, int32_t depth, 286 CheckDependency check, void *context, 287 UErrorCode *pErrorCode) { 288 switch(res_getPublicType(res)) { 289 case URES_STRING: 290 { 291 UBool useResSuffix = TRUE; 292 // Check for %%ALIAS 293 if(depth==1 && inKey!=NULL) { 294 if(0!=strcmp(inKey, "%%ALIAS")) { 295 break; 296 } 297 } 298 // Check for %%DEPENDENCY 299 else if(depth==2 && parentKey!=NULL) { 300 if(0!=strcmp(parentKey, "%%DEPENDENCY")) { 301 break; 302 } 303 useResSuffix = FALSE; 304 } else { 305 // we ignore all other strings 306 break; 307 } 308 int32_t length; 309 const UChar *alias=res_getString(pResData, res, &length); 310 checkAlias(itemName, res, alias, length, useResSuffix, check, context, pErrorCode); 311 } 312 break; 313 case URES_ALIAS: 314 { 315 int32_t length; 316 const UChar *alias=res_getAlias(pResData, res, &length); 317 checkAlias(itemName, res, alias, length, TRUE, check, context, pErrorCode); 318 } 319 break; 320 case URES_TABLE: 321 { 322 /* recurse */ 323 int32_t count=res_countArrayItems(pResData, res); 324 for(int32_t i=0; i<count; ++i) { 325 const char *itemKey; 326 Resource item=res_getTableItemByIndex(pResData, res, i, &itemKey); 327 ures_enumDependencies( 328 itemName, pResData, 329 item, itemKey, 330 inKey, depth+1, 331 check, context, 332 pErrorCode); 333 if(U_FAILURE(*pErrorCode)) { 334 fprintf(stderr, "icupkg/ures_enumDependencies(%s table res=%08x)[%d].recurse(%s: %08x) failed\n", 335 itemName, res, i, itemKey, item); 336 break; 337 } 338 } 339 } 340 break; 341 case URES_ARRAY: 342 { 343 /* recurse */ 344 int32_t count=res_countArrayItems(pResData, res); 345 for(int32_t i=0; i<count; ++i) { 346 Resource item=res_getArrayItem(pResData, res, i); 347 ures_enumDependencies( 348 itemName, pResData, 349 item, NULL, 350 inKey, depth+1, 351 check, context, 352 pErrorCode); 353 if(U_FAILURE(*pErrorCode)) { 354 fprintf(stderr, "icupkg/ures_enumDependencies(%s array res=%08x)[%d].recurse(%08x) failed\n", 355 itemName, res, i, item); 356 break; 357 } 358 } 359 } 360 break; 361 default: 362 break; 363 } 364 } 365 366 static void 367 ures_enumDependencies(const char *itemName, const UDataInfo *pInfo, 368 const uint8_t *inBytes, int32_t length, 369 CheckDependency check, void *context, 370 UErrorCode *pErrorCode) { 371 ResourceData resData; 372 373 res_read(&resData, pInfo, inBytes, length, pErrorCode); 374 if(U_FAILURE(*pErrorCode)) { 375 fprintf(stderr, "icupkg: .res format version %02x.%02x not supported, or bundle malformed\n", 376 pInfo->formatVersion[0], pInfo->formatVersion[1]); 377 exit(U_UNSUPPORTED_ERROR); 378 } 379 380 /* 381 * if the bundle attributes are present and the nofallback flag is not set, 382 * then add the parent bundle as a dependency 383 */ 384 if(pInfo->formatVersion[0]>1 || (pInfo->formatVersion[0]==1 && pInfo->formatVersion[1]>=1)) { 385 if(!resData.noFallback) { 386 /* this bundle participates in locale fallback */ 387 checkParent(itemName, check, context, pErrorCode); 388 } 389 } 390 391 U_NAMESPACE_QUALIFIER NativeItem nativePool; 392 393 if(resData.usesPoolBundle) { 394 char poolName[200]; 395 makeTargetName(itemName, "pool", 4, ".res", poolName, (int32_t)sizeof(poolName), pErrorCode); 396 if(U_FAILURE(*pErrorCode)) { 397 return; 398 } 399 check(context, itemName, poolName); 400 // TODO: The Package should be passed in. 401 // Since the context is always a Package, we could just redeclare it. 402 U_NAMESPACE_QUALIFIER Package *pkg=(U_NAMESPACE_QUALIFIER Package *)context; 403 int32_t index=pkg->findItem(poolName); 404 if(index<0) { 405 // We cannot work with a bundle if its pool resource is missing. 406 // check() already printed a complaint. 407 return; 408 } 409 // TODO: Cache the native version in the Item itself. 410 nativePool.setItem(pkg->getItem(index), ures_swap); 411 const UDataInfo *poolInfo=nativePool.getDataInfo(); 412 if(poolInfo->formatVersion[0]<=1) { 413 fprintf(stderr, "icupkg: %s is not a pool bundle\n", poolName); 414 return; 415 } 416 const int32_t *poolIndexes=(const int32_t *)nativePool.getBytes()+1; 417 int32_t poolIndexLength=poolIndexes[URES_INDEX_LENGTH]&0xff; 418 if(!(poolIndexLength>URES_INDEX_POOL_CHECKSUM && 419 (poolIndexes[URES_INDEX_ATTRIBUTES]&URES_ATT_IS_POOL_BUNDLE)) 420 ) { 421 fprintf(stderr, "icupkg: %s is not a pool bundle\n", poolName); 422 return; 423 } 424 if(resData.pRoot[1+URES_INDEX_POOL_CHECKSUM]==poolIndexes[URES_INDEX_POOL_CHECKSUM]) { 425 resData.poolBundleKeys=(const char *)(poolIndexes+poolIndexLength); 426 } else { 427 fprintf(stderr, "icupkg: %s has mismatched checksum for %s\n", poolName, itemName); 428 return; 429 } 430 } 431 432 ures_enumDependencies( 433 itemName, &resData, 434 resData.rootRes, NULL, NULL, 0, 435 check, context, 436 pErrorCode); 437 } 438 439 // get dependencies from conversion tables --------------------------------- *** 440 441 /* code adapted from ucnv_swap() */ 442 static void 443 ucnv_enumDependencies(const UDataSwapper *ds, 444 const char *itemName, const UDataInfo *pInfo, 445 const uint8_t *inBytes, int32_t length, 446 CheckDependency check, void *context, 447 UErrorCode *pErrorCode) { 448 uint32_t staticDataSize; 449 450 const UConverterStaticData *inStaticData; 451 452 const _MBCSHeader *inMBCSHeader; 453 uint8_t outputType; 454 455 /* check format version */ 456 if(!( 457 pInfo->formatVersion[0]==6 && 458 pInfo->formatVersion[1]>=2 459 )) { 460 fprintf(stderr, "icupkg/ucnv_enumDependencies(): .cnv format version %02x.%02x not supported\n", 461 pInfo->formatVersion[0], pInfo->formatVersion[1]); 462 exit(U_UNSUPPORTED_ERROR); 463 } 464 465 /* read the initial UConverterStaticData structure after the UDataInfo header */ 466 inStaticData=(const UConverterStaticData *)inBytes; 467 468 if( length<(int32_t)sizeof(UConverterStaticData) || 469 (uint32_t)length<(staticDataSize=ds->readUInt32(inStaticData->structSize)) 470 ) { 471 udata_printError(ds, "icupkg/ucnv_enumDependencies(): too few bytes (%d after header) for an ICU .cnv conversion table\n", 472 length); 473 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; 474 return; 475 } 476 477 inBytes+=staticDataSize; 478 length-=(int32_t)staticDataSize; 479 480 /* check for supported conversionType values */ 481 if(inStaticData->conversionType==UCNV_MBCS) { 482 /* MBCS data */ 483 uint32_t mbcsHeaderLength, mbcsHeaderFlags, mbcsHeaderOptions; 484 int32_t extOffset; 485 486 inMBCSHeader=(const _MBCSHeader *)inBytes; 487 488 if(length<(int32_t)sizeof(_MBCSHeader)) { 489 udata_printError(ds, "icupkg/ucnv_enumDependencies(): too few bytes (%d after headers) for an ICU MBCS .cnv conversion table\n", 490 length); 491 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; 492 return; 493 } 494 if(inMBCSHeader->version[0]==4 && inMBCSHeader->version[1]>=1) { 495 mbcsHeaderLength=MBCS_HEADER_V4_LENGTH; 496 } else if(inMBCSHeader->version[0]==5 && inMBCSHeader->version[1]>=3 && 497 ((mbcsHeaderOptions=ds->readUInt32(inMBCSHeader->options))& 498 MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK)==0 499 ) { 500 mbcsHeaderLength=mbcsHeaderOptions&MBCS_OPT_LENGTH_MASK; 501 } else { 502 udata_printError(ds, "icupkg/ucnv_enumDependencies(): unsupported _MBCSHeader.version %d.%d\n", 503 inMBCSHeader->version[0], inMBCSHeader->version[1]); 504 *pErrorCode=U_UNSUPPORTED_ERROR; 505 return; 506 } 507 508 mbcsHeaderFlags=ds->readUInt32(inMBCSHeader->flags); 509 extOffset=(int32_t)(mbcsHeaderFlags>>8); 510 outputType=(uint8_t)mbcsHeaderFlags; 511 512 if(outputType==MBCS_OUTPUT_EXT_ONLY) { 513 /* 514 * extension-only file, 515 * contains a base name instead of normal base table data 516 */ 517 char baseName[32]; 518 int32_t baseNameLength; 519 520 /* there is extension data after the base data, see ucnv_ext.h */ 521 if(length<(extOffset+UCNV_EXT_INDEXES_MIN_LENGTH*4)) { 522 udata_printError(ds, "icupkg/ucnv_enumDependencies(): too few bytes (%d after headers) for an ICU MBCS .cnv conversion table with extension data\n", 523 length); 524 *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; 525 return; 526 } 527 528 /* swap the base name, between the header and the extension data */ 529 const char *inBaseName=(const char *)inBytes+mbcsHeaderLength*4; 530 baseNameLength=(int32_t)strlen(inBaseName); 531 if(baseNameLength>=(int32_t)sizeof(baseName)) { 532 udata_printError(ds, "icupkg/ucnv_enumDependencies(%s): base name length %ld too long\n", 533 itemName, baseNameLength); 534 *pErrorCode=U_UNSUPPORTED_ERROR; 535 return; 536 } 537 ds->swapInvChars(ds, inBaseName, baseNameLength+1, baseName, pErrorCode); 538 539 checkIDSuffix(itemName, baseName, -1, ".cnv", check, context, pErrorCode); 540 } 541 } 542 } 543 544 // ICU data formats -------------------------------------------------------- *** 545 546 static const struct { 547 uint8_t dataFormat[4]; 548 } dataFormats[]={ 549 { { 0x52, 0x65, 0x73, 0x42 } }, /* dataFormat="ResB" */ 550 { { 0x63, 0x6e, 0x76, 0x74 } }, /* dataFormat="cnvt" */ 551 { { 0x43, 0x76, 0x41, 0x6c } } /* dataFormat="CvAl" */ 552 }; 553 554 enum { 555 FMT_RES, 556 FMT_CNV, 557 FMT_ALIAS, 558 FMT_COUNT 559 }; 560 561 static int32_t 562 getDataFormat(const uint8_t dataFormat[4]) { 563 int32_t i; 564 565 for(i=0; i<FMT_COUNT; ++i) { 566 if(0==memcmp(dataFormats[i].dataFormat, dataFormat, 4)) { 567 return i; 568 } 569 } 570 return -1; 571 } 572 573 // enumerate dependencies of a package item -------------------------------- *** 574 575 U_NAMESPACE_BEGIN 576 577 void 578 Package::enumDependencies(Item *pItem, void *context, CheckDependency check) { 579 int32_t infoLength, itemHeaderLength; 580 UErrorCode errorCode=U_ZERO_ERROR; 581 const UDataInfo *pInfo=getDataInfo(pItem->data, pItem->length, infoLength, itemHeaderLength, &errorCode); 582 if(U_FAILURE(errorCode)) { 583 return; // should not occur because readFile() checks headers 584 } 585 586 // find the data format and call the corresponding function, if any 587 int32_t format=getDataFormat(pInfo->dataFormat); 588 if(format>=0) { 589 switch(format) { 590 case FMT_RES: 591 { 592 /* 593 * Swap the resource bundle (if necessary) so that we can use 594 * the normal runtime uresdata.c code to read it. 595 * We do not want to duplicate that code, especially not together with on-the-fly swapping. 596 */ 597 NativeItem nrb(pItem, ures_swap); 598 ures_enumDependencies(pItem->name, nrb.getDataInfo(), nrb.getBytes(), nrb.getLength(), check, context, &errorCode); 599 break; 600 } 601 case FMT_CNV: 602 { 603 // TODO: share/cache swappers 604 UDataSwapper *ds=udata_openSwapper( 605 (UBool)pInfo->isBigEndian, pInfo->charsetFamily, 606 U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, 607 &errorCode); 608 if(U_FAILURE(errorCode)) { 609 fprintf(stderr, "icupkg: udata_openSwapper(\"%s\") failed - %s\n", 610 pItem->name, u_errorName(errorCode)); 611 exit(errorCode); 612 } 613 614 ds->printError=printError; 615 ds->printErrorContext=stderr; 616 617 const uint8_t *inBytes=pItem->data+itemHeaderLength; 618 int32_t length=pItem->length-itemHeaderLength; 619 620 ucnv_enumDependencies(ds, pItem->name, pInfo, inBytes, length, check, context, &errorCode); 621 udata_closeSwapper(ds); 622 break; 623 } 624 default: 625 break; 626 } 627 628 if(U_FAILURE(errorCode)) { 629 exit(errorCode); 630 } 631 } 632 } 633 634 U_NAMESPACE_END 635