1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ****************************************************************************** 5 * 6 * Copyright (C) 1999-2011, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************/ 10 11 12 /*------------------------------------------------------------------------------ 13 * 14 * UCommonData An abstract interface for dealing with ICU Common Data Files. 15 * ICU Common Data Files are a grouping of a number of individual 16 * data items (resources, converters, tables, anything) into a 17 * single file or dll. The combined format includes a table of 18 * contents for locating the individual items by name. 19 * 20 * Two formats for the table of contents are supported, which is 21 * why there is an abstract inteface involved. 22 * 23 */ 24 25 #include "unicode/utypes.h" 26 #include "unicode/udata.h" 27 #include "cstring.h" 28 #include "ucmndata.h" 29 #include "udatamem.h" 30 31 #if defined(UDATA_DEBUG) || defined(UDATA_DEBUG_DUMP) 32 # include <stdio.h> 33 #endif 34 35 U_CFUNC uint16_t 36 udata_getHeaderSize(const DataHeader *udh) { 37 if(udh==NULL) { 38 return 0; 39 } else if(udh->info.isBigEndian==U_IS_BIG_ENDIAN) { 40 /* same endianness */ 41 return udh->dataHeader.headerSize; 42 } else { 43 /* opposite endianness */ 44 uint16_t x=udh->dataHeader.headerSize; 45 return (uint16_t)((x<<8)|(x>>8)); 46 } 47 } 48 49 U_CFUNC uint16_t 50 udata_getInfoSize(const UDataInfo *info) { 51 if(info==NULL) { 52 return 0; 53 } else if(info->isBigEndian==U_IS_BIG_ENDIAN) { 54 /* same endianness */ 55 return info->size; 56 } else { 57 /* opposite endianness */ 58 uint16_t x=info->size; 59 return (uint16_t)((x<<8)|(x>>8)); 60 } 61 } 62 63 /*-----------------------------------------------------------------------------* 64 * * 65 * Pointer TOCs. TODO: This form of table-of-contents should be removed * 66 * because DLLs must be relocated on loading to correct the * 67 * pointer values and this operation makes shared memory * 68 * mapping of the data much less likely to work. * 69 * * 70 *-----------------------------------------------------------------------------*/ 71 typedef struct { 72 const char *entryName; 73 const DataHeader *pHeader; 74 } PointerTOCEntry; 75 76 77 typedef struct { 78 uint32_t count; 79 uint32_t reserved; 80 PointerTOCEntry entry[2]; /* Actual size is from count. */ 81 } PointerTOC; 82 83 84 /* definition of OffsetTOC struct types moved to ucmndata.h */ 85 86 /*-----------------------------------------------------------------------------* 87 * * 88 * entry point lookup implementations * 89 * * 90 *-----------------------------------------------------------------------------*/ 91 92 #ifndef MIN 93 #define MIN(a,b) (((a)<(b)) ? (a) : (b)) 94 #endif 95 96 /** 97 * Compare strings where we know the shared prefix length, 98 * and advance the prefix length as we find that the strings share even more characters. 99 */ 100 static int32_t 101 strcmpAfterPrefix(const char *s1, const char *s2, int32_t *pPrefixLength) { 102 int32_t pl=*pPrefixLength; 103 int32_t cmp=0; 104 s1+=pl; 105 s2+=pl; 106 for(;;) { 107 int32_t c1=(uint8_t)*s1++; 108 int32_t c2=(uint8_t)*s2++; 109 cmp=c1-c2; 110 if(cmp!=0 || c1==0) { /* different or done */ 111 break; 112 } 113 ++pl; /* increment shared same-prefix length */ 114 } 115 *pPrefixLength=pl; 116 return cmp; 117 } 118 119 static int32_t 120 offsetTOCPrefixBinarySearch(const char *s, const char *names, 121 const UDataOffsetTOCEntry *toc, int32_t count) { 122 int32_t start=0; 123 int32_t limit=count; 124 /* 125 * Remember the shared prefix between s, start and limit, 126 * and don't compare that shared prefix again. 127 * The shared prefix should get longer as we narrow the [start, limit[ range. 128 */ 129 int32_t startPrefixLength=0; 130 int32_t limitPrefixLength=0; 131 if(count==0) { 132 return -1; 133 } 134 /* 135 * Prime the prefix lengths so that we don't keep prefixLength at 0 until 136 * both the start and limit indexes have moved. 137 * At the same time, we find if s is one of the start and (limit-1) names, 138 * and if not, exclude them from the actual binary search. 139 */ 140 if(0==strcmpAfterPrefix(s, names+toc[0].nameOffset, &startPrefixLength)) { 141 return 0; 142 } 143 ++start; 144 --limit; 145 if(0==strcmpAfterPrefix(s, names+toc[limit].nameOffset, &limitPrefixLength)) { 146 return limit; 147 } 148 while(start<limit) { 149 int32_t i=(start+limit)/2; 150 int32_t prefixLength=MIN(startPrefixLength, limitPrefixLength); 151 int32_t cmp=strcmpAfterPrefix(s, names+toc[i].nameOffset, &prefixLength); 152 if(cmp<0) { 153 limit=i; 154 limitPrefixLength=prefixLength; 155 } else if(cmp==0) { 156 return i; 157 } else { 158 start=i+1; 159 startPrefixLength=prefixLength; 160 } 161 } 162 return -1; 163 } 164 165 static int32_t 166 pointerTOCPrefixBinarySearch(const char *s, const PointerTOCEntry *toc, int32_t count) { 167 int32_t start=0; 168 int32_t limit=count; 169 /* 170 * Remember the shared prefix between s, start and limit, 171 * and don't compare that shared prefix again. 172 * The shared prefix should get longer as we narrow the [start, limit[ range. 173 */ 174 int32_t startPrefixLength=0; 175 int32_t limitPrefixLength=0; 176 if(count==0) { 177 return -1; 178 } 179 /* 180 * Prime the prefix lengths so that we don't keep prefixLength at 0 until 181 * both the start and limit indexes have moved. 182 * At the same time, we find if s is one of the start and (limit-1) names, 183 * and if not, exclude them from the actual binary search. 184 */ 185 if(0==strcmpAfterPrefix(s, toc[0].entryName, &startPrefixLength)) { 186 return 0; 187 } 188 ++start; 189 --limit; 190 if(0==strcmpAfterPrefix(s, toc[limit].entryName, &limitPrefixLength)) { 191 return limit; 192 } 193 while(start<limit) { 194 int32_t i=(start+limit)/2; 195 int32_t prefixLength=MIN(startPrefixLength, limitPrefixLength); 196 int32_t cmp=strcmpAfterPrefix(s, toc[i].entryName, &prefixLength); 197 if(cmp<0) { 198 limit=i; 199 limitPrefixLength=prefixLength; 200 } else if(cmp==0) { 201 return i; 202 } else { 203 start=i+1; 204 startPrefixLength=prefixLength; 205 } 206 } 207 return -1; 208 } 209 210 static uint32_t offsetTOCEntryCount(const UDataMemory *pData) { 211 int32_t retVal=0; 212 const UDataOffsetTOC *toc = (UDataOffsetTOC *)pData->toc; 213 if (toc != NULL) { 214 retVal = toc->count; 215 } 216 return retVal; 217 } 218 219 static const DataHeader * 220 offsetTOCLookupFn(const UDataMemory *pData, 221 const char *tocEntryName, 222 int32_t *pLength, 223 UErrorCode *pErrorCode) { 224 const UDataOffsetTOC *toc = (UDataOffsetTOC *)pData->toc; 225 if(toc!=NULL) { 226 const char *base=(const char *)toc; 227 int32_t number, count=(int32_t)toc->count; 228 229 /* perform a binary search for the data in the common data's table of contents */ 230 #if defined (UDATA_DEBUG_DUMP) 231 /* list the contents of the TOC each time .. not recommended */ 232 for(number=0; number<count; ++number) { 233 fprintf(stderr, "\tx%d: %s\n", number, &base[toc->entry[number].nameOffset]); 234 } 235 #endif 236 number=offsetTOCPrefixBinarySearch(tocEntryName, base, toc->entry, count); 237 if(number>=0) { 238 /* found it */ 239 const UDataOffsetTOCEntry *entry=toc->entry+number; 240 #ifdef UDATA_DEBUG 241 fprintf(stderr, "%s: Found.\n", tocEntryName); 242 #endif 243 if((number+1) < count) { 244 *pLength = (int32_t)(entry[1].dataOffset - entry->dataOffset); 245 } else { 246 *pLength = -1; 247 } 248 return (const DataHeader *)(base+entry->dataOffset); 249 } else { 250 #ifdef UDATA_DEBUG 251 fprintf(stderr, "%s: Not found.\n", tocEntryName); 252 #endif 253 return NULL; 254 } 255 } else { 256 #ifdef UDATA_DEBUG 257 fprintf(stderr, "returning header\n"); 258 #endif 259 260 return pData->pHeader; 261 } 262 } 263 264 265 static uint32_t pointerTOCEntryCount(const UDataMemory *pData) { 266 const PointerTOC *toc = (PointerTOC *)pData->toc; 267 return (uint32_t)((toc != NULL) ? (toc->count) : 0); 268 } 269 270 271 static const DataHeader *pointerTOCLookupFn(const UDataMemory *pData, 272 const char *name, 273 int32_t *pLength, 274 UErrorCode *pErrorCode) { 275 if(pData->toc!=NULL) { 276 const PointerTOC *toc = (PointerTOC *)pData->toc; 277 int32_t number, count=(int32_t)toc->count; 278 279 #if defined (UDATA_DEBUG_DUMP) 280 /* list the contents of the TOC each time .. not recommended */ 281 for(number=0; number<count; ++number) { 282 fprintf(stderr, "\tx%d: %s\n", number, toc->entry[number].entryName); 283 } 284 #endif 285 number=pointerTOCPrefixBinarySearch(name, toc->entry, count); 286 if(number>=0) { 287 /* found it */ 288 #ifdef UDATA_DEBUG 289 fprintf(stderr, "%s: Found.\n", toc->entry[number].entryName); 290 #endif 291 *pLength=-1; 292 return UDataMemory_normalizeDataPointer(toc->entry[number].pHeader); 293 } else { 294 #ifdef UDATA_DEBUG 295 fprintf(stderr, "%s: Not found.\n", name); 296 #endif 297 return NULL; 298 } 299 } else { 300 return pData->pHeader; 301 } 302 } 303 304 static const commonDataFuncs CmnDFuncs = {offsetTOCLookupFn, offsetTOCEntryCount}; 305 static const commonDataFuncs ToCPFuncs = {pointerTOCLookupFn, pointerTOCEntryCount}; 306 307 308 309 /*----------------------------------------------------------------------* 310 * * 311 * checkCommonData Validate the format of a common data file. * 312 * Fill in the virtual function ptr based on TOC type * 313 * If the data is invalid, close the UDataMemory * 314 * and set the appropriate error code. * 315 * * 316 *----------------------------------------------------------------------*/ 317 U_CFUNC void udata_checkCommonData(UDataMemory *udm, UErrorCode *err) { 318 if (U_FAILURE(*err)) { 319 return; 320 } 321 322 if(udm==NULL || udm->pHeader==NULL) { 323 *err=U_INVALID_FORMAT_ERROR; 324 } else if(!(udm->pHeader->dataHeader.magic1==0xda && 325 udm->pHeader->dataHeader.magic2==0x27 && 326 udm->pHeader->info.isBigEndian==U_IS_BIG_ENDIAN && 327 udm->pHeader->info.charsetFamily==U_CHARSET_FAMILY) 328 ) { 329 /* header not valid */ 330 *err=U_INVALID_FORMAT_ERROR; 331 } 332 else if (udm->pHeader->info.dataFormat[0]==0x43 && 333 udm->pHeader->info.dataFormat[1]==0x6d && 334 udm->pHeader->info.dataFormat[2]==0x6e && 335 udm->pHeader->info.dataFormat[3]==0x44 && 336 udm->pHeader->info.formatVersion[0]==1 337 ) { 338 /* dataFormat="CmnD" */ 339 udm->vFuncs = &CmnDFuncs; 340 udm->toc=(const char *)udm->pHeader+udata_getHeaderSize(udm->pHeader); 341 } 342 else if(udm->pHeader->info.dataFormat[0]==0x54 && 343 udm->pHeader->info.dataFormat[1]==0x6f && 344 udm->pHeader->info.dataFormat[2]==0x43 && 345 udm->pHeader->info.dataFormat[3]==0x50 && 346 udm->pHeader->info.formatVersion[0]==1 347 ) { 348 /* dataFormat="ToCP" */ 349 udm->vFuncs = &ToCPFuncs; 350 udm->toc=(const char *)udm->pHeader+udata_getHeaderSize(udm->pHeader); 351 } 352 else { 353 /* dataFormat not recognized */ 354 *err=U_INVALID_FORMAT_ERROR; 355 } 356 357 if (U_FAILURE(*err)) { 358 /* If the data is no good and we memory-mapped it ourselves, 359 * close the memory mapping so it doesn't leak. Note that this has 360 * no effect on non-memory mapped data, other than clearing fields in udm. 361 */ 362 udata_close(udm); 363 } 364 } 365 366 /* 367 * TODO: Add a udata_swapPackageHeader() function that swaps an ICU .dat package 368 * header but not its sub-items. 369 * This function will be needed for automatic runtime swapping. 370 * Sub-items should not be swapped to limit the swapping to the parts of the 371 * package that are actually used. 372 * 373 * Since lengths of items are implicit in the order and offsets of their 374 * ToC entries, and since offsets are relative to the start of the ToC, 375 * a swapped version may need to generate a different data structure 376 * with pointers to the original data items and with their lengths 377 * (-1 for the last one if it is not known), and maybe even pointers to the 378 * swapped versions of the items. 379 * These pointers to swapped versions would establish a cache; 380 * instead, each open data item could simply own the storage for its swapped 381 * data. This fits better with the current design. 382 * 383 * markus 2003sep18 Jitterbug 2235 384 */ 385