1 /* 2 ******************************************************************************* 3 * Copyright (C) 2013-2015, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ******************************************************************************* 6 * collationdatareader.cpp 7 * 8 * created on: 2013feb07 9 * created by: Markus W. Scherer 10 */ 11 12 #include "unicode/utypes.h" 13 14 #if !UCONFIG_NO_COLLATION 15 16 #include "unicode/ucol.h" 17 #include "unicode/udata.h" 18 #include "unicode/uscript.h" 19 #include "cmemory.h" 20 #include "collation.h" 21 #include "collationdata.h" 22 #include "collationdatareader.h" 23 #include "collationfastlatin.h" 24 #include "collationkeys.h" 25 #include "collationrootelements.h" 26 #include "collationsettings.h" 27 #include "collationtailoring.h" 28 #include "normalizer2impl.h" 29 #include "uassert.h" 30 #include "ucmndata.h" 31 #include "utrie2.h" 32 33 U_NAMESPACE_BEGIN 34 35 namespace { 36 37 int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) { 38 return (i < length) ? indexes[i] : -1; 39 } 40 41 } // namespace 42 43 void 44 CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength, 45 CollationTailoring &tailoring, UErrorCode &errorCode) { 46 if(U_FAILURE(errorCode)) { return; } 47 if(base != NULL) { 48 if(inBytes == NULL || (0 <= inLength && inLength < 24)) { 49 errorCode = U_ILLEGAL_ARGUMENT_ERROR; 50 return; 51 } 52 const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes); 53 if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 && 54 isAcceptable(tailoring.version, NULL, NULL, &header->info))) { 55 errorCode = U_INVALID_FORMAT_ERROR; 56 return; 57 } 58 if(base->getUCAVersion() != tailoring.getUCAVersion()) { 59 errorCode = U_COLLATOR_VERSION_MISMATCH; 60 return; 61 } 62 int32_t headerLength = header->dataHeader.headerSize; 63 inBytes += headerLength; 64 if(inLength >= 0) { 65 inLength -= headerLength; 66 } 67 } 68 69 if(inBytes == NULL || (0 <= inLength && inLength < 8)) { 70 errorCode = U_ILLEGAL_ARGUMENT_ERROR; 71 return; 72 } 73 const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes); 74 int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH]; 75 if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) { 76 errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes. 77 return; 78 } 79 80 // Assume that the tailoring data is in initial state, 81 // with NULL pointers and 0 lengths. 82 83 // Set pointers to non-empty data parts. 84 // Do this in order of their byte offsets. (Should help porting to Java.) 85 86 int32_t index; // one of the indexes[] slots 87 int32_t offset; // byte offset for the index part 88 int32_t length; // number of bytes in the index part 89 90 if(indexesLength > IX_TOTAL_SIZE) { 91 length = inIndexes[IX_TOTAL_SIZE]; 92 } else if(indexesLength > IX_REORDER_CODES_OFFSET) { 93 length = inIndexes[indexesLength - 1]; 94 } else { 95 length = 0; // only indexes, and inLength was already checked for them 96 } 97 if(0 <= inLength && inLength < length) { 98 errorCode = U_INVALID_FORMAT_ERROR; 99 return; 100 } 101 102 const CollationData *baseData = base == NULL ? NULL : base->data; 103 const int32_t *reorderCodes = NULL; 104 int32_t reorderCodesLength = 0; 105 const uint32_t *reorderRanges = NULL; 106 int32_t reorderRangesLength = 0; 107 index = IX_REORDER_CODES_OFFSET; 108 offset = getIndex(inIndexes, indexesLength, index); 109 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 110 if(length >= 4) { 111 if(baseData == NULL) { 112 // We assume for collation settings that 113 // the base data does not have a reordering. 114 errorCode = U_INVALID_FORMAT_ERROR; 115 return; 116 } 117 reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset); 118 reorderCodesLength = length / 4; 119 120 // The reorderRanges (if any) are the trailing reorderCodes entries. 121 // Split the array at the boundary. 122 // Script or reorder codes do not exceed 16-bit values. 123 // Range limits are stored in the upper 16 bits, and are never 0. 124 while(reorderRangesLength < reorderCodesLength && 125 (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) { 126 ++reorderRangesLength; 127 } 128 U_ASSERT(reorderRangesLength < reorderCodesLength); 129 if(reorderRangesLength != 0) { 130 reorderCodesLength -= reorderRangesLength; 131 reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength); 132 } 133 } 134 135 // There should be a reorder table only if there are reorder codes. 136 // However, when there are reorder codes the reorder table may be omitted to reduce 137 // the data size. 138 const uint8_t *reorderTable = NULL; 139 index = IX_REORDER_TABLE_OFFSET; 140 offset = getIndex(inIndexes, indexesLength, index); 141 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 142 if(length >= 256) { 143 if(reorderCodesLength == 0) { 144 errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes. 145 return; 146 } 147 reorderTable = inBytes + offset; 148 } else { 149 // If we have reorder codes, then build the reorderTable at the end, 150 // when the CollationData is otherwise complete. 151 } 152 153 if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) { 154 errorCode = U_INVALID_FORMAT_ERROR; 155 return; 156 } 157 CollationData *data = NULL; // Remains NULL if there are no mappings. 158 159 index = IX_TRIE_OFFSET; 160 offset = getIndex(inIndexes, indexesLength, index); 161 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 162 if(length >= 8) { 163 if(!tailoring.ensureOwnedData(errorCode)) { return; } 164 data = tailoring.ownedData; 165 data->base = baseData; 166 data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000; 167 data->trie = tailoring.trie = utrie2_openFromSerialized( 168 UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL, 169 &errorCode); 170 if(U_FAILURE(errorCode)) { return; } 171 } else if(baseData != NULL) { 172 // Use the base data. Only the settings are tailored. 173 tailoring.data = baseData; 174 } else { 175 errorCode = U_INVALID_FORMAT_ERROR; // No mappings. 176 return; 177 } 178 179 index = IX_CES_OFFSET; 180 offset = getIndex(inIndexes, indexesLength, index); 181 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 182 if(length >= 8) { 183 if(data == NULL) { 184 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie. 185 return; 186 } 187 data->ces = reinterpret_cast<const int64_t *>(inBytes + offset); 188 data->cesLength = length / 8; 189 } 190 191 index = IX_CE32S_OFFSET; 192 offset = getIndex(inIndexes, indexesLength, index); 193 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 194 if(length >= 4) { 195 if(data == NULL) { 196 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie. 197 return; 198 } 199 data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset); 200 data->ce32sLength = length / 4; 201 } 202 203 int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START); 204 if(jamoCE32sStart >= 0) { 205 if(data == NULL || data->ce32s == NULL) { 206 errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[]. 207 return; 208 } 209 data->jamoCE32s = data->ce32s + jamoCE32sStart; 210 } else if(data == NULL) { 211 // Nothing to do. 212 } else if(baseData != NULL) { 213 data->jamoCE32s = baseData->jamoCE32s; 214 } else { 215 errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing. 216 return; 217 } 218 219 index = IX_ROOT_ELEMENTS_OFFSET; 220 offset = getIndex(inIndexes, indexesLength, index); 221 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 222 if(length >= 4) { 223 length /= 4; 224 if(data == NULL || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) { 225 errorCode = U_INVALID_FORMAT_ERROR; 226 return; 227 } 228 data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset); 229 data->rootElementsLength = length; 230 uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE]; 231 if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) { 232 errorCode = U_INVALID_FORMAT_ERROR; 233 return; 234 } 235 uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES]; 236 if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) { 237 // [fixed last secondary common byte] is too low, 238 // and secondary weights would collide with compressed common secondaries. 239 errorCode = U_INVALID_FORMAT_ERROR; 240 return; 241 } 242 } 243 244 index = IX_CONTEXTS_OFFSET; 245 offset = getIndex(inIndexes, indexesLength, index); 246 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 247 if(length >= 2) { 248 if(data == NULL) { 249 errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie. 250 return; 251 } 252 data->contexts = reinterpret_cast<const UChar *>(inBytes + offset); 253 data->contextsLength = length / 2; 254 } 255 256 index = IX_UNSAFE_BWD_OFFSET; 257 offset = getIndex(inIndexes, indexesLength, index); 258 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 259 if(length >= 2) { 260 if(data == NULL) { 261 errorCode = U_INVALID_FORMAT_ERROR; 262 return; 263 } 264 if(baseData == NULL) { 265 // Create the unsafe-backward set for the root collator. 266 // Include all non-zero combining marks and trail surrogates. 267 // We do this at load time, rather than at build time, 268 // to simplify Unicode version bootstrapping: 269 // The root data builder only needs the new FractionalUCA.txt data, 270 // but it need not be built with a version of ICU already updated to 271 // the corresponding new Unicode Character Database. 272 // 273 // The following is an optimized version of 274 // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]"). 275 // It is faster and requires fewer code dependencies. 276 tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates 277 if(tailoring.unsafeBackwardSet == NULL) { 278 errorCode = U_MEMORY_ALLOCATION_ERROR; 279 return; 280 } 281 data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet); 282 } else { 283 // Clone the root collator's set contents. 284 tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>( 285 baseData->unsafeBackwardSet->cloneAsThawed()); 286 if(tailoring.unsafeBackwardSet == NULL) { 287 errorCode = U_MEMORY_ALLOCATION_ERROR; 288 return; 289 } 290 } 291 // Add the ranges from the data file to the unsafe-backward set. 292 USerializedSet sset; 293 const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset); 294 if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) { 295 errorCode = U_INVALID_FORMAT_ERROR; 296 return; 297 } 298 int32_t count = uset_getSerializedRangeCount(&sset); 299 for(int32_t i = 0; i < count; ++i) { 300 UChar32 start, end; 301 uset_getSerializedRange(&sset, i, &start, &end); 302 tailoring.unsafeBackwardSet->add(start, end); 303 } 304 // Mark each lead surrogate as "unsafe" 305 // if any of its 1024 associated supplementary code points is "unsafe". 306 UChar32 c = 0x10000; 307 for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) { 308 if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) { 309 tailoring.unsafeBackwardSet->add(lead); 310 } 311 } 312 tailoring.unsafeBackwardSet->freeze(); 313 data->unsafeBackwardSet = tailoring.unsafeBackwardSet; 314 } else if(data == NULL) { 315 // Nothing to do. 316 } else if(baseData != NULL) { 317 // No tailoring-specific data: Alias the root collator's set. 318 data->unsafeBackwardSet = baseData->unsafeBackwardSet; 319 } else { 320 errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet. 321 return; 322 } 323 324 // If the fast Latin format version is different, 325 // or the version is set to 0 for "no fast Latin table", 326 // then just always use the normal string comparison path. 327 if(data != NULL) { 328 data->fastLatinTable = NULL; 329 data->fastLatinTableLength = 0; 330 if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) { 331 index = IX_FAST_LATIN_TABLE_OFFSET; 332 offset = getIndex(inIndexes, indexesLength, index); 333 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 334 if(length >= 2) { 335 data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset); 336 data->fastLatinTableLength = length / 2; 337 if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) { 338 errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch 339 return; 340 } 341 } else if(baseData != NULL) { 342 data->fastLatinTable = baseData->fastLatinTable; 343 data->fastLatinTableLength = baseData->fastLatinTableLength; 344 } 345 } 346 } 347 348 index = IX_SCRIPTS_OFFSET; 349 offset = getIndex(inIndexes, indexesLength, index); 350 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 351 if(length >= 2) { 352 if(data == NULL) { 353 errorCode = U_INVALID_FORMAT_ERROR; 354 return; 355 } 356 const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset); 357 int32_t scriptsLength = length / 2; 358 data->numScripts = scripts[0]; 359 // There must be enough entries for both arrays, including more than two range starts. 360 data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16); 361 if(data->scriptStartsLength <= 2 || 362 CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) { 363 errorCode = U_INVALID_FORMAT_ERROR; 364 return; 365 } 366 data->scriptsIndex = scripts + 1; 367 data->scriptStarts = scripts + 1 + data->numScripts + 16; 368 if(!(data->scriptStarts[0] == 0 && 369 data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) && 370 data->scriptStarts[data->scriptStartsLength - 1] == 371 (Collation::TRAIL_WEIGHT_BYTE << 8))) { 372 errorCode = U_INVALID_FORMAT_ERROR; 373 return; 374 } 375 } else if(data == NULL) { 376 // Nothing to do. 377 } else if(baseData != NULL) { 378 data->numScripts = baseData->numScripts; 379 data->scriptsIndex = baseData->scriptsIndex; 380 data->scriptStarts = baseData->scriptStarts; 381 data->scriptStartsLength = baseData->scriptStartsLength; 382 } 383 384 index = IX_COMPRESSIBLE_BYTES_OFFSET; 385 offset = getIndex(inIndexes, indexesLength, index); 386 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 387 if(length >= 256) { 388 if(data == NULL) { 389 errorCode = U_INVALID_FORMAT_ERROR; 390 return; 391 } 392 data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset); 393 } else if(data == NULL) { 394 // Nothing to do. 395 } else if(baseData != NULL) { 396 data->compressibleBytes = baseData->compressibleBytes; 397 } else { 398 errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[]. 399 return; 400 } 401 402 const CollationSettings &ts = *tailoring.settings; 403 int32_t options = inIndexes[IX_OPTIONS] & 0xffff; 404 uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT]; 405 int32_t fastLatinOptions = CollationFastLatin::getOptions( 406 tailoring.data, ts, fastLatinPrimaries, UPRV_LENGTHOF(fastLatinPrimaries)); 407 if(options == ts.options && ts.variableTop != 0 && 408 reorderCodesLength == ts.reorderCodesLength && 409 uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0 && 410 fastLatinOptions == ts.fastLatinOptions && 411 (fastLatinOptions < 0 || 412 uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries, 413 sizeof(fastLatinPrimaries)) == 0)) { 414 return; 415 } 416 417 CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings); 418 if(settings == NULL) { 419 errorCode = U_MEMORY_ALLOCATION_ERROR; 420 return; 421 } 422 settings->options = options; 423 // Set variableTop from options and scripts data. 424 settings->variableTop = tailoring.data->getLastPrimaryForGroup( 425 UCOL_REORDER_CODE_FIRST + settings->getMaxVariable()); 426 if(settings->variableTop == 0) { 427 errorCode = U_INVALID_FORMAT_ERROR; 428 return; 429 } 430 431 if(reorderCodesLength != 0) { 432 settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength, 433 reorderRanges, reorderRangesLength, 434 reorderTable, errorCode); 435 } 436 437 settings->fastLatinOptions = CollationFastLatin::getOptions( 438 tailoring.data, *settings, 439 settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries)); 440 } 441 442 UBool U_CALLCONV 443 CollationDataReader::isAcceptable(void *context, 444 const char * /* type */, const char * /*name*/, 445 const UDataInfo *pInfo) { 446 if( 447 pInfo->size >= 20 && 448 pInfo->isBigEndian == U_IS_BIG_ENDIAN && 449 pInfo->charsetFamily == U_CHARSET_FAMILY && 450 pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol" 451 pInfo->dataFormat[1] == 0x43 && 452 pInfo->dataFormat[2] == 0x6f && 453 pInfo->dataFormat[3] == 0x6c && 454 pInfo->formatVersion[0] == 5 455 ) { 456 UVersionInfo *version = static_cast<UVersionInfo *>(context); 457 if(version != NULL) { 458 uprv_memcpy(version, pInfo->dataVersion, 4); 459 } 460 return TRUE; 461 } else { 462 return FALSE; 463 } 464 } 465 466 U_NAMESPACE_END 467 468 #endif // !UCONFIG_NO_COLLATION 469
