1 /* 2 ******************************************************************************* 3 * Copyright (C) 2013-2014, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ******************************************************************************* 6 * collationdatareader.cpp 7 * 8 * created on: 2013feb07 9 * created by: Markus W. Scherer 10 */ 11 12 #include "unicode/utypes.h" 13 14 #if !UCONFIG_NO_COLLATION 15 16 #include "unicode/ucol.h" 17 #include "unicode/udata.h" 18 #include "unicode/uscript.h" 19 #include "cmemory.h" 20 #include "collation.h" 21 #include "collationdata.h" 22 #include "collationdatareader.h" 23 #include "collationfastlatin.h" 24 #include "collationkeys.h" 25 #include "collationrootelements.h" 26 #include "collationsettings.h" 27 #include "collationtailoring.h" 28 #include "normalizer2impl.h" 29 #include "uassert.h" 30 #include "ucmndata.h" 31 #include "utrie2.h" 32 33 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) 34 35 U_NAMESPACE_BEGIN 36 37 namespace { 38 39 int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) { 40 return (i < length) ? indexes[i] : -1; 41 } 42 43 } // namespace 44 45 void 46 CollationDataReader::read(const CollationTailoring *base, const uint8_t *inBytes, int32_t inLength, 47 CollationTailoring &tailoring, UErrorCode &errorCode) { 48 if(U_FAILURE(errorCode)) { return; } 49 if(base != NULL) { 50 if(inBytes == NULL || (0 <= inLength && inLength < 24)) { 51 errorCode = U_ILLEGAL_ARGUMENT_ERROR; 52 return; 53 } 54 const DataHeader *header = reinterpret_cast<const DataHeader *>(inBytes); 55 if(!(header->dataHeader.magic1 == 0xda && header->dataHeader.magic2 == 0x27 && 56 isAcceptable(tailoring.version, NULL, NULL, &header->info))) { 57 errorCode = U_INVALID_FORMAT_ERROR; 58 return; 59 } 60 if(base->getUCAVersion() != tailoring.getUCAVersion()) { 61 errorCode = U_COLLATOR_VERSION_MISMATCH; 62 return; 63 } 64 int32_t headerLength = header->dataHeader.headerSize; 65 inBytes += headerLength; 66 if(inLength >= 0) { 67 inLength -= headerLength; 68 } 69 } 70 71 if(inBytes == NULL || (0 <= inLength && inLength < 8)) { 72 errorCode = U_ILLEGAL_ARGUMENT_ERROR; 73 return; 74 } 75 const int32_t *inIndexes = reinterpret_cast<const int32_t *>(inBytes); 76 int32_t indexesLength = inIndexes[IX_INDEXES_LENGTH]; 77 if(indexesLength < 2 || (0 <= inLength && inLength < indexesLength * 4)) { 78 errorCode = U_INVALID_FORMAT_ERROR; // Not enough indexes. 79 return; 80 } 81 82 // Assume that the tailoring data is in initial state, 83 // with NULL pointers and 0 lengths. 84 85 // Set pointers to non-empty data parts. 86 // Do this in order of their byte offsets. (Should help porting to Java.) 87 88 int32_t index; // one of the indexes[] slots 89 int32_t offset; // byte offset for the index part 90 int32_t length; // number of bytes in the index part 91 92 if(indexesLength > IX_TOTAL_SIZE) { 93 length = inIndexes[IX_TOTAL_SIZE]; 94 } else if(indexesLength > IX_REORDER_CODES_OFFSET) { 95 length = inIndexes[indexesLength - 1]; 96 } else { 97 length = 0; // only indexes, and inLength was already checked for them 98 } 99 if(0 <= inLength && inLength < length) { 100 errorCode = U_INVALID_FORMAT_ERROR; 101 return; 102 } 103 104 const CollationData *baseData = base == NULL ? NULL : base->data; 105 const int32_t *reorderCodes = NULL; 106 int32_t reorderCodesLength = 0; 107 index = IX_REORDER_CODES_OFFSET; 108 offset = getIndex(inIndexes, indexesLength, index); 109 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 110 if(length >= 4) { 111 if(baseData == NULL) { 112 // We assume for collation settings that 113 // the base data does not have a reordering. 114 errorCode = U_INVALID_FORMAT_ERROR; 115 return; 116 } 117 reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset); 118 reorderCodesLength = length / 4; 119 } 120 121 // There should be a reorder table only if there are reorder codes. 122 // However, when there are reorder codes the reorder table may be omitted to reduce 123 // the data size. 124 const uint8_t *reorderTable = NULL; 125 index = IX_REORDER_TABLE_OFFSET; 126 offset = getIndex(inIndexes, indexesLength, index); 127 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 128 if(length >= 256) { 129 if(reorderCodesLength == 0) { 130 errorCode = U_INVALID_FORMAT_ERROR; // Reordering table without reordering codes. 131 return; 132 } 133 reorderTable = inBytes + offset; 134 } else { 135 // If we have reorder codes, then build the reorderTable at the end, 136 // when the CollationData is otherwise complete. 137 } 138 139 if(baseData != NULL && baseData->numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000)) { 140 errorCode = U_INVALID_FORMAT_ERROR; 141 return; 142 } 143 CollationData *data = NULL; // Remains NULL if there are no mappings. 144 145 index = IX_TRIE_OFFSET; 146 offset = getIndex(inIndexes, indexesLength, index); 147 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 148 if(length >= 8) { 149 if(!tailoring.ensureOwnedData(errorCode)) { return; } 150 data = tailoring.ownedData; 151 data->base = baseData; 152 data->numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000; 153 data->trie = tailoring.trie = utrie2_openFromSerialized( 154 UTRIE2_32_VALUE_BITS, inBytes + offset, length, NULL, 155 &errorCode); 156 if(U_FAILURE(errorCode)) { return; } 157 } else if(baseData != NULL) { 158 // Use the base data. Only the settings are tailored. 159 tailoring.data = baseData; 160 } else { 161 errorCode = U_INVALID_FORMAT_ERROR; // No mappings. 162 return; 163 } 164 165 index = IX_CES_OFFSET; 166 offset = getIndex(inIndexes, indexesLength, index); 167 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 168 if(length >= 8) { 169 if(data == NULL) { 170 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ces without tailored trie. 171 return; 172 } 173 data->ces = reinterpret_cast<const int64_t *>(inBytes + offset); 174 data->cesLength = length / 8; 175 } 176 177 index = IX_CE32S_OFFSET; 178 offset = getIndex(inIndexes, indexesLength, index); 179 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 180 if(length >= 4) { 181 if(data == NULL) { 182 errorCode = U_INVALID_FORMAT_ERROR; // Tailored ce32s without tailored trie. 183 return; 184 } 185 data->ce32s = reinterpret_cast<const uint32_t *>(inBytes + offset); 186 data->ce32sLength = length / 4; 187 } 188 189 int32_t jamoCE32sStart = getIndex(inIndexes, indexesLength, IX_JAMO_CE32S_START); 190 if(jamoCE32sStart >= 0) { 191 if(data == NULL || data->ce32s == NULL) { 192 errorCode = U_INVALID_FORMAT_ERROR; // Index into non-existent ce32s[]. 193 return; 194 } 195 data->jamoCE32s = data->ce32s + jamoCE32sStart; 196 } else if(data == NULL) { 197 // Nothing to do. 198 } else if(baseData != NULL) { 199 data->jamoCE32s = baseData->jamoCE32s; 200 } else { 201 errorCode = U_INVALID_FORMAT_ERROR; // No Jamo CE32s for Hangul processing. 202 return; 203 } 204 205 index = IX_ROOT_ELEMENTS_OFFSET; 206 offset = getIndex(inIndexes, indexesLength, index); 207 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 208 if(length >= 4) { 209 length /= 4; 210 if(data == NULL || length <= CollationRootElements::IX_SEC_TER_BOUNDARIES) { 211 errorCode = U_INVALID_FORMAT_ERROR; 212 return; 213 } 214 data->rootElements = reinterpret_cast<const uint32_t *>(inBytes + offset); 215 data->rootElementsLength = length; 216 uint32_t commonSecTer = data->rootElements[CollationRootElements::IX_COMMON_SEC_AND_TER_CE]; 217 if(commonSecTer != Collation::COMMON_SEC_AND_TER_CE) { 218 errorCode = U_INVALID_FORMAT_ERROR; 219 return; 220 } 221 uint32_t secTerBoundaries = data->rootElements[CollationRootElements::IX_SEC_TER_BOUNDARIES]; 222 if((secTerBoundaries >> 24) < CollationKeys::SEC_COMMON_HIGH) { 223 // [fixed last secondary common byte] is too low, 224 // and secondary weights would collide with compressed common secondaries. 225 errorCode = U_INVALID_FORMAT_ERROR; 226 return; 227 } 228 } 229 230 index = IX_CONTEXTS_OFFSET; 231 offset = getIndex(inIndexes, indexesLength, index); 232 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 233 if(length >= 2) { 234 if(data == NULL) { 235 errorCode = U_INVALID_FORMAT_ERROR; // Tailored contexts without tailored trie. 236 return; 237 } 238 data->contexts = reinterpret_cast<const UChar *>(inBytes + offset); 239 data->contextsLength = length / 2; 240 } 241 242 index = IX_UNSAFE_BWD_OFFSET; 243 offset = getIndex(inIndexes, indexesLength, index); 244 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 245 if(length >= 2) { 246 if(data == NULL) { 247 errorCode = U_INVALID_FORMAT_ERROR; 248 return; 249 } 250 if(baseData == NULL) { 251 // Create the unsafe-backward set for the root collator. 252 // Include all non-zero combining marks and trail surrogates. 253 // We do this at load time, rather than at build time, 254 // to simplify Unicode version bootstrapping: 255 // The root data builder only needs the new FractionalUCA.txt data, 256 // but it need not be built with a version of ICU already updated to 257 // the corresponding new Unicode Character Database. 258 // 259 // The following is an optimized version of 260 // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]"). 261 // It is faster and requires fewer code dependencies. 262 tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff); // trail surrogates 263 if(tailoring.unsafeBackwardSet == NULL) { 264 errorCode = U_MEMORY_ALLOCATION_ERROR; 265 return; 266 } 267 data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet); 268 } else { 269 // Clone the root collator's set contents. 270 tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>( 271 baseData->unsafeBackwardSet->cloneAsThawed()); 272 if(tailoring.unsafeBackwardSet == NULL) { 273 errorCode = U_MEMORY_ALLOCATION_ERROR; 274 return; 275 } 276 } 277 // Add the ranges from the data file to the unsafe-backward set. 278 USerializedSet sset; 279 const uint16_t *unsafeData = reinterpret_cast<const uint16_t *>(inBytes + offset); 280 if(!uset_getSerializedSet(&sset, unsafeData, length / 2)) { 281 errorCode = U_INVALID_FORMAT_ERROR; 282 return; 283 } 284 int32_t count = uset_getSerializedRangeCount(&sset); 285 for(int32_t i = 0; i < count; ++i) { 286 UChar32 start, end; 287 uset_getSerializedRange(&sset, i, &start, &end); 288 tailoring.unsafeBackwardSet->add(start, end); 289 } 290 // Mark each lead surrogate as "unsafe" 291 // if any of its 1024 associated supplementary code points is "unsafe". 292 UChar32 c = 0x10000; 293 for(UChar lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400) { 294 if(!tailoring.unsafeBackwardSet->containsNone(c, c + 0x3ff)) { 295 tailoring.unsafeBackwardSet->add(lead); 296 } 297 } 298 tailoring.unsafeBackwardSet->freeze(); 299 data->unsafeBackwardSet = tailoring.unsafeBackwardSet; 300 } else if(data == NULL) { 301 // Nothing to do. 302 } else if(baseData != NULL) { 303 // No tailoring-specific data: Alias the root collator's set. 304 data->unsafeBackwardSet = baseData->unsafeBackwardSet; 305 } else { 306 errorCode = U_INVALID_FORMAT_ERROR; // No unsafeBackwardSet. 307 return; 308 } 309 310 // If the fast Latin format version is different, 311 // or the version is set to 0 for "no fast Latin table", 312 // then just always use the normal string comparison path. 313 if(data != NULL) { 314 data->fastLatinTable = NULL; 315 data->fastLatinTableLength = 0; 316 if(((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin::VERSION) { 317 index = IX_FAST_LATIN_TABLE_OFFSET; 318 offset = getIndex(inIndexes, indexesLength, index); 319 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 320 if(length >= 2) { 321 data->fastLatinTable = reinterpret_cast<const uint16_t *>(inBytes + offset); 322 data->fastLatinTableLength = length / 2; 323 if((*data->fastLatinTable >> 8) != CollationFastLatin::VERSION) { 324 errorCode = U_INVALID_FORMAT_ERROR; // header vs. table version mismatch 325 return; 326 } 327 } else if(baseData != NULL) { 328 data->fastLatinTable = baseData->fastLatinTable; 329 data->fastLatinTableLength = baseData->fastLatinTableLength; 330 } 331 } 332 } 333 334 index = IX_SCRIPTS_OFFSET; 335 offset = getIndex(inIndexes, indexesLength, index); 336 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 337 if(length >= 2) { 338 if(data == NULL) { 339 errorCode = U_INVALID_FORMAT_ERROR; 340 return; 341 } 342 data->scripts = reinterpret_cast<const uint16_t *>(inBytes + offset); 343 data->scriptsLength = length / 2; 344 } else if(data == NULL) { 345 // Nothing to do. 346 } else if(baseData != NULL) { 347 data->scripts = baseData->scripts; 348 data->scriptsLength = baseData->scriptsLength; 349 } 350 351 index = IX_COMPRESSIBLE_BYTES_OFFSET; 352 offset = getIndex(inIndexes, indexesLength, index); 353 length = getIndex(inIndexes, indexesLength, index + 1) - offset; 354 if(length >= 256) { 355 if(data == NULL) { 356 errorCode = U_INVALID_FORMAT_ERROR; 357 return; 358 } 359 data->compressibleBytes = reinterpret_cast<const UBool *>(inBytes + offset); 360 } else if(data == NULL) { 361 // Nothing to do. 362 } else if(baseData != NULL) { 363 data->compressibleBytes = baseData->compressibleBytes; 364 } else { 365 errorCode = U_INVALID_FORMAT_ERROR; // No compressibleBytes[]. 366 return; 367 } 368 369 const CollationSettings &ts = *tailoring.settings; 370 int32_t options = inIndexes[IX_OPTIONS] & 0xffff; 371 uint16_t fastLatinPrimaries[CollationFastLatin::LATIN_LIMIT]; 372 int32_t fastLatinOptions = CollationFastLatin::getOptions( 373 tailoring.data, ts, fastLatinPrimaries, LENGTHOF(fastLatinPrimaries)); 374 if(options == ts.options && ts.variableTop != 0 && 375 reorderCodesLength == ts.reorderCodesLength && 376 uprv_memcmp(reorderCodes, ts.reorderCodes, reorderCodesLength * 4) == 0 && 377 fastLatinOptions == ts.fastLatinOptions && 378 (fastLatinOptions < 0 || 379 uprv_memcmp(fastLatinPrimaries, ts.fastLatinPrimaries, 380 sizeof(fastLatinPrimaries)) == 0)) { 381 return; 382 } 383 384 CollationSettings *settings = SharedObject::copyOnWrite(tailoring.settings); 385 if(settings == NULL) { 386 errorCode = U_MEMORY_ALLOCATION_ERROR; 387 return; 388 } 389 settings->options = options; 390 // Set variableTop from options and scripts data. 391 settings->variableTop = tailoring.data->getLastPrimaryForGroup( 392 UCOL_REORDER_CODE_FIRST + settings->getMaxVariable()); 393 if(settings->variableTop == 0) { 394 errorCode = U_INVALID_FORMAT_ERROR; 395 return; 396 } 397 398 if(reorderCodesLength == 0 || reorderTable != NULL) { 399 settings->aliasReordering(reorderCodes, reorderCodesLength, reorderTable); 400 } else { 401 uint8_t table[256]; 402 baseData->makeReorderTable(reorderCodes, reorderCodesLength, table, errorCode); 403 if(U_FAILURE(errorCode)) { return; } 404 if(!settings->setReordering(reorderCodes, reorderCodesLength,table)) { 405 errorCode = U_MEMORY_ALLOCATION_ERROR; 406 return; 407 } 408 } 409 410 settings->fastLatinOptions = CollationFastLatin::getOptions( 411 tailoring.data, *settings, 412 settings->fastLatinPrimaries, LENGTHOF(settings->fastLatinPrimaries)); 413 } 414 415 UBool U_CALLCONV 416 CollationDataReader::isAcceptable(void *context, 417 const char * /* type */, const char * /*name*/, 418 const UDataInfo *pInfo) { 419 if( 420 pInfo->size >= 20 && 421 pInfo->isBigEndian == U_IS_BIG_ENDIAN && 422 pInfo->charsetFamily == U_CHARSET_FAMILY && 423 pInfo->dataFormat[0] == 0x55 && // dataFormat="UCol" 424 pInfo->dataFormat[1] == 0x43 && 425 pInfo->dataFormat[2] == 0x6f && 426 pInfo->dataFormat[3] == 0x6c && 427 pInfo->formatVersion[0] == 4 428 ) { 429 UVersionInfo *version = static_cast<UVersionInfo *>(context); 430 if(version != NULL) { 431 uprv_memcpy(version, pInfo->dataVersion, 4); 432 } 433 return TRUE; 434 } else { 435 return FALSE; 436 } 437 } 438 439 U_NAMESPACE_END 440 441 #endif // !UCONFIG_NO_COLLATION 442
