1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 1998-2009, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * 9 * Private implementation header for C collation 10 * file name: ucol_imp.h 11 * encoding: US-ASCII 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 2000dec11 16 * created by: Vladimir Weinstein 17 * 18 * Modification history 19 * Date Name Comments 20 * 02/16/2001 synwee Added UCOL_GETPREVCE for the use in ucoleitr 21 * 02/27/2001 synwee Added getMaxExpansion data structure in UCollator 22 * 03/02/2001 synwee Added UCOL_IMPLICIT_CE 23 * 03/12/2001 synwee Added pointer start to collIterate. 24 */ 25 26 #ifndef UCOL_IMP_H 27 #define UCOL_IMP_H 28 29 #include "unicode/utypes.h" 30 31 #define UCA_DATA_TYPE "icu" 32 #define UCA_DATA_NAME "ucadata" 33 #define INVC_DATA_TYPE "icu" 34 #define INVC_DATA_NAME "invuca" 35 36 /** 37 * Convenience string denoting the Collation data tree 38 * @internal ICU 3.0 39 */ 40 #define U_ICUDATA_COLL U_ICUDATA_NAME U_TREE_SEPARATOR_STRING "coll" 41 42 #if !UCONFIG_NO_COLLATION 43 44 #include "unicode/ucol.h" 45 #include "utrie.h" 46 #include "cmemory.h" 47 48 /* This is the internal header file which contains important declarations for 49 * the collation framework. 50 * Ready to use collators are stored as binary images. Both UCA and tailorings 51 * share the same binary format. Individual files (currently only UCA) have a 52 * udata header in front of the image and should be opened using udata_open. 53 * Tailoring images are currently stored inside resource bundles and are intialized 54 * through ucol_open API. 55 * 56 * The following describes the formats for collation binaries 57 * (UCA & tailorings) and for the inverse UCA table. 58 * Substructures are described in the collation design document at 59 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm 60 * 61 * ------------------------------------------------------------- 62 * 63 * Here is the format of binary collation image. 64 * 65 * Physical order of structures: 66 * - header (UCATableHeader) 67 * - options (UColOptionSet) 68 * - expansions (CE[]) 69 * - contractions (UChar[contractionSize] + CE[contractionSize]) 70 * - serialized UTrie with mappings of code points to CEs 71 * - max expansion tables (CE[endExpansionCECount] + uint8_t[endExpansionCECount]) 72 * - two bit sets for backward processing in strcoll (identical prefixes) 73 * and for backward CE iteration (each set is uint8_t[UCOL_UNSAFECP_TABLE_SIZE]) 74 * - UCA constants (UCAConstants) 75 * - UCA contractions (UChar[contractionUCACombosSize][contractionUCACombosWidth]) 76 * 77 * UCATableHeader fields: 78 * 79 * int32_t size; - image size in bytes 80 * 81 * Offsets to interesting data. All offsets are in bytes. 82 * to get the address add to the header address and cast properly. 83 * Some offsets are zero if the corresponding structures are empty. 84 * 85 * Tailoring binaries that only set options and contain no mappings etc. 86 * will have all offsets 0 except for the options and expansion offsets, 87 * which give the position and length of the options array. 88 * 89 * uint32_t options; - offset to default collator options (UColOptionSet *), 90 * a set of 32-bit values. See declaration of UColOptionSet for more details 91 * 92 * uint32_t UCAConsts; - only used (!=0) in UCA image - structure which holds values for indirect positioning and implicit ranges 93 * See declaration of UCAConstants structure. This is a set of unsigned 32-bit values used to store 94 * important constant values that are defined in the UCA and used for building and runtime. 95 * 96 * uint32_t contractionUCACombos; - only used (!=0) in UCA image - list of UCA contractions. This is a zero terminated array of UChar[contractionUCACombosWidth], 97 * containing contractions from the UCA. These are needed in the build process to copy UCA contractions 98 * in case the base contraction symbol is tailored. 99 * 100 * uint32_t magic; - must contain UCOL_HEADER_MAGIC (formatVersion 2.3) 101 * 102 * uint32_t mappingPosition; - offset to UTrie (const uint8_t *mappingPosition). This is a serialized UTrie and should be treated as such. 103 * Used as a primary lookup table for collation elements. 104 * 105 * uint32_t expansion; - offset to expansion table (uint32_t *expansion). This is an array of expansion CEs. Never 0. 106 * 107 * uint32_t contractionIndex; - offset to contraction table (UChar *contractionIndex). Used to look up contraction sequences. Contents 108 * are aligned with the contents of contractionCEs table. 0 if no contractions. 109 * 110 * uint32_t contractionCEs; - offset to resulting contraction CEs (uint32_t *contractionCEs). When a contraction is resolved in the 111 * in the contractionIndex table, the resulting index is used to look up corresponding CE in this table. 112 * 0 if no contractions. 113 * uint32_t contractionSize; - size of contraction table in elements (both Index and CEs). 114 * 115 * Tables described below are used for Boyer-Moore searching algorithm - they define the size of longest expansion 116 * and last CEs in expansions. 117 * uint32_t endExpansionCE; - offset to array of last collation element in expansion (uint32_t *). 118 * Never 0. 119 * uint32_t expansionCESize; - array of maximum expansion sizes (uint8_t *) 120 * int32_t endExpansionCECount; - size of endExpansionCE. See UCOL_GETMAXEXPANSION 121 * for the usage model 122 * 123 * These two offsets point to byte tables that are used in the backup heuristics. 124 * uint32_t unsafeCP; - hash table of unsafe code points (uint8_t *). See ucol_unsafeCP function. 125 * uint32_t contrEndCP; - hash table of final code points in contractions (uint8_t *). See ucol_contractionEndCP. 126 * 127 * int32_t contractionUCACombosSize; - number of UChar[contractionUCACombosWidth] in contractionUCACombos 128 * (formatVersion 2.3) 129 * UBool jamoSpecial; - Jamo special indicator (uint8_t). If TRUE, Jamos are special, so we cannot use simple Hangul decomposition. 130 * UBool isBigEndian; - endianness of this collation binary (formatVersion 2.3) 131 * uint8_t charSetFamily; - charset family of this collation binary (formatVersion 2.3) 132 * uint8_t contractionUCACombosWidth; - number of UChars per UCA contraction in contractionUCACombos (formatVersion 2.3) 133 * 134 * Various version fields 135 * UVersionInfo version; - version 4 uint8_t 136 * UVersionInfo UCAVersion; - version 4 uint8_t 137 * UVersionInfo UCDVersion; - version 4 uint8_t 138 * UVersionInfo formatVersion; - version of the format of the collation binary 139 * same formatVersion as in ucadata.icu's UDataInfo header 140 * (formatVersion 2.3) 141 * 142 * uint8_t reserved[84]; - currently unused 143 * 144 * ------------------------------------------------------------- 145 * 146 * Inverse UCA is used for constructing collators from rules. It is always an individual file 147 * and always has a UDataInfo header. 148 * here is the structure: 149 * 150 * uint32_t byteSize; - size of inverse UCA image in bytes 151 * uint32_t tableSize; - length of inverse table (number of uint32_t[3] rows) 152 * uint32_t contsSize; - size of continuation table (number of UChars in table) 153 * 154 * uint32_t table; - offset to inverse table (uint32_t *) 155 * Inverse table contains of rows of 3 uint32_t values. First two values are CE and a possible continuation 156 * the third value is either a code unit (if there is only one code unit for element) or an index to continuation 157 * (number of code units combined with an index). 158 * table. If more than one codepoint have the same CE, continuation table contains code units separated by FFFF and final 159 * code unit sequence for a CE is terminated by FFFE. 160 * uint32_t conts; - offset to continuation table (uint16_t *). Contains code units that transform to a same CE. 161 * 162 * UVersionInfo UCAVersion; - version of the UCA, read from file 4 uint8_t 163 * uint8_t padding[8]; - padding 8 uint8_t 164 * Header is followed by the table and continuation table. 165 */ 166 167 /* let us know whether reserved fields are reset to zero or junked */ 168 #define UCOL_HEADER_MAGIC 0x20030618 169 170 /* UDataInfo for UCA mapping table */ 171 /* dataFormat="UCol" */ 172 #define UCA_DATA_FORMAT_0 ((uint8_t)0x55) 173 #define UCA_DATA_FORMAT_1 ((uint8_t)0x43) 174 #define UCA_DATA_FORMAT_2 ((uint8_t)0x6f) 175 #define UCA_DATA_FORMAT_3 ((uint8_t)0x6c) 176 177 #define UCA_FORMAT_VERSION_0 ((uint8_t)2) 178 #define UCA_FORMAT_VERSION_1 ((uint8_t)3) 179 #define UCA_FORMAT_VERSION_2 ((uint8_t)0) 180 #define UCA_FORMAT_VERSION_3 ((uint8_t)0) 181 182 /* UDataInfo for inverse UCA table */ 183 /* dataFormat="InvC" */ 184 #define INVUCA_DATA_FORMAT_0 ((uint8_t)0x49) 185 #define INVUCA_DATA_FORMAT_1 ((uint8_t)0x6E) 186 #define INVUCA_DATA_FORMAT_2 ((uint8_t)0x76) 187 #define INVUCA_DATA_FORMAT_3 ((uint8_t)0x43) 188 189 #define INVUCA_FORMAT_VERSION_0 ((uint8_t)2) 190 #define INVUCA_FORMAT_VERSION_1 ((uint8_t)1) 191 #define INVUCA_FORMAT_VERSION_2 ((uint8_t)0) 192 #define INVUCA_FORMAT_VERSION_3 ((uint8_t)0) 193 194 /* This is the size of the stack allocated buffer for sortkey generation and similar operations */ 195 /* if it is too small, heap allocation will occur.*/ 196 /* you can change this value if you need memory - it will affect the performance, though, since we're going to malloc */ 197 #define UCOL_MAX_BUFFER 128 198 #define UCOL_PRIMARY_MAX_BUFFER 8*UCOL_MAX_BUFFER 199 #define UCOL_SECONDARY_MAX_BUFFER UCOL_MAX_BUFFER 200 #define UCOL_TERTIARY_MAX_BUFFER UCOL_MAX_BUFFER 201 #define UCOL_CASE_MAX_BUFFER UCOL_MAX_BUFFER/4 202 #define UCOL_QUAD_MAX_BUFFER 2*UCOL_MAX_BUFFER 203 204 #define UCOL_NORMALIZATION_GROWTH 2 205 #define UCOL_NORMALIZATION_MAX_BUFFER UCOL_MAX_BUFFER*UCOL_NORMALIZATION_GROWTH 206 207 /* This writable buffer is used if we encounter Thai and need to reorder the string on the fly */ 208 /* Sometimes we already have a writable buffer (like in case of normalized strings). */ 209 /* 210 you can change this value to any value >= 4 if you need memory - 211 it will affect the performance, though, since we're going to malloc. 212 Note 3 is the minimum value for Thai collation and 4 is the 213 minimum number for special Jamo 214 */ 215 #define UCOL_WRITABLE_BUFFER_SIZE 256 216 217 /* This is the size of the buffer for expansion CE's */ 218 /* In reality we should not have to deal with expm sequences longer then 16 */ 219 /* you can change this value if you need memory */ 220 /* WARNING THIS BUFFER DOES HAVE MALLOC FALLBACK. If you make it too small, you'll get into performance trouble */ 221 /* Reasonable small value is around 10, if you don't do Arabic or other funky collations that have long expansion sequence */ 222 /* This is the longest expansion sequence we can handle without bombing out */ 223 #define UCOL_EXPAND_CE_BUFFER_SIZE 64 224 225 /* This is the size to increase the buffer for expansion CE's */ 226 #define UCOL_EXPAND_CE_BUFFER_EXTEND_SIZE 64 227 228 229 /* Unsafe UChar hash table table size. */ 230 /* size is 32 bytes for 1 bit for each latin 1 char + some power of two for */ 231 /* hashing the rest of the chars. Size in bytes */ 232 #define UCOL_UNSAFECP_TABLE_SIZE 1056 233 /* mask value down to "some power of two"-1 */ 234 /* number of bits, not num of bytes. */ 235 #define UCOL_UNSAFECP_TABLE_MASK 0x1fff 236 237 238 /* flags bits for collIterate.flags */ 239 /* */ 240 /* NORM - set for incremental normalize of source string */ 241 #define UCOL_ITER_NORM 1 242 243 #define UCOL_ITER_HASLEN 2 244 245 /* UCOL_ITER_INNORMBUF - set if the "pos" is in */ 246 /* the writable side buffer, handling */ 247 /* incrementally normalized characters. */ 248 #define UCOL_ITER_INNORMBUF 4 249 250 /* UCOL_ITER_ALLOCATED - set if this iterator has */ 251 /* malloced storage to expand a buffer. */ 252 #define UCOL_ITER_ALLOCATED 8 253 /* UCOL_HIRAGANA_Q - note if the codepoint was hiragana */ 254 #define UCOL_HIRAGANA_Q 16 255 /* UCOL_WAS_HIRAGANA - set to TRUE if there was a Hiragana */ 256 /* otherwise set to false */ 257 #define UCOL_WAS_HIRAGANA 32 258 /* UCOL_USE_ITERATOR - set this if collIterate uses a */ 259 /* character iterator instead of simply accessing string */ 260 /* by index */ 261 #define UCOL_USE_ITERATOR 64 262 263 #define UCOL_FORCE_HAN_IMPLICIT 128 264 265 #define NFC_ZERO_CC_BLOCK_LIMIT_ 0x300 266 267 typedef struct collIterate { 268 UChar *string; /* Original string */ 269 /* UChar *start; Pointer to the start of the source string. Either points to string 270 or to writableBuffer */ 271 UChar *endp; /* string end ptr. Is undefined for null terminated strings */ 272 UChar *pos; /* This is position in the string. Can be to original or writable buf */ 273 274 uint32_t *toReturn; /* This is the CE from CEs buffer that should be returned */ 275 uint32_t *CEpos; /* This is the position to which we have stored processed CEs */ 276 277 int32_t *offsetReturn; /* This is the offset to return, if non-NULL */ 278 int32_t *offsetStore; /* This is the pointer for storing offsets */ 279 int32_t offsetRepeatCount; /* Repeat stored offset if non-zero */ 280 int32_t offsetRepeatValue; /* offset value to repeat */ 281 282 UChar *writableBuffer; 283 uint32_t writableBufSize; 284 UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ 285 const UCollator *coll; 286 uint8_t flags; 287 uint8_t origFlags; 288 uint32_t *extendCEs; /* This is use if CEs is not big enough */ 289 int32_t extendCEsSize; /* Holds the size of the dynamic CEs buffer */ 290 uint32_t CEs[UCOL_EXPAND_CE_BUFFER_SIZE]; /* This is where we store CEs */ 291 UChar stackWritableBuffer[UCOL_WRITABLE_BUFFER_SIZE]; /* A writable buffer. */ 292 293 int32_t *offsetBuffer; /* A dynamic buffer to hold offsets */ 294 int32_t offsetBufferSize; /* The size of the offset buffer */ 295 296 UCharIterator *iterator; 297 /*int32_t iteratorIndex;*/ 298 } collIterate; 299 300 #define paddedsize(something) ((something)+((((something)%4)!=0)?(4-(something)%4):0)) 301 #define headersize (paddedsize(sizeof(UCATableHeader))+paddedsize(sizeof(UColOptionSet))) 302 303 /* 304 struct used internally in getSpecial*CE. 305 data similar to collIterate. 306 */ 307 struct collIterateState { 308 UChar *pos; /* This is position in the string. Can be to original or writable buf */ 309 UChar *returnPos; 310 UChar *fcdPosition; /* Position in the original string to continue FCD check from. */ 311 UChar *bufferaddress; /* address of the normalization buffer */ 312 uint32_t buffersize; 313 uint8_t flags; 314 uint8_t origFlags; 315 uint32_t iteratorIndex; 316 int32_t iteratorMove; 317 }; 318 319 U_CAPI void U_EXPORT2 320 uprv_init_collIterate(const UCollator *collator, const UChar *sourceString, int32_t sourceLen, collIterate *s); 321 322 U_NAMESPACE_BEGIN 323 324 struct UCollationPCE; 325 typedef struct UCollationPCE UCollationPCE; 326 327 U_NAMESPACE_END 328 329 struct UCollationElements 330 { 331 /** 332 * Struct wrapper for source data 333 */ 334 collIterate iteratordata_; 335 /** 336 * Indicates if this data has been reset. 337 */ 338 UBool reset_; 339 /** 340 * Indicates if the data should be deleted. 341 */ 342 UBool isWritable; 343 344 /** 345 * Data for getNextProcessed, getPreviousProcessed. 346 */ 347 U_NAMESPACE_QUALIFIER UCollationPCE *pce; 348 }; 349 350 351 U_CAPI void U_EXPORT2 352 uprv_init_pce(const struct UCollationElements *elems); 353 354 #define UCOL_LEVELTERMINATOR 1 355 356 /* mask off anything but primary order */ 357 #define UCOL_PRIMARYORDERMASK 0xffff0000 358 /* mask off anything but secondary order */ 359 #define UCOL_SECONDARYORDERMASK 0x0000ff00 360 /* mask off anything but tertiary order */ 361 #define UCOL_TERTIARYORDERMASK 0x000000ff 362 /* primary order shift */ 363 #define UCOL_PRIMARYORDERSHIFT 16 364 /* secondary order shift */ 365 #define UCOL_SECONDARYORDERSHIFT 8 366 367 #define UCOL_BYTE_SIZE_MASK 0xFF 368 369 #define UCOL_CASE_BYTE_START 0x80 370 #define UCOL_CASE_SHIFT_START 7 371 372 #define UCOL_IGNORABLE 0 373 374 /* get weights from a CE */ 375 #define UCOL_PRIMARYORDER(order) (((order) & UCOL_PRIMARYORDERMASK)>> UCOL_PRIMARYORDERSHIFT) 376 #define UCOL_SECONDARYORDER(order) (((order) & UCOL_SECONDARYORDERMASK)>> UCOL_SECONDARYORDERSHIFT) 377 #define UCOL_TERTIARYORDER(order) ((order) & UCOL_TERTIARYORDERMASK) 378 379 /** 380 * Determine if a character is a Thai vowel (which sorts after 381 * its base consonant). 382 */ 383 #define UCOL_ISTHAIPREVOWEL(ch) ((((uint32_t)(ch) - 0xe40) <= (0xe44 - 0xe40)) || \ 384 (((uint32_t)(ch) - 0xec0) <= (0xec4 - 0xec0))) 385 386 /** 387 * Determine if a character is a Thai base consonant 388 */ 389 #define UCOL_ISTHAIBASECONSONANT(ch) ((uint32_t)(ch) - 0xe01) <= (0xe2e - 0xe01) 390 391 #define UCOL_ISJAMO(ch) ((((uint32_t)(ch) - 0x1100) <= (0x1112 - 0x1100)) || \ 392 (((uint32_t)(ch) - 0x1161) <= (0x1175 - 0x1161)) || \ 393 (((uint32_t)(ch) - 0x11A8) <= (0x11C2 - 0x11A8))) 394 395 /* Han character ranges */ 396 #define UCOL_FIRST_HAN 0x4E00 397 #define UCOL_LAST_HAN 0x9FFF 398 #define UCOL_FIRST_HAN_A 0x3400 399 #define UCOL_LAST_HAN_A 0x4DBF 400 #define UCOL_FIRST_HAN_COMPAT 0xFAE0 401 #define UCOL_LAST_HAN_COMPAT 0xFA2F 402 403 /* Han extension B is in plane 2 */ 404 #define UCOL_FIRST_HAN_B 0x20000 405 #define UCOL_LAST_HAN_B 0x2A6DF 406 407 /* Hangul range */ 408 #define UCOL_FIRST_HANGUL 0xAC00 409 #define UCOL_LAST_HANGUL 0xD7AF 410 411 /* Jamo ranges */ 412 #define UCOL_FIRST_L_JAMO 0x1100 413 #define UCOL_FIRST_V_JAMO 0x1161 414 #define UCOL_FIRST_T_JAMO 0x11A8 415 #define UCOL_LAST_T_JAMO 0x11F9 416 417 418 #if 0 419 /* initializes collIterate structure */ 420 /* made as macro to speed up things */ 421 #define init_collIterate(collator, sourceString, sourceLen, s) { \ 422 (s)->start = (s)->string = (s)->pos = (UChar *)(sourceString); \ 423 (s)->endp = (sourceLen) == -1 ? NULL :(UChar *)(sourceString)+(sourceLen); \ 424 (s)->CEpos = (s)->toReturn = (s)->CEs; \ 425 (s)->isThai = TRUE; \ 426 (s)->writableBuffer = (s)->stackWritableBuffer; \ 427 (s)->writableBufSize = UCOL_WRITABLE_BUFFER_SIZE; \ 428 (s)->coll = (collator); \ 429 (s)->fcdPosition = 0; \ 430 (s)->flags = 0; \ 431 if(((collator)->normalizationMode == UCOL_ON)) (s)->flags |= UCOL_ITER_NORM; \ 432 } 433 #endif 434 435 436 437 /* 438 * Macro to get the maximum size of an expansion ending with the argument ce. 439 * Used in the Boyer Moore algorithm. 440 * Note for tailoring, the UCA maxexpansion table has been merged. 441 * Hence we only have to search the tailored collator only. 442 * @param coll const UCollator pointer 443 * @param order last collation element of the expansion sequence 444 * @param result size of the longest expansion with argument collation element 445 * as the last element 446 */ 447 #define UCOL_GETMAXEXPANSION(coll, order, result) { \ 448 const uint32_t *start; \ 449 const uint32_t *limit; \ 450 const uint32_t *mid; \ 451 start = (coll)->endExpansionCE; \ 452 limit = (coll)->lastEndExpansionCE; \ 453 while (start < limit - 1) { \ 454 mid = start + ((limit - start) >> 1); \ 455 if ((order) <= *mid) { \ 456 limit = mid; \ 457 } \ 458 else { \ 459 start = mid; \ 460 } \ 461 } \ 462 if (*start == order) { \ 463 result = *((coll)->expansionCESize + (start - (coll)->endExpansionCE)); \ 464 } \ 465 else if (*limit == order) { \ 466 result = *(coll->expansionCESize + (limit - coll->endExpansionCE)); \ 467 } \ 468 else if ((order & 0xFFFF) == 0x00C0) { \ 469 result = 2; \ 470 } \ 471 else { \ 472 result = 1; \ 473 } \ 474 } 475 476 U_CFUNC 477 uint32_t ucol_prv_getSpecialCE(const UCollator *coll, UChar ch, uint32_t CE, collIterate *source, UErrorCode *status); 478 479 U_CFUNC 480 uint32_t ucol_prv_getSpecialPrevCE(const UCollator *coll, UChar ch, uint32_t CE, 481 collIterate *source, UErrorCode *status); 482 U_CAPI uint32_t U_EXPORT2 ucol_getNextCE(const UCollator *coll, collIterate *collationSource, UErrorCode *status); 483 U_CFUNC uint32_t U_EXPORT2 ucol_getPrevCE(const UCollator *coll, 484 collIterate *collationSource, 485 UErrorCode *status); 486 /* function used by C++ getCollationKey to prevent restarting the calculation */ 487 U_CFUNC int32_t 488 ucol_getSortKeyWithAllocation(const UCollator *coll, 489 const UChar *source, int32_t sourceLength, 490 uint8_t **pResult, 491 UErrorCode *pErrorCode); 492 493 /* get some memory */ 494 void *ucol_getABuffer(const UCollator *coll, uint32_t size); 495 496 /* worker function for generating sortkeys */ 497 U_CFUNC 498 int32_t U_CALLCONV 499 ucol_calcSortKey(const UCollator *coll, 500 const UChar *source, 501 int32_t sourceLength, 502 uint8_t **result, 503 uint32_t resultLength, 504 UBool allocatePrimary, 505 UErrorCode *status); 506 507 U_CFUNC 508 int32_t U_CALLCONV 509 ucol_calcSortKeySimpleTertiary(const UCollator *coll, 510 const UChar *source, 511 int32_t sourceLength, 512 uint8_t **result, 513 uint32_t resultLength, 514 UBool allocatePrimary, 515 UErrorCode *status); 516 517 U_CFUNC 518 int32_t 519 ucol_getSortKeySize(const UCollator *coll, collIterate *s, 520 int32_t currentSize, UColAttributeValue strength, 521 int32_t len); 522 /** 523 * Makes a copy of the Collator's rule data. The format is 524 * that of .col files. 525 * 526 * @param length returns the length of the data, in bytes. 527 * @param status the error status 528 * @return memory, owned by the caller, of size 'length' bytes. 529 * @internal INTERNAL USE ONLY 530 */ 531 U_CFUNC uint8_t* U_EXPORT2 532 ucol_cloneRuleData(const UCollator *coll, int32_t *length, UErrorCode *status); 533 534 /** 535 * Used to set requested and valid locales on a collator returned by the collator 536 * service. 537 */ 538 U_CFUNC void U_EXPORT2 539 ucol_setReqValidLocales(UCollator *coll, char *requestedLocaleToAdopt, char *validLocaleToAdopt, char *actualLocaleToAdopt); 540 541 #define UCOL_SPECIAL_FLAG 0xF0000000 542 #define UCOL_TAG_SHIFT 24 543 #define UCOL_TAG_MASK 0x0F000000 544 #define INIT_EXP_TABLE_SIZE 1024 545 #define UCOL_NOT_FOUND 0xF0000000 546 #define UCOL_EXPANSION 0xF1000000 547 #define UCOL_CONTRACTION 0xF2000000 548 #define UCOL_THAI 0xF3000000 549 #define UCOL_UNMARKED 0x03 550 #define UCOL_NEW_TERTIARYORDERMASK 0x0000003f 551 552 /* Bit mask for primary collation strength. */ 553 #define UCOL_PRIMARYMASK 0xFFFF0000 554 555 /* Bit mask for secondary collation strength. */ 556 #define UCOL_SECONDARYMASK 0x0000FF00 557 558 /* Bit mask for tertiary collation strength. */ 559 #define UCOL_TERTIARYMASK 0x000000FF 560 561 /** 562 * Internal. 563 * This indicates the last element in a UCollationElements has been consumed. 564 * Compare with the UCOL_NULLORDER, UCOL_NULLORDER is returned if error occurs. 565 */ 566 #define UCOL_NO_MORE_CES 0x00010101 567 #define UCOL_NO_MORE_CES_PRIMARY 0x00010000 568 #define UCOL_NO_MORE_CES_SECONDARY 0x00000100 569 #define UCOL_NO_MORE_CES_TERTIARY 0x00000001 570 571 #define isSpecial(CE) ((((CE)&UCOL_SPECIAL_FLAG)>>28)==0xF) 572 573 #define UCOL_UPPER_CASE 0x80 574 #define UCOL_MIXED_CASE 0x40 575 #define UCOL_LOWER_CASE 0x00 576 577 #define UCOL_CONTINUATION_MARKER 0xC0 578 #define UCOL_REMOVE_CONTINUATION 0xFFFFFF3F 579 580 #define isContinuation(CE) (((CE) & UCOL_CONTINUATION_MARKER) == UCOL_CONTINUATION_MARKER) 581 #define isFlagged(CE) (((CE) & 0x80) == 0x80) 582 #define isLongPrimary(CE) (((CE) & 0xC0) == 0xC0) 583 584 #define getCETag(CE) (((CE)&UCOL_TAG_MASK)>>UCOL_TAG_SHIFT) 585 #define isContraction(CE) (isSpecial((CE)) && (getCETag((CE)) == CONTRACTION_TAG)) 586 #define isPrefix(CE) (isSpecial((CE)) && (getCETag((CE)) == SPEC_PROC_TAG)) 587 #define constructContractCE(tag, CE) (UCOL_SPECIAL_FLAG | ((tag)<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF) 588 #define constructSpecProcCE(CE) (UCOL_SPECIAL_FLAG | (SPEC_PROC_TAG<<UCOL_TAG_SHIFT) | ((CE))&0xFFFFFF) 589 #define getContractOffset(CE) ((CE)&0xFFFFFF) 590 #define getExpansionOffset(CE) (((CE)&0x00FFFFF0)>>4) 591 #define getExpansionCount(CE) ((CE)&0xF) 592 #define isCEIgnorable(CE) (((CE) & 0xFFFFFFBF) == 0) 593 594 /* StringSearch internal use */ 595 #define inNormBuf(coleiter) ((coleiter)->iteratordata_.flags & UCOL_ITER_INNORMBUF) 596 #define isFCDPointerNull(coleiter) ((coleiter)->iteratordata_.fcdPosition == NULL) 597 #define hasExpansion(coleiter) ((coleiter)->iteratordata_.CEpos != (coleiter)->iteratordata_.CEs) 598 #define getExpansionPrefix(coleiter) ((coleiter)->iteratordata_.toReturn - (coleiter)->iteratordata_.CEs) 599 #define setExpansionPrefix(coleiter, offset) ((coleiter)->iteratordata_.CEs + offset) 600 #define getExpansionSuffix(coleiter) ((coleiter)->iteratordata_.CEpos - (coleiter)->iteratordata_.toReturn) 601 #define setExpansionSuffix(coleiter, offset) ((coleiter)->iteratordata_.toReturn = (coleiter)->iteratordata_.CEpos - leftoverces) 602 603 /* This is an enum that lists magic special byte values from the fractional UCA */ 604 /* TODO: all the #defines that refer to special byte values from the UCA should be changed to point here */ 605 606 enum { 607 UCOL_BYTE_ZERO = 0x00, 608 UCOL_BYTE_LEVEL_SEPARATOR = 0x01, 609 UCOL_BYTE_SORTKEY_GLUE = 0x02, 610 UCOL_BYTE_SHIFT_PREFIX = 0x03, 611 UCOL_BYTE_UNSHIFTED_MIN = UCOL_BYTE_SHIFT_PREFIX, 612 UCOL_BYTE_FIRST_TAILORED = 0x04, 613 UCOL_BYTE_COMMON = 0x05, 614 UCOL_BYTE_FIRST_UCA = UCOL_BYTE_COMMON, 615 UCOL_CODAN_PLACEHOLDER = 0x27, 616 UCOL_BYTE_LAST_LATIN_PRIMARY = 0x4C, 617 UCOL_BYTE_FIRST_NON_LATIN_PRIMARY = 0x4D, 618 UCOL_BYTE_UNSHIFTED_MAX = 0xFF 619 }; 620 621 #if 0 622 #define UCOL_RESET_TOP_VALUE 0x9F000303 623 #define UCOL_FIRST_PRIMARY_IGNORABLE 0x00008705 624 #define UCOL_LAST_PRIMARY_IGNORABLE 0x0000DD05 625 #define UCOL_LAST_PRIMARY_IGNORABLE_CONT 0x000051C0 626 #define UCOL_FIRST_SECONDARY_IGNORABLE 0x00000000 627 #define UCOL_LAST_SECONDARY_IGNORABLE 0x00000500 628 #define UCOL_FIRST_TERTIARY_IGNORABLE 0x00000000 629 #define UCOL_LAST_TERTIARY_IGNORABLE 0x00000000 630 #define UCOL_FIRST_VARIABLE 0x05070505 631 #define UCOL_LAST_VARIABLE 0x179B0505 632 #define UCOL_FIRST_NON_VARIABLE 0x1A200505 633 #define UCOL_LAST_NON_VARIABLE 0x7B41058F 634 635 #define UCOL_NEXT_TOP_VALUE 0xE8960303 636 #define UCOL_NEXT_FIRST_PRIMARY_IGNORABLE 0x00008905 637 #define UCOL_NEXT_LAST_PRIMARY_IGNORABLE 0x03000303 638 #define UCOL_NEXT_FIRST_SECONDARY_IGNORABLE 0x00008705 639 #define UCOL_NEXT_LAST_SECONDARY_IGNORABLE 0x00000500 640 #define UCOL_NEXT_FIRST_TERTIARY_IGNORABLE 0x00000000 641 #define UCOL_NEXT_LAST_TERTIARY_IGNORABLE 0x00000000 642 #define UCOL_NEXT_FIRST_VARIABLE 0x05090505 643 #define UCOL_NEXT_LAST_VARIABLE 0x1A200505 644 645 #define PRIMARY_IMPLICIT_MIN 0xE8000000 646 #define PRIMARY_IMPLICIT_MAX 0xF0000000 647 #endif 648 649 /* These constants can be changed - sortkey size is affected by them */ 650 #define UCOL_PROPORTION2 0.5 651 #define UCOL_PROPORTION3 0.667 652 653 /* These values come from the UCA */ 654 #define UCOL_COMMON_BOT2 UCOL_BYTE_COMMON 655 #define UCOL_COMMON_TOP2 0x86u 656 #define UCOL_TOTAL2 (UCOL_COMMON_TOP2-UCOL_COMMON_BOT2-1) 657 658 #define UCOL_FLAG_BIT_MASK_CASE_SW_OFF 0x80 659 #define UCOL_FLAG_BIT_MASK_CASE_SW_ON 0x40 660 #define UCOL_COMMON_TOP3_CASE_SW_OFF 0x85 661 #define UCOL_COMMON_TOP3_CASE_SW_LOWER 0x45 662 #define UCOL_COMMON_TOP3_CASE_SW_UPPER 0xC5 663 664 /* These values come from the UCA */ 665 #define UCOL_COMMON_BOT3 0x05 666 667 #define UCOL_COMMON_BOTTOM3_CASE_SW_UPPER 0x86; 668 #define UCOL_COMMON_BOTTOM3_CASE_SW_LOWER UCOL_COMMON_BOT3; 669 670 #define UCOL_TOP_COUNT2 (UCOL_PROPORTION2*UCOL_TOTAL2) 671 #define UCOL_BOT_COUNT2 (UCOL_TOTAL2-UCOL_TOP_COUNT2) 672 673 674 #define UCOL_COMMON2 UCOL_COMMON_BOT2 675 #define UCOL_COMMON3_UPPERFIRST 0xC5 676 #define UCOL_COMMON3_NORMAL UCOL_COMMON_BOT3 677 678 #define UCOL_COMMON4 0xFF 679 680 /* constants for case level/case first handling */ 681 /* used to instantiate UCollators fields in ucol_updateInternalState */ 682 #define UCOL_CASE_SWITCH 0xC0 683 #define UCOL_NO_CASE_SWITCH 0x00 684 685 #define UCOL_REMOVE_CASE 0x3F 686 #define UCOL_KEEP_CASE 0xFF 687 688 #define UCOL_CASE_BIT_MASK 0xC0 689 690 #define UCOL_TERT_CASE_MASK 0xFF 691 692 #define UCOL_ENDOFLATINONERANGE 0xFF 693 #define UCOL_LATINONETABLELEN (UCOL_ENDOFLATINONERANGE+50) 694 #define UCOL_BAIL_OUT_CE 0xFF000000 695 696 697 typedef enum { 698 NOT_FOUND_TAG = 0, 699 EXPANSION_TAG = 1, /* This code point results in an expansion */ 700 CONTRACTION_TAG = 2, /* Start of a contraction */ 701 THAI_TAG = 3, /* Thai character - do the reordering */ 702 CHARSET_TAG = 4, /* Charset processing, not yet implemented */ 703 SURROGATE_TAG = 5, /* Lead surrogate that is tailored and doesn't start a contraction */ 704 HANGUL_SYLLABLE_TAG = 6, /* AC00-D7AF*/ 705 LEAD_SURROGATE_TAG = 7, /* D800-DBFF*/ 706 TRAIL_SURROGATE_TAG = 8, /* DC00-DFFF*/ 707 CJK_IMPLICIT_TAG = 9, /* 0x3400-0x4DB5, 0x4E00-0x9FA5, 0xF900-0xFA2D*/ 708 IMPLICIT_TAG = 10, 709 SPEC_PROC_TAG = 11, 710 /* ICU 2.1 */ 711 LONG_PRIMARY_TAG = 12, /* This is a three byte primary with starting secondaries and tertiaries */ 712 /* It fits in a single 32 bit CE and is used instead of expansion to save */ 713 /* space without affecting the performance (hopefully) */ 714 715 DIGIT_TAG = 13, /* COllate Digits As Numbers (CODAN) implementation */ 716 717 CE_TAGS_COUNT 718 } UColCETags; 719 720 /* 721 ***************************************************************************************** 722 * set to zero 723 * NON_CHARACTER FDD0 - FDEF, FFFE, FFFF, 1FFFE, 1FFFF, 2FFFE, 2FFFF,...e.g. **FFFE, **FFFF 724 ****************************************************************************************** 725 */ 726 727 typedef struct { 728 uint32_t variableTopValue; 729 /*UColAttributeValue*/ int32_t frenchCollation; 730 /*UColAttributeValue*/ int32_t alternateHandling; /* attribute for handling variable elements*/ 731 /*UColAttributeValue*/ int32_t caseFirst; /* who goes first, lower case or uppercase */ 732 /*UColAttributeValue*/ int32_t caseLevel; /* do we have an extra case level */ 733 /*UColAttributeValue*/ int32_t normalizationMode; /* attribute for normalization */ 734 /*UColAttributeValue*/ int32_t strength; /* attribute for strength */ 735 /*UColAttributeValue*/ int32_t hiraganaQ; /* attribute for special Hiragana */ 736 /*UColAttributeValue*/ int32_t numericCollation; /* attribute for numeric collation */ 737 uint32_t reserved[15]; /* for future use */ 738 } UColOptionSet; 739 740 typedef struct { 741 uint32_t UCA_FIRST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ 742 uint32_t UCA_LAST_TERTIARY_IGNORABLE[2]; /*0x00000000*/ 743 uint32_t UCA_FIRST_PRIMARY_IGNORABLE[2]; /*0x00008705*/ 744 uint32_t UCA_FIRST_SECONDARY_IGNORABLE[2]; /*0x00000000*/ 745 uint32_t UCA_LAST_SECONDARY_IGNORABLE[2]; /*0x00000500*/ 746 uint32_t UCA_LAST_PRIMARY_IGNORABLE[2]; /*0x0000DD05*/ 747 uint32_t UCA_FIRST_VARIABLE[2]; /*0x05070505*/ 748 uint32_t UCA_LAST_VARIABLE[2]; /*0x13CF0505*/ 749 uint32_t UCA_FIRST_NON_VARIABLE[2]; /*0x16200505*/ 750 uint32_t UCA_LAST_NON_VARIABLE[2]; /*0x767C0505*/ 751 uint32_t UCA_RESET_TOP_VALUE[2]; /*0x9F000303*/ 752 uint32_t UCA_FIRST_IMPLICIT[2]; 753 uint32_t UCA_LAST_IMPLICIT[2]; 754 uint32_t UCA_FIRST_TRAILING[2]; 755 uint32_t UCA_LAST_TRAILING[2]; 756 757 #if 0 758 uint32_t UCA_NEXT_TOP_VALUE[2]; /*0xE8960303*/ 759 uint32_t UCA_NEXT_FIRST_PRIMARY_IGNORABLE; /*0x00008905*/ 760 uint32_t UCA_NEXT_LAST_PRIMARY_IGNORABLE; /*0x03000303*/ 761 uint32_t UCA_NEXT_FIRST_SECONDARY_IGNORABLE; /*0x00008705*/ 762 uint32_t UCA_NEXT_LAST_SECONDARY_IGNORABLE; /*0x00000500*/ 763 uint32_t UCA_NEXT_FIRST_TERTIARY_IGNORABLE; /*0x00000000*/ 764 uint32_t UCA_NEXT_LAST_TERTIARY_IGNORABLE; /*0x00000000*/ 765 uint32_t UCA_NEXT_FIRST_VARIABLE; /*0x05090505*/ 766 uint32_t UCA_NEXT_LAST_VARIABLE; /*0x16200505*/ 767 #endif 768 769 uint32_t UCA_PRIMARY_TOP_MIN; 770 uint32_t UCA_PRIMARY_IMPLICIT_MIN; /*0xE8000000*/ 771 uint32_t UCA_PRIMARY_IMPLICIT_MAX; /*0xF0000000*/ 772 uint32_t UCA_PRIMARY_TRAILING_MIN; /*0xE8000000*/ 773 uint32_t UCA_PRIMARY_TRAILING_MAX; /*0xF0000000*/ 774 uint32_t UCA_PRIMARY_SPECIAL_MIN; /*0xE8000000*/ 775 uint32_t UCA_PRIMARY_SPECIAL_MAX; /*0xF0000000*/ 776 } UCAConstants; 777 778 typedef struct { 779 int32_t size; 780 /* all the offsets are in bytes */ 781 /* to get the address add to the header address and cast properly */ 782 uint32_t options; /* these are the default options for the collator */ 783 uint32_t UCAConsts; /* structure which holds values for indirect positioning and implicit ranges */ 784 uint32_t contractionUCACombos; /* this one is needed only for UCA, to copy the appropriate contractions */ 785 uint32_t magic; /* magic number - lets us know whether reserved data is reset or junked */ 786 uint32_t mappingPosition; /* const uint8_t *mappingPosition; */ 787 uint32_t expansion; /* uint32_t *expansion; */ 788 uint32_t contractionIndex; /* UChar *contractionIndex; */ 789 uint32_t contractionCEs; /* uint32_t *contractionCEs; */ 790 uint32_t contractionSize; /* needed for various closures */ 791 /*int32_t latinOneMapping;*/ /* this is now handled in the trie itself *//* fast track to latin1 chars */ 792 793 uint32_t endExpansionCE; /* array of last collation element in 794 expansion */ 795 uint32_t expansionCESize; /* array of maximum expansion size 796 corresponding to the expansion 797 collation elements with last element 798 in endExpansionCE*/ 799 int32_t endExpansionCECount; /* size of endExpansionCE */ 800 uint32_t unsafeCP; /* hash table of unsafe code points */ 801 uint32_t contrEndCP; /* hash table of final code points */ 802 /* in contractions. */ 803 804 int32_t contractionUCACombosSize; /* number of UCA contraction items. */ 805 /*Length is contractionUCACombosSize*contractionUCACombosWidth*sizeof(UChar) */ 806 UBool jamoSpecial; /* is jamoSpecial */ 807 UBool isBigEndian; /* is this data big endian? from the UDataInfo header*/ 808 uint8_t charSetFamily; /* what is the charset family of this data from the UDataInfo header*/ 809 uint8_t contractionUCACombosWidth; /* width of UCA combos field */ 810 UVersionInfo version; 811 UVersionInfo UCAVersion; /* version of the UCA, read from file */ 812 UVersionInfo UCDVersion; /* UCD version, obtained by u_getUnicodeVersion */ 813 UVersionInfo formatVersion; /* format version from the UDataInfo header */ 814 uint8_t reserved[84]; /* for future use */ 815 } UCATableHeader; 816 817 #define U_UNKNOWN_STATE 0 818 #define U_COLLATOR_STATE 0x01 819 #define U_STATE_LIMIT 0x02 820 821 /* This is the first structure in a state */ 822 /* it should be machine independent */ 823 typedef struct { 824 /* this structure is supposed to be readable on all the platforms.*/ 825 /* first 2 fields hold the size of the structure in a platform independent way */ 826 uint8_t sizeLo; 827 uint8_t sizeHi; 828 /* identifying the writing platform */ 829 uint8_t isBigEndian; 830 /* see U_CHARSET_FAMILY values in utypes.h */ 831 uint8_t charsetFamily; 832 /* version of ICU this state structure comes from */ 833 uint8_t icuVersion[4]; 834 /* What is the data following this state */ 835 uint8_t type; 836 /* more stuff to come, keep it on 16 byte boundary */ 837 uint8_t reserved[7]; 838 } UStateStruct; 839 840 /* This structure follows UStatusStruct */ 841 /* and contains data specific for the collators */ 842 /* Endianess needs to be decided before accessing this structure */ 843 /* However, it's size IS endianess independent */ 844 typedef struct { 845 /* size of this structure */ 846 uint8_t sizeLo; 847 uint8_t sizeHi; 848 /* This state is followed by the frozen tailoring */ 849 uint8_t containsTailoring; 850 /* This state is followed by the frozen UCA */ 851 uint8_t containsUCA; 852 /* Version info - the same one */ 853 uint8_t versionInfo[4]; 854 855 /* for charset CEs */ 856 uint8_t charsetName[32]; 857 /* this is the resolved locale name*/ 858 uint8_t locale[32]; 859 860 /* Attributes. Open ended */ 861 /* all the following will be moved to uint32_t because of portability */ 862 /* variable top value */ 863 uint32_t variableTopValue; 864 /* attribute for handling variable elements*/ 865 uint32_t /*UColAttributeValue*/ alternateHandling; 866 /* how to handle secondary weights */ 867 uint32_t /*UColAttributeValue*/ frenchCollation; 868 /* who goes first, lower case or uppercase */ 869 uint32_t /*UColAttributeValue*/ caseFirst; 870 /* do we have an extra case level */ 871 uint32_t /*UColAttributeValue*/ caseLevel; 872 /* attribute for normalization */ 873 uint32_t /*UColAttributeValue*/ normalizationMode; 874 /* attribute for strength */ 875 uint32_t /*UColAttributeValue*/ strength; 876 /* to be immediately 16 byte aligned */ 877 uint8_t reserved[12]; 878 } UColStateStruct; 879 880 #define UCOL_INV_SIZEMASK 0xFFF00000 881 #define UCOL_INV_OFFSETMASK 0x000FFFFF 882 #define UCOL_INV_SHIFTVALUE 20 883 884 U_CDECL_BEGIN 885 886 typedef struct { 887 uint32_t byteSize; 888 uint32_t tableSize; 889 uint32_t contsSize; 890 uint32_t table; 891 uint32_t conts; 892 UVersionInfo UCAVersion; /* version of the UCA, read from file */ 893 uint8_t padding[8]; 894 } InverseUCATableHeader; 895 896 typedef int32_t U_CALLCONV 897 SortKeyGenerator(const UCollator *coll, 898 const UChar *source, 899 int32_t sourceLength, 900 uint8_t **result, 901 uint32_t resultLength, 902 UBool allocatePrimary, 903 UErrorCode *status); 904 905 typedef void U_CALLCONV 906 ResourceCleaner(UCollator *coll); 907 908 909 struct UCollator { 910 UColOptionSet *options; 911 SortKeyGenerator *sortKeyGen; 912 uint32_t *latinOneCEs; 913 char* actualLocale; 914 char* validLocale; 915 char* requestedLocale; 916 const UChar *rules; 917 const UChar *ucaRules; 918 const UCollator *UCA; 919 const UCATableHeader *image; 920 UTrie mapping; 921 const uint32_t *latinOneMapping; 922 const uint32_t *expansion; 923 const UChar *contractionIndex; 924 const uint32_t *contractionCEs; 925 /*const uint8_t *scriptOrder;*/ 926 927 const uint32_t *endExpansionCE; /* array of last ces in an expansion ce. 928 corresponds to expansionCESize */ 929 const uint32_t *lastEndExpansionCE;/* pointer to the last element in endExpansionCE */ 930 const uint8_t *expansionCESize; /* array of the maximum size of a 931 expansion ce with the last ce 932 corresponding to endExpansionCE, 933 terminated with a null */ 934 const uint8_t *unsafeCP; /* unsafe code points hashtable */ 935 const uint8_t *contrEndCP; /* Contraction ending chars hash table */ 936 UChar minUnsafeCP; /* Smallest unsafe Code Point. */ 937 UChar minContrEndCP; /* Smallest code point at end of a contraction */ 938 939 int32_t rulesLength; 940 int32_t latinOneTableLen; 941 942 uint32_t variableTopValue; 943 UColAttributeValue frenchCollation; 944 UColAttributeValue alternateHandling; /* attribute for handling variable elements*/ 945 UColAttributeValue caseFirst; /* who goes first, lower case or uppercase */ 946 UColAttributeValue caseLevel; /* do we have an extra case level */ 947 UColAttributeValue normalizationMode; /* attribute for normalization */ 948 UColAttributeValue strength; /* attribute for strength */ 949 UColAttributeValue hiraganaQ; /* attribute for Hiragana */ 950 UColAttributeValue numericCollation; 951 UBool variableTopValueisDefault; 952 UBool frenchCollationisDefault; 953 UBool alternateHandlingisDefault; /* attribute for handling variable elements*/ 954 UBool caseFirstisDefault; /* who goes first, lower case or uppercase */ 955 UBool caseLevelisDefault; /* do we have an extra case level */ 956 UBool normalizationModeisDefault; /* attribute for normalization */ 957 UBool strengthisDefault; /* attribute for strength */ 958 UBool hiraganaQisDefault; /* attribute for Hiragana */ 959 UBool numericCollationisDefault; 960 UBool hasRealData; /* some collators have only options, like French, no rules */ 961 /* to speed up things, we use the UCA image, but we don't want it */ 962 /* to run around */ 963 964 UBool freeOnClose; 965 UBool freeOptionsOnClose; 966 UBool freeRulesOnClose; 967 UBool freeImageOnClose; 968 969 UBool latinOneUse; 970 UBool latinOneRegenTable; 971 UBool latinOneFailed; 972 973 int8_t tertiaryAddition; /* when switching case, we need to add or subtract different values */ 974 uint8_t caseSwitch; 975 uint8_t tertiaryCommon; 976 uint8_t tertiaryMask; 977 uint8_t tertiaryTop; /* Upper range when compressing */ 978 uint8_t tertiaryBottom; /* Upper range when compressing */ 979 uint8_t tertiaryTopCount; 980 uint8_t tertiaryBottomCount; 981 982 UVersionInfo dataVersion; /* Data info of UCA table */ 983 }; 984 985 U_CDECL_END 986 987 /* various internal functions */ 988 989 /* do not close UCA returned by ucol_initUCA! */ 990 U_CFUNC 991 UCollator* ucol_initUCA(UErrorCode *status); 992 993 U_CFUNC 994 UCollator* ucol_initCollator(const UCATableHeader *image, UCollator *fillIn, const UCollator *UCA, UErrorCode *status); 995 996 U_CFUNC 997 void ucol_setOptionsFromHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); 998 999 U_CFUNC 1000 UCollator* ucol_open_internal(const char* loc, UErrorCode* status); 1001 1002 #if 0 1003 U_CFUNC 1004 void ucol_putOptionsToHeader(UCollator* result, UColOptionSet * opts, UErrorCode *status); 1005 #endif 1006 1007 U_CFUNC 1008 void ucol_updateInternalState(UCollator *coll, UErrorCode *status); 1009 1010 U_CFUNC uint32_t U_EXPORT2 ucol_getFirstCE(const UCollator *coll, UChar u, UErrorCode *status); 1011 U_CAPI UBool U_EXPORT2 ucol_isTailored(const UCollator *coll, const UChar u, UErrorCode *status); 1012 1013 U_CAPI const InverseUCATableHeader* U_EXPORT2 ucol_initInverseUCA(UErrorCode *status); 1014 1015 U_CAPI void U_EXPORT2 1016 uprv_uca_initImplicitConstants(UErrorCode *status); 1017 1018 U_CAPI uint32_t U_EXPORT2 1019 uprv_uca_getImplicitFromRaw(UChar32 cp); 1020 1021 /*U_CFUNC uint32_t U_EXPORT2 1022 uprv_uca_getImplicitPrimary(UChar32 cp);*/ 1023 1024 U_CAPI UChar32 U_EXPORT2 1025 uprv_uca_getRawFromImplicit(uint32_t implicit); 1026 1027 U_CAPI UChar32 U_EXPORT2 1028 uprv_uca_getRawFromCodePoint(UChar32 i); 1029 1030 U_CAPI UChar32 U_EXPORT2 1031 uprv_uca_getCodePointFromRaw(UChar32 i); 1032 1033 1034 1035 #ifdef XP_CPLUSPLUS 1036 /* 1037 * Test whether a character is potentially "unsafe" for use as a collation 1038 * starting point. Unsafe chars are those with combining class != 0 plus 1039 * those that are the 2nd thru nth character in a contraction sequence. 1040 * 1041 * Function is in header file because it's used in both collation and string search, 1042 * and needs to be inline for performance. 1043 */ 1044 static inline UBool ucol_unsafeCP(UChar c, const UCollator *coll) { 1045 int32_t hash; 1046 uint8_t htbyte; 1047 1048 if (c < coll->minUnsafeCP) { 1049 return FALSE; 1050 } 1051 1052 hash = c; 1053 if (hash >= UCOL_UNSAFECP_TABLE_SIZE*8) { 1054 if(UTF_IS_SURROGATE(c)) { 1055 /* Lead or trail surrogate */ 1056 /* These are always considered unsafe. */ 1057 return TRUE; 1058 } 1059 hash = (hash & UCOL_UNSAFECP_TABLE_MASK) + 256; 1060 } 1061 htbyte = coll->unsafeCP[hash>>3]; 1062 return ((htbyte >> (hash & 7)) & 1); 1063 } 1064 #endif /* XP_CPLUSPLUS */ 1065 1066 /* The offsetBuffer in collIterate might need to be freed to avoid memory leaks. */ 1067 void ucol_freeOffsetBuffer(collIterate *s); 1068 1069 1070 #endif /* #if !UCONFIG_NO_COLLATION */ 1071 1072 #endif 1073
