Home | History | Annotate | Download | only in i18n 
      1 /*
      2 *******************************************************************************
      3 * Copyright (C) 2010-2015, International Business Machines
      4 * Corporation and others.  All Rights Reserved.
      5 *******************************************************************************
      6 * collationdata.h
      7 *
      8 * created on: 2010oct27
      9 * created by: Markus W. Scherer
     10 */
     11 
     12 #ifndef __COLLATIONDATA_H__
     13 #define __COLLATIONDATA_H__
     14 
     15 #include "unicode/utypes.h"
     16 
     17 #if !UCONFIG_NO_COLLATION
     18 
     19 #include "unicode/ucol.h"
     20 #include "unicode/uniset.h"
     21 #include "collation.h"
     22 #include "normalizer2impl.h"
     23 #include "utrie2.h"
     24 
     25 struct UDataMemory;
     26 
     27 U_NAMESPACE_BEGIN
     28 
     29 class UVector32;
     30 
     31 /**
     32  * Collation data container.
     33  * Immutable data created by a CollationDataBuilder, or loaded from a file,
     34  * or deserialized from API-provided binary data.
     35  *
     36  * Includes data for the collation base (root/default), aliased if this is not the base.
     37  */
     38 struct U_I18N_API CollationData : public UMemory {
     39     // Note: The ucadata.icu loader could discover the reserved ranges by setting an array
     40     // parallel with the ranges, and resetting ranges that are indexed.
     41     // The reordering builder code could clone the resulting template array.
     42     enum {
     43         REORDER_RESERVED_BEFORE_LATIN = UCOL_REORDER_CODE_FIRST + 14,
     44         REORDER_RESERVED_AFTER_LATIN
     45     };
     46 
     47     enum {
     48         MAX_NUM_SPECIAL_REORDER_CODES = 8,
     49         /** C++ only, data reader check scriptStartsLength. */
     50         MAX_NUM_SCRIPT_RANGES = 256
     51     };
     52 
     53     CollationData(const Normalizer2Impl &nfc)
     54             : trie(NULL),
     55               ce32s(NULL), ces(NULL), contexts(NULL), base(NULL),
     56               jamoCE32s(NULL),
     57               nfcImpl(nfc),
     58               numericPrimary(0x12000000),
     59               ce32sLength(0), cesLength(0), contextsLength(0),
     60               compressibleBytes(NULL),
     61               unsafeBackwardSet(NULL),
     62               fastLatinTable(NULL), fastLatinTableLength(0),
     63               numScripts(0), scriptsIndex(NULL), scriptStarts(NULL), scriptStartsLength(0),
     64               rootElements(NULL), rootElementsLength(0) {}
     65 
     66     uint32_t getCE32(UChar32 c) const {
     67         return UTRIE2_GET32(trie, c);
     68     }
     69 
     70     uint32_t getCE32FromSupplementary(UChar32 c) const {
     71         return UTRIE2_GET32_FROM_SUPP(trie, c);
     72     }
     73 
     74     UBool isDigit(UChar32 c) const {
     75         return c < 0x660 ? c <= 0x39 && 0x30 <= c :
     76                 Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
     77     }
     78 
     79     UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
     80         return unsafeBackwardSet->contains(c) || (numeric && isDigit(c));
     81     }
     82 
     83     UBool isCompressibleLeadByte(uint32_t b) const {
     84         return compressibleBytes[b];
     85     }
     86 
     87     inline UBool isCompressiblePrimary(uint32_t p) const {
     88         return isCompressibleLeadByte(p >> 24);
     89     }
     90 
     91     /**
     92      * Returns the CE32 from two contexts words.
     93      * Access to the defaultCE32 for contraction and prefix matching.
     94      */
     95     static uint32_t readCE32(const UChar *p) {
     96         return ((uint32_t)p[0] << 16) | p[1];
     97     }
     98 
     99     /**
    100      * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
    101      * Requires that ce32 is special.
    102      */
    103     uint32_t getIndirectCE32(uint32_t ce32) const;
    104     /**
    105      * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
    106      * if ce32 is special.
    107      */
    108     uint32_t getFinalCE32(uint32_t ce32) const;
    109 
    110     /**
    111      * Computes a CE from c's ce32 which has the OFFSET_TAG.
    112      */
    113     int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
    114         int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
    115         return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
    116     }
    117 
    118     /**
    119      * Returns the single CE that c maps to.
    120      * Sets U_UNSUPPORTED_ERROR if c does not map to a single CE.
    121      */
    122     int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
    123 
    124     /**
    125      * Returns the FCD16 value for code point c. c must be >= 0.
    126      */
    127     uint16_t getFCD16(UChar32 c) const {
    128         return nfcImpl.getFCD16(c);
    129     }
    130 
    131     /**
    132      * Returns the first primary for the script's reordering group.
    133      * @return the primary with only the first primary lead byte of the group
    134      *         (not necessarily an actual root collator primary weight),
    135      *         or 0 if the script is unknown
    136      */
    137     uint32_t getFirstPrimaryForGroup(int32_t script) const;
    138 
    139     /**
    140      * Returns the last primary for the script's reordering group.
    141      * @return the last primary of the group
    142      *         (not an actual root collator primary weight),
    143      *         or 0 if the script is unknown
    144      */
    145     uint32_t getLastPrimaryForGroup(int32_t script) const;
    146 
    147     /**
    148      * Finds the reordering group which contains the primary weight.
    149      * @return the first script of the group, or -1 if the weight is beyond the last group
    150      */
    151     int32_t getGroupForPrimary(uint32_t p) const;
    152 
    153     int32_t getEquivalentScripts(int32_t script,
    154                                  int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;
    155 
    156     /**
    157      * Writes the permutation of primary-weight ranges
    158      * for the given reordering of scripts and groups.
    159      * The caller checks for illegal arguments and
    160      * takes care of [DEFAULT] and memory allocation.
    161      *
    162      * Each list element will be a (limit, offset) pair as described
    163      * for the CollationSettings::reorderRanges.
    164      * The list will be empty if no ranges are reordered.
    165      */
    166     void makeReorderRanges(const int32_t *reorder, int32_t length,
    167                            UVector32 &ranges, UErrorCode &errorCode) const;
    168 
    169     /** @see jamoCE32s */
    170     static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;
    171 
    172     /** Main lookup trie. */
    173     const UTrie2 *trie;
    174     /**
    175      * Array of CE32 values.
    176      * At index 0 there must be CE32(U+0000)
    177      * to support U+0000's special-tag for NUL-termination handling.
    178      */
    179     const uint32_t *ce32s;
    180     /** Array of CE values for expansions and OFFSET_TAG. */
    181     const int64_t *ces;
    182     /** Array of prefix and contraction-suffix matching data. */
    183     const UChar *contexts;
    184     /** Base collation data, or NULL if this data itself is a base. */
    185     const CollationData *base;
    186     /**
    187      * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
    188      * They are normally simple CE32s, rarely expansions.
    189      * For fast handling of HANGUL_TAG.
    190      */
    191     const uint32_t *jamoCE32s;
    192     const Normalizer2Impl &nfcImpl;
    193     /** The single-byte primary weight (xx000000) for numeric collation. */
    194     uint32_t numericPrimary;
    195 
    196     int32_t ce32sLength;
    197     int32_t cesLength;
    198     int32_t contextsLength;
    199 
    200     /** 256 flags for which primary-weight lead bytes are compressible. */
    201     const UBool *compressibleBytes;
    202     /**
    203      * Set of code points that are unsafe for starting string comparison after an identical prefix,
    204      * or in backwards CE iteration.
    205      */
    206     const UnicodeSet *unsafeBackwardSet;
    207 
    208     /**
    209      * Fast Latin table for common-Latin-text string comparisons.
    210      * Data structure see class CollationFastLatin.
    211      */
    212     const uint16_t *fastLatinTable;
    213     int32_t fastLatinTableLength;
    214 
    215     /**
    216      * Data for scripts and reordering groups.
    217      * Uses include building a reordering permutation table and
    218      * providing script boundaries to AlphabeticIndex.
    219      */
    220     int32_t numScripts;
    221     /**
    222      * The length of scriptsIndex is numScripts+16.
    223      * It maps from a UScriptCode or a special reorder code to an entry in scriptStarts.
    224      * 16 special reorder codes (not all used) are mapped starting at numScripts.
    225      * Up to MAX_NUM_SPECIAL_REORDER_CODES are codes for special groups like space/punct/digit.
    226      * There are special codes at the end for reorder-reserved primary ranges.
    227      *
    228      * Multiple scripts may share a range and index, for example Hira & Kana.
    229      */
    230     const uint16_t *scriptsIndex;
    231     /**
    232      * Start primary weight (top 16 bits only) for a group/script/reserved range
    233      * indexed by scriptsIndex.
    234      * The first range (separators & terminators) and the last range (trailing weights)
    235      * are not reorderable, and no scriptsIndex entry points to them.
    236      */
    237     const uint16_t *scriptStarts;
    238     int32_t scriptStartsLength;
    239 
    240     /**
    241      * Collation elements in the root collator.
    242      * Used by the CollationRootElements class. The data structure is described there.
    243      * NULL in a tailoring.
    244      */
    245     const uint32_t *rootElements;
    246     int32_t rootElementsLength;
    247 
    248 private:
    249     int32_t getScriptIndex(int32_t script) const;
    250     void makeReorderRanges(const int32_t *reorder, int32_t length,
    251                            UBool latinMustMove,
    252                            UVector32 &ranges, UErrorCode &errorCode) const;
    253     int32_t addLowScriptRange(uint8_t table[], int32_t index, int32_t lowStart) const;
    254     int32_t addHighScriptRange(uint8_t table[], int32_t index, int32_t highLimit) const;
    255 };
    256 
    257 U_NAMESPACE_END
    258 
    259 #endif  // !UCONFIG_NO_COLLATION
    260 #endif  // __COLLATIONDATA_H__
    261