Home | History | Annotate | Download | only in i18n 
      1 /*
      2 *******************************************************************************
      3 * Copyright (C) 2010-2014, International Business Machines
      4 * Corporation and others.  All Rights Reserved.
      5 *******************************************************************************
      6 * collationdata.h
      7 *
      8 * created on: 2010oct27
      9 * created by: Markus W. Scherer
     10 */
     11 
     12 #ifndef __COLLATIONDATA_H__
     13 #define __COLLATIONDATA_H__
     14 
     15 #include "unicode/utypes.h"
     16 
     17 #if !UCONFIG_NO_COLLATION
     18 
     19 #include "unicode/uniset.h"
     20 #include "collation.h"
     21 #include "normalizer2impl.h"
     22 #include "utrie2.h"
     23 
     24 struct UDataMemory;
     25 
     26 U_NAMESPACE_BEGIN
     27 
     28 /**
     29  * Collation data container.
     30  * Immutable data created by a CollationDataBuilder, or loaded from a file,
     31  * or deserialized from API-provided binary data.
     32  *
     33  * Includes data for the collation base (root/default), aliased if this is not the base.
     34  */
     35 struct U_I18N_API CollationData : public UMemory {
     36     CollationData(const Normalizer2Impl &nfc)
     37             : trie(NULL),
     38               ce32s(NULL), ces(NULL), contexts(NULL), base(NULL),
     39               jamoCE32s(NULL),
     40               nfcImpl(nfc),
     41               numericPrimary(0x12000000),
     42               ce32sLength(0), cesLength(0), contextsLength(0),
     43               compressibleBytes(NULL),
     44               unsafeBackwardSet(NULL),
     45               fastLatinTable(NULL), fastLatinTableLength(0),
     46               scripts(NULL), scriptsLength(0),
     47               rootElements(NULL), rootElementsLength(0) {}
     48 
     49     uint32_t getCE32(UChar32 c) const {
     50         return UTRIE2_GET32(trie, c);
     51     }
     52 
     53     uint32_t getCE32FromSupplementary(UChar32 c) const {
     54         return UTRIE2_GET32_FROM_SUPP(trie, c);
     55     }
     56 
     57     UBool isDigit(UChar32 c) const {
     58         return c < 0x660 ? c <= 0x39 && 0x30 <= c :
     59                 Collation::hasCE32Tag(getCE32(c), Collation::DIGIT_TAG);
     60     }
     61 
     62     UBool isUnsafeBackward(UChar32 c, UBool numeric) const {
     63         return unsafeBackwardSet->contains(c) || (numeric && isDigit(c));
     64     }
     65 
     66     UBool isCompressibleLeadByte(uint32_t b) const {
     67         return compressibleBytes[b];
     68     }
     69 
     70     inline UBool isCompressiblePrimary(uint32_t p) const {
     71         return isCompressibleLeadByte(p >> 24);
     72     }
     73 
     74     /**
     75      * Returns the CE32 from two contexts words.
     76      * Access to the defaultCE32 for contraction and prefix matching.
     77      */
     78     static uint32_t readCE32(const UChar *p) {
     79         return ((uint32_t)p[0] << 16) | p[1];
     80     }
     81 
     82     /**
     83      * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG).
     84      * Requires that ce32 is special.
     85      */
     86     uint32_t getIndirectCE32(uint32_t ce32) const;
     87     /**
     88      * Returns the CE32 for an indirect special CE32 (e.g., with DIGIT_TAG),
     89      * if ce32 is special.
     90      */
     91     uint32_t getFinalCE32(uint32_t ce32) const;
     92 
     93     /**
     94      * Computes a CE from c's ce32 which has the OFFSET_TAG.
     95      */
     96     int64_t getCEFromOffsetCE32(UChar32 c, uint32_t ce32) const {
     97         int64_t dataCE = ces[Collation::indexFromCE32(ce32)];
     98         return Collation::makeCE(Collation::getThreeBytePrimaryForOffsetData(c, dataCE));
     99     }
    100 
    101     /**
    102      * Returns the FCD16 value for code point c. c must be >= 0.
    103      */
    104     uint16_t getFCD16(UChar32 c) const {
    105         return nfcImpl.getFCD16(c);
    106     }
    107 
    108     /**
    109      * Returns the first primary for the script's reordering group.
    110      * @return the primary with only the first primary lead byte of the group
    111      *         (not necessarily an actual root collator primary weight),
    112      *         or 0 if the script is unknown
    113      */
    114     uint32_t getFirstPrimaryForGroup(int32_t script) const;
    115 
    116     /**
    117      * Returns the last primary for the script's reordering group.
    118      * @return the last primary of the group
    119      *         (not an actual root collator primary weight),
    120      *         or 0 if the script is unknown
    121      */
    122     uint32_t getLastPrimaryForGroup(int32_t script) const;
    123 
    124     /**
    125      * Finds the reordering group which contains the primary weight.
    126      * @return the first script of the group, or -1 if the weight is beyond the last group
    127      */
    128     int32_t getGroupForPrimary(uint32_t p) const;
    129 
    130     int32_t getEquivalentScripts(int32_t script,
    131                                  int32_t dest[], int32_t capacity, UErrorCode &errorCode) const;
    132 
    133     /**
    134      * Writes the permutation table for the given reordering of scripts and groups,
    135      * mapping from default-order primary-weight lead bytes to reordered lead bytes.
    136      * The caller checks for illegal arguments and
    137      * takes care of [DEFAULT] and memory allocation.
    138      */
    139     void makeReorderTable(const int32_t *reorder, int32_t length,
    140                           uint8_t table[256], UErrorCode &errorCode) const;
    141 
    142     /** @see jamoCE32s */
    143     static const int32_t JAMO_CE32S_LENGTH = 19 + 21 + 27;
    144 
    145     /** Main lookup trie. */
    146     const UTrie2 *trie;
    147     /**
    148      * Array of CE32 values.
    149      * At index 0 there must be CE32(U+0000)
    150      * to support U+0000's special-tag for NUL-termination handling.
    151      */
    152     const uint32_t *ce32s;
    153     /** Array of CE values for expansions and OFFSET_TAG. */
    154     const int64_t *ces;
    155     /** Array of prefix and contraction-suffix matching data. */
    156     const UChar *contexts;
    157     /** Base collation data, or NULL if this data itself is a base. */
    158     const CollationData *base;
    159     /**
    160      * Simple array of JAMO_CE32S_LENGTH=19+21+27 CE32s, one per canonical Jamo L/V/T.
    161      * They are normally simple CE32s, rarely expansions.
    162      * For fast handling of HANGUL_TAG.
    163      */
    164     const uint32_t *jamoCE32s;
    165     const Normalizer2Impl &nfcImpl;
    166     /** The single-byte primary weight (xx000000) for numeric collation. */
    167     uint32_t numericPrimary;
    168 
    169     int32_t ce32sLength;
    170     int32_t cesLength;
    171     int32_t contextsLength;
    172 
    173     /** 256 flags for which primary-weight lead bytes are compressible. */
    174     const UBool *compressibleBytes;
    175     /**
    176      * Set of code points that are unsafe for starting string comparison after an identical prefix,
    177      * or in backwards CE iteration.
    178      */
    179     const UnicodeSet *unsafeBackwardSet;
    180 
    181     /**
    182      * Fast Latin table for common-Latin-text string comparisons.
    183      * Data structure see class CollationFastLatin.
    184      */
    185     const uint16_t *fastLatinTable;
    186     int32_t fastLatinTableLength;
    187 
    188     /**
    189      * Data for scripts and reordering groups.
    190      * Uses include building a reordering permutation table and
    191      * providing script boundaries to AlphabeticIndex.
    192      *
    193      * This data is a sorted list of primary-weight lead byte ranges (reordering groups),
    194      * each with a list of pairs sorted in base collation order;
    195      * each pair contains a script/reorder code and the lowest primary weight for that script.
    196      *
    197      * Data structure:
    198      * - Each reordering group is encoded in n+2 16-bit integers.
    199      *   - First integer:
    200      *     Bits 15..8: First byte of the reordering group's range.
    201      *     Bits  7..0: Last byte of the reordering group's range.
    202      *   - Second integer:
    203      *     Length n of the list of script/reordering codes.
    204      *   - Each further integer is a script or reordering code.
    205      */
    206     const uint16_t *scripts;
    207     int32_t scriptsLength;
    208 
    209     /**
    210      * Collation elements in the root collator.
    211      * Used by the CollationRootElements class. The data structure is described there.
    212      * NULL in a tailoring.
    213      */
    214     const uint32_t *rootElements;
    215     int32_t rootElementsLength;
    216 
    217 private:
    218     int32_t findScript(int32_t script) const;
    219 };
    220 
    221 U_NAMESPACE_END
    222 
    223 #endif  // !UCONFIG_NO_COLLATION
    224 #endif  // __COLLATIONDATA_H__
    225