Home | History | Annotate | Download | only in i18n 
      1 //  2016 and later: Unicode, Inc. and others.
      2 // License & terms of use: http://www.unicode.org/copyright.html
      3 /*
      4 *******************************************************************************
      5 * Copyright (C) 2012-2014, International Business Machines
      6 * Corporation and others.  All Rights Reserved.
      7 *******************************************************************************
      8 * collationdatabuilder.h
      9 *
     10 * created on: 2012apr01
     11 * created by: Markus W. Scherer
     12 */
     13 
     14 #ifndef __COLLATIONDATABUILDER_H__
     15 #define __COLLATIONDATABUILDER_H__
     16 
     17 #include "unicode/utypes.h"
     18 
     19 #if !UCONFIG_NO_COLLATION
     20 
     21 #include "unicode/uniset.h"
     22 #include "unicode/unistr.h"
     23 #include "unicode/uversion.h"
     24 #include "collation.h"
     25 #include "collationdata.h"
     26 #include "collationsettings.h"
     27 #include "normalizer2impl.h"
     28 #include "utrie2.h"
     29 #include "uvectr32.h"
     30 #include "uvectr64.h"
     31 #include "uvector.h"
     32 
     33 U_NAMESPACE_BEGIN
     34 
     35 struct ConditionalCE32;
     36 
     37 class CollationFastLatinBuilder;
     38 class CopyHelper;
     39 class DataBuilderCollationIterator;
     40 class UCharsTrieBuilder;
     41 
     42 /**
     43  * Low-level CollationData builder.
     44  * Takes (character, CE) pairs and builds them into runtime data structures.
     45  * Supports characters with context prefixes and contraction suffixes.
     46  */
     47 class U_I18N_API CollationDataBuilder : public UObject {
     48 public:
     49     /**
     50      * Collation element modifier. Interface class for a modifier
     51      * that changes a tailoring builder's temporary CEs to final CEs.
     52      * Called for every non-special CE32 and every expansion CE.
     53      */
     54     class CEModifier : public UObject {
     55     public:
     56         virtual ~CEModifier();
     57         /** Returns a new CE to replace the non-special input CE32, or else Collation::NO_CE. */
     58         virtual int64_t modifyCE32(uint32_t ce32) const = 0;
     59         /** Returns a new CE to replace the input CE, or else Collation::NO_CE. */
     60         virtual int64_t modifyCE(int64_t ce) const = 0;
     61     };
     62 
     63     CollationDataBuilder(UErrorCode &errorCode);
     64 
     65     virtual ~CollationDataBuilder();
     66 
     67     void initForTailoring(const CollationData *b, UErrorCode &errorCode);
     68 
     69     virtual UBool isCompressibleLeadByte(uint32_t b) const;
     70 
     71     inline UBool isCompressiblePrimary(uint32_t p) const {
     72         return isCompressibleLeadByte(p >> 24);
     73     }
     74 
     75     /**
     76      * @return TRUE if this builder has mappings (e.g., add() has been called)
     77      */
     78     UBool hasMappings() const { return modified; }
     79 
     80     /**
     81      * @return TRUE if c has CEs in this builder
     82      */
     83     UBool isAssigned(UChar32 c) const;
     84 
     85     /**
     86      * @return the three-byte primary if c maps to a single such CE and has no context data,
     87      * otherwise returns 0.
     88      */
     89     uint32_t getLongPrimaryIfSingleCE(UChar32 c) const;
     90 
     91     /**
     92      * @return the single CE for c.
     93      * Sets an error code if c does not have a single CE.
     94      */
     95     int64_t getSingleCE(UChar32 c, UErrorCode &errorCode) const;
     96 
     97     void add(const UnicodeString &prefix, const UnicodeString &s,
     98              const int64_t ces[], int32_t cesLength,
     99              UErrorCode &errorCode);
    100 
    101     /**
    102      * Encodes the ces as either the returned ce32 by itself,
    103      * or by storing an expansion, with the returned ce32 referring to that.
    104      *
    105      * add(p, s, ces, cesLength) = addCE32(p, s, encodeCEs(ces, cesLength))
    106      */
    107     virtual uint32_t encodeCEs(const int64_t ces[], int32_t cesLength, UErrorCode &errorCode);
    108     void addCE32(const UnicodeString &prefix, const UnicodeString &s,
    109                  uint32_t ce32, UErrorCode &errorCode);
    110 
    111     /**
    112      * Sets three-byte-primary CEs for a range of code points in code point order,
    113      * if it is worth doing; otherwise no change is made.
    114      * None of the code points in the range should have complex mappings so far
    115      * (expansions/contractions/prefixes).
    116      * @param start first code point
    117      * @param end last code point (inclusive)
    118      * @param primary primary weight for 'start'
    119      * @param step per-code point primary-weight increment
    120      * @param errorCode ICU in/out error code
    121      * @return TRUE if an OFFSET_TAG range was used for start..end
    122      */
    123     UBool maybeSetPrimaryRange(UChar32 start, UChar32 end,
    124                                uint32_t primary, int32_t step,
    125                                UErrorCode &errorCode);
    126 
    127     /**
    128      * Sets three-byte-primary CEs for a range of code points in code point order.
    129      * Sets range values if that is worth doing, or else individual values.
    130      * None of the code points in the range should have complex mappings so far
    131      * (expansions/contractions/prefixes).
    132      * @param start first code point
    133      * @param end last code point (inclusive)
    134      * @param primary primary weight for 'start'
    135      * @param step per-code point primary-weight increment
    136      * @param errorCode ICU in/out error code
    137      * @return the next primary after 'end': start primary incremented by ((end-start)+1)*step
    138      */
    139     uint32_t setPrimaryRangeAndReturnNext(UChar32 start, UChar32 end,
    140                                           uint32_t primary, int32_t step,
    141                                           UErrorCode &errorCode);
    142 
    143     /**
    144      * Copies all mappings from the src builder, with modifications.
    145      * This builder here must not be built yet, and should be empty.
    146      */
    147     void copyFrom(const CollationDataBuilder &src, const CEModifier &modifier,
    148                   UErrorCode &errorCode);
    149 
    150     void optimize(const UnicodeSet &set, UErrorCode &errorCode);
    151     void suppressContractions(const UnicodeSet &set, UErrorCode &errorCode);
    152 
    153     void enableFastLatin() { fastLatinEnabled = TRUE; }
    154     virtual void build(CollationData &data, UErrorCode &errorCode);
    155 
    156     /**
    157      * Looks up CEs for s and appends them to the ces array.
    158      * Does not handle normalization: s should be in FCD form.
    159      *
    160      * Does not write completely ignorable CEs.
    161      * Does not write beyond Collation::MAX_EXPANSION_LENGTH.
    162      *
    163      * @return incremented cesLength
    164      */
    165     int32_t getCEs(const UnicodeString &s, int64_t ces[], int32_t cesLength);
    166     int32_t getCEs(const UnicodeString &prefix, const UnicodeString &s,
    167                    int64_t ces[], int32_t cesLength);
    168 
    169 protected:
    170     friend class CopyHelper;
    171     friend class DataBuilderCollationIterator;
    172 
    173     uint32_t getCE32FromOffsetCE32(UBool fromBase, UChar32 c, uint32_t ce32) const;
    174 
    175     int32_t addCE(int64_t ce, UErrorCode &errorCode);
    176     int32_t addCE32(uint32_t ce32, UErrorCode &errorCode);
    177     int32_t addConditionalCE32(const UnicodeString &context, uint32_t ce32, UErrorCode &errorCode);
    178 
    179     inline ConditionalCE32 *getConditionalCE32(int32_t index) const {
    180         return static_cast<ConditionalCE32 *>(conditionalCE32s[index]);
    181     }
    182     inline ConditionalCE32 *getConditionalCE32ForCE32(uint32_t ce32) const {
    183         return getConditionalCE32(Collation::indexFromCE32(ce32));
    184     }
    185 
    186     static uint32_t makeBuilderContextCE32(int32_t index) {
    187         return Collation::makeCE32FromTagAndIndex(Collation::BUILDER_DATA_TAG, index);
    188     }
    189     static inline UBool isBuilderContextCE32(uint32_t ce32) {
    190         return Collation::hasCE32Tag(ce32, Collation::BUILDER_DATA_TAG);
    191     }
    192 
    193     static uint32_t encodeOneCEAsCE32(int64_t ce);
    194     uint32_t encodeOneCE(int64_t ce, UErrorCode &errorCode);
    195     uint32_t encodeExpansion(const int64_t ces[], int32_t length, UErrorCode &errorCode);
    196     uint32_t encodeExpansion32(const int32_t newCE32s[], int32_t length, UErrorCode &errorCode);
    197 
    198     uint32_t copyFromBaseCE32(UChar32 c, uint32_t ce32, UBool withContext, UErrorCode &errorCode);
    199     /**
    200      * Copies base contractions to a list of ConditionalCE32.
    201      * Sets cond->next to the index of the first new item
    202      * and returns the index of the last new item.
    203      */
    204     int32_t copyContractionsFromBaseCE32(UnicodeString &context, UChar32 c, uint32_t ce32,
    205                                          ConditionalCE32 *cond, UErrorCode &errorCode);
    206 
    207     UBool getJamoCE32s(uint32_t jamoCE32s[], UErrorCode &errorCode);
    208     void setDigitTags(UErrorCode &errorCode);
    209     void setLeadSurrogates(UErrorCode &errorCode);
    210 
    211     void buildMappings(CollationData &data, UErrorCode &errorCode);
    212 
    213     void clearContexts();
    214     void buildContexts(UErrorCode &errorCode);
    215     uint32_t buildContext(ConditionalCE32 *head, UErrorCode &errorCode);
    216     int32_t addContextTrie(uint32_t defaultCE32, UCharsTrieBuilder &trieBuilder,
    217                            UErrorCode &errorCode);
    218 
    219     void buildFastLatinTable(CollationData &data, UErrorCode &errorCode);
    220 
    221     int32_t getCEs(const UnicodeString &s, int32_t start, int64_t ces[], int32_t cesLength);
    222 
    223     static UChar32 jamoCpFromIndex(int32_t i) {
    224         // 0 <= i < CollationData::JAMO_CE32S_LENGTH = 19 + 21 + 27
    225         if(i < Hangul::JAMO_L_COUNT) { return Hangul::JAMO_L_BASE + i; }
    226         i -= Hangul::JAMO_L_COUNT;
    227         if(i < Hangul::JAMO_V_COUNT) { return Hangul::JAMO_V_BASE + i; }
    228         i -= Hangul::JAMO_V_COUNT;
    229         // i < 27
    230         return Hangul::JAMO_T_BASE + 1 + i;
    231     }
    232 
    233     /** @see Collation::BUILDER_DATA_TAG */
    234     static const uint32_t IS_BUILDER_JAMO_CE32 = 0x100;
    235 
    236     const Normalizer2Impl &nfcImpl;
    237     const CollationData *base;
    238     const CollationSettings *baseSettings;
    239     UTrie2 *trie;
    240     UVector32 ce32s;
    241     UVector64 ce64s;
    242     UVector conditionalCE32s;  // vector of ConditionalCE32
    243     // Characters that have context (prefixes or contraction suffixes).
    244     UnicodeSet contextChars;
    245     // Serialized UCharsTrie structures for finalized contexts.
    246     UnicodeString contexts;
    247     UnicodeSet unsafeBackwardSet;
    248     UBool modified;
    249 
    250     UBool fastLatinEnabled;
    251     CollationFastLatinBuilder *fastLatinBuilder;
    252 
    253     DataBuilderCollationIterator *collIter;
    254 };
    255 
    256 U_NAMESPACE_END
    257 
    258 #endif  // !UCONFIG_NO_COLLATION
    259 #endif  // __COLLATIONDATABUILDER_H__
    260