Home | History | Annotate | Download | only in common
      1 /*
      2 *******************************************************************************
      3 * Copyright (C) 2014, International Business Machines
      4 * Corporation and others.  All Rights Reserved.
      5 *******************************************************************************
      6 * dictionarydata.h
      7 *
      8 * created on: 2012may31
      9 * created by: Markus W. Scherer & Maxime Serrano
     10 */
     11 
     12 #ifndef __DICTIONARYDATA_H__
     13 #define __DICTIONARYDATA_H__
     14 
     15 #include "unicode/utypes.h"
     16 
     17 #if !UCONFIG_NO_BREAK_ITERATION
     18 
     19 #include "unicode/utext.h"
     20 #include "unicode/udata.h"
     21 #include "udataswp.h"
     22 #include "unicode/uobject.h"
     23 #include "unicode/ustringtrie.h"
     24 
     25 U_NAMESPACE_BEGIN
     26 
     27 class UCharsTrie;
     28 class BytesTrie;
     29 
     30 class U_COMMON_API DictionaryData : public UMemory {
     31 public:
     32     static const int32_t TRIE_TYPE_BYTES; // = 0;
     33     static const int32_t TRIE_TYPE_UCHARS; // = 1;
     34     static const int32_t TRIE_TYPE_MASK; // = 7;
     35     static const int32_t TRIE_HAS_VALUES; // = 8;
     36 
     37     static const int32_t TRANSFORM_NONE; // = 0;
     38     static const int32_t TRANSFORM_TYPE_OFFSET; // = 0x1000000;
     39     static const int32_t TRANSFORM_TYPE_MASK; // = 0x7f000000;
     40     static const int32_t TRANSFORM_OFFSET_MASK; // = 0x1fffff;
     41 
     42     enum {
     43         // Byte offsets from the start of the data, after the generic header.
     44         IX_STRING_TRIE_OFFSET,
     45         IX_RESERVED1_OFFSET,
     46         IX_RESERVED2_OFFSET,
     47         IX_TOTAL_SIZE,
     48 
     49         // Trie type: TRIE_HAS_VALUES | TRIE_TYPE_BYTES etc.
     50         IX_TRIE_TYPE,
     51         // Transform specification: TRANSFORM_TYPE_OFFSET | 0xe00 etc.
     52         IX_TRANSFORM,
     53 
     54         IX_RESERVED6,
     55         IX_RESERVED7,
     56         IX_COUNT
     57     };
     58 };
     59 
     60 /**
     61  * Wrapper class around generic dictionaries, implementing matches().
     62  * getType() should return a TRIE_TYPE_??? constant from DictionaryData.
     63  *
     64  * All implementations of this interface must be thread-safe if they are to be used inside of the
     65  * dictionary-based break iteration code.
     66  */
     67 class U_COMMON_API DictionaryMatcher : public UMemory {
     68 public:
     69     DictionaryMatcher() {};
     70     virtual ~DictionaryMatcher();
     71     // this should emulate CompactTrieDictionary::matches()
     72     /*  @param text      The text in which to look for matching words. Matching begins
     73      *                   at the current position of the UText.
     74      *  @param maxLength The max length of match to consider. Units are the native indexing
     75      *                   units of the UText.
     76      *  @param limit     Capacity of output arrays, which is also the maximum number of
     77      *                   matching words to be found.
     78      *  @param lengths   output array, filled with the lengths of the matches, in order,
     79      *                   from shortest to longest. Lengths are in native indexing units
     80      *                   of the UText. May be NULL.
     81      *  @param cpLengths output array, filled with the lengths of the matches, in order,
     82      *                   from shortest to longest. Lengths are the number of Unicode code points.
     83      *                   May be NULL.
     84      *  @param values    Output array, filled with the values associated with the words found.
     85      *                   May be NULL.
     86      *  @param prefix    Output parameter, the code point length of the prefix match, even if that
     87      *                   prefix didn't lead to a complete word. Will always be >= the cpLength
     88      *                   of the longest complete word matched. May be NULL.
     89      *  @return          Number of matching words found.
     90      */
     91     virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit,
     92                             int32_t *lengths, int32_t *cpLengths, int32_t *values,
     93                             int32_t *prefix) const = 0;
     94 
     95     /** @return DictionaryData::TRIE_TYPE_XYZ */
     96     virtual int32_t getType() const = 0;
     97 };
     98 
     99 // Implementation of the DictionaryMatcher interface for a UCharsTrie dictionary
    100 class U_COMMON_API UCharsDictionaryMatcher : public DictionaryMatcher {
    101 public:
    102     // constructs a new UCharsDictionaryMatcher.
    103     // The UDataMemory * will be closed on this object's destruction.
    104     UCharsDictionaryMatcher(const UChar *c, UDataMemory *f) : characters(c), file(f) { }
    105     virtual ~UCharsDictionaryMatcher();
    106     virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit,
    107                             int32_t *lengths, int32_t *cpLengths, int32_t *values,
    108                             int32_t *prefix) const;
    109     virtual int32_t getType() const;
    110 private:
    111     const UChar *characters;
    112     UDataMemory *file;
    113 };
    114 
    115 // Implementation of the DictionaryMatcher interface for a BytesTrie dictionary
    116 class U_COMMON_API BytesDictionaryMatcher : public DictionaryMatcher {
    117 public:
    118     // constructs a new BytesTrieDictionaryMatcher
    119     // the transform constant should be the constant read from the file, not a masked version!
    120     // the UDataMemory * fed in here will be closed on this object's destruction
    121     BytesDictionaryMatcher(const char *c, int32_t t, UDataMemory *f)
    122             : characters(c), transformConstant(t), file(f) { }
    123     virtual ~BytesDictionaryMatcher();
    124     virtual int32_t matches(UText *text, int32_t maxLength, int32_t limit,
    125                             int32_t *lengths, int32_t *cpLengths, int32_t *values,
    126                             int32_t *prefix) const;
    127     virtual int32_t getType() const;
    128 private:
    129     UChar32 transform(UChar32 c) const;
    130 
    131     const char *characters;
    132     int32_t transformConstant;
    133     UDataMemory *file;
    134 };
    135 
    136 U_NAMESPACE_END
    137 
    138 U_CAPI int32_t U_EXPORT2
    139 udict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, UErrorCode *pErrorCode);
    140 
    141 /**
    142  * Format of dictionary .dict data files.
    143  * Format version 1.0.
    144  *
    145  * A dictionary .dict data file contains a byte-serialized BytesTrie or
    146  * a UChars-serialized UCharsTrie.
    147  * Such files are used in dictionary-based break iteration (DBBI).
    148  *
    149  * For a BytesTrie, a transformation type is specified for
    150  * transforming Unicode strings into byte sequences.
    151  *
    152  * A .dict file begins with a standard ICU data file header
    153  * (DataHeader, see ucmndata.h and unicode/udata.h).
    154  * The UDataInfo.dataVersion field is currently unused (set to 0.0.0.0).
    155  *
    156  * After the header, the file contains the following parts.
    157  * Constants are defined in the DictionaryData class.
    158  *
    159  * For the data structure of BytesTrie & UCharsTrie see
    160  * http://site.icu-project.org/design/struct/tries
    161  * and the bytestrie.h and ucharstrie.h header files.
    162  *
    163  * int32_t indexes[indexesLength]; -- indexesLength=indexes[IX_STRING_TRIE_OFFSET]/4;
    164  *
    165  *      The first four indexes are byte offsets in ascending order.
    166  *      Each byte offset marks the start of the next part in the data file,
    167  *      and the end of the previous one.
    168  *      When two consecutive byte offsets are the same, then the corresponding part is empty.
    169  *      Byte offsets are offsets from after the header,
    170  *      that is, from the beginning of the indexes[].
    171  *      Each part starts at an offset with proper alignment for its data.
    172  *      If necessary, the previous part may include padding bytes to achieve this alignment.
    173  *
    174  *      trieType=indexes[IX_TRIE_TYPE] defines the trie type.
    175  *      transform=indexes[IX_TRANSFORM] defines the Unicode-to-bytes transformation.
    176  *          If the transformation type is TRANSFORM_TYPE_OFFSET,
    177  *          then the lower 21 bits contain the offset code point.
    178  *          Each code point c is mapped to byte b = (c - offset).
    179  *          Code points outside the range offset..(offset+0xff) cannot be mapped
    180  *          and do not occur in the dictionary.
    181  *
    182  * stringTrie; -- a serialized BytesTrie or UCharsTrie
    183  *
    184  *      The dictionary maps strings to specific values (TRIE_HAS_VALUES bit set in trieType),
    185  *      or it maps all strings to 0 (TRIE_HAS_VALUES bit not set).
    186  */
    187 
    188 #endif  /* !UCONFIG_NO_BREAK_ITERATION */
    189 #endif  /* __DICTIONARYDATA_H__ */
    190