Home | History | Annotate | Download | only in src 
      1 /*
      2  * Copyright (C) 2009 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #ifndef LATINIME_DICTIONARY_H
     18 #define LATINIME_DICTIONARY_H
     19 
     20 #include "basechars.h"
     21 #include "bigram_dictionary.h"
     22 #include "char_utils.h"
     23 #include "defines.h"
     24 #include "proximity_info.h"
     25 #include "unigram_dictionary.h"
     26 
     27 namespace latinime {
     28 
     29 class Dictionary {
     30 public:
     31     Dictionary(void *dict, int dictSize, int mmapFd, int dictBufAdjust, int typedLetterMultipler,
     32             int fullWordMultiplier, int maxWordLength, int maxWords, int maxAlternatives);
     33     int getSuggestions(ProximityInfo *proximityInfo, int *xcoordinates, int *ycoordinates,
     34             int *codes, int codesSize, int flags, unsigned short *outWords, int *frequencies) {
     35         return mUnigramDictionary->getSuggestions(proximityInfo, xcoordinates, ycoordinates, codes,
     36                 codesSize, flags, outWords, frequencies);
     37     }
     38 
     39     // TODO: Call mBigramDictionary instead of mUnigramDictionary
     40     int getBigrams(unsigned short *word, int length, int *codes, int codesSize,
     41             unsigned short *outWords, int *frequencies, int maxWordLength, int maxBigrams,
     42             int maxAlternatives) {
     43         return mBigramDictionary->getBigrams(word, length, codes, codesSize, outWords, frequencies,
     44                 maxWordLength, maxBigrams, maxAlternatives);
     45     }
     46 
     47     bool isValidWord(unsigned short *word, int length);
     48     void *getDict() { return (void *)mDict; }
     49     int getDictSize() { return mDictSize; }
     50     int getMmapFd() { return mMmapFd; }
     51     int getDictBufAdjust() { return mDictBufAdjust; }
     52     ~Dictionary();
     53 
     54     // public static utility methods
     55     // static inline methods should be defined in the header file
     56     static unsigned short getChar(const unsigned char *dict, int *pos);
     57     static int getCount(const unsigned char *dict, int *pos);
     58     static bool getTerminal(const unsigned char *dict, int *pos);
     59     static int getAddress(const unsigned char *dict, int *pos);
     60     static int getFreq(const unsigned char *dict, const bool isLatestDictVersion, int *pos);
     61     static int wideStrLen(unsigned short *str);
     62     // returns next sibling's position
     63     static int setDictionaryValues(const unsigned char *dict, const bool isLatestDictVersion,
     64             const int pos, unsigned short *c, int *childrenPosition,
     65             bool *terminal, int *freq);
     66     static inline unsigned short toBaseLowerCase(unsigned short c);
     67 
     68 private:
     69     bool hasBigram();
     70 
     71     const unsigned char *mDict;
     72 
     73     // Used only for the mmap version of dictionary loading, but we use these as dummy variables
     74     // also for the malloc version.
     75     const int mDictSize;
     76     const int mMmapFd;
     77     const int mDictBufAdjust;
     78 
     79     const bool IS_LATEST_DICT_VERSION;
     80     UnigramDictionary *mUnigramDictionary;
     81     BigramDictionary *mBigramDictionary;
     82 };
     83 
     84 // public static utility methods
     85 // static inline methods should be defined in the header file
     86 inline unsigned short Dictionary::getChar(const unsigned char *dict, int *pos) {
     87     unsigned short ch = (unsigned short) (dict[(*pos)++] & 0xFF);
     88     // If the code is 255, then actual 16 bit code follows (in big endian)
     89     if (ch == 0xFF) {
     90         ch = ((dict[*pos] & 0xFF) << 8) | (dict[*pos + 1] & 0xFF);
     91         (*pos) += 2;
     92     }
     93     return ch;
     94 }
     95 
     96 inline int Dictionary::getCount(const unsigned char *dict, int *pos) {
     97     return dict[(*pos)++] & 0xFF;
     98 }
     99 
    100 inline bool Dictionary::getTerminal(const unsigned char *dict, int *pos) {
    101     return (dict[*pos] & FLAG_TERMINAL_MASK) > 0;
    102 }
    103 
    104 inline int Dictionary::getAddress(const unsigned char *dict, int *pos) {
    105     int address = 0;
    106     if ((dict[*pos] & FLAG_ADDRESS_MASK) == 0) {
    107         *pos += 1;
    108     } else {
    109         address += (dict[*pos] & (ADDRESS_MASK >> 16)) << 16;
    110         address += (dict[*pos + 1] & 0xFF) << 8;
    111         address += (dict[*pos + 2] & 0xFF);
    112         *pos += 3;
    113     }
    114     return address;
    115 }
    116 
    117 inline int Dictionary::getFreq(const unsigned char *dict,
    118         const bool isLatestDictVersion, int *pos) {
    119     int freq = dict[(*pos)++] & 0xFF;
    120     if (isLatestDictVersion) {
    121         // skipping bigram
    122         int bigramExist = (dict[*pos] & FLAG_BIGRAM_READ);
    123         if (bigramExist > 0) {
    124             int nextBigramExist = 1;
    125             while (nextBigramExist > 0) {
    126                 (*pos) += 3;
    127                 nextBigramExist = (dict[(*pos)++] & FLAG_BIGRAM_CONTINUED);
    128             }
    129         } else {
    130             (*pos)++;
    131         }
    132     }
    133     return freq;
    134 }
    135 
    136 inline int Dictionary::wideStrLen(unsigned short *str) {
    137     if (!str) return 0;
    138     unsigned short *end = str;
    139     while (*end)
    140         end++;
    141     return end - str;
    142 }
    143 
    144 inline int Dictionary::setDictionaryValues(const unsigned char *dict,
    145         const bool isLatestDictVersion, const int pos, unsigned short *c,int *childrenPosition,
    146         bool *terminal, int *freq) {
    147     int position = pos;
    148     // -- at char
    149     *c = Dictionary::getChar(dict, &position);
    150     // -- at flag/add
    151     *terminal = Dictionary::getTerminal(dict, &position);
    152     *childrenPosition = Dictionary::getAddress(dict, &position);
    153     // -- after address or flag
    154     *freq = (*terminal) ? Dictionary::getFreq(dict, isLatestDictVersion, &position) : 1;
    155     // returns next sibling's position
    156     return position;
    157 }
    158 
    159 
    160 inline unsigned short Dictionary::toBaseLowerCase(unsigned short c) {
    161     if (c < sizeof(BASE_CHARS) / sizeof(BASE_CHARS[0])) {
    162         c = BASE_CHARS[c];
    163     }
    164     if (c >='A' && c <= 'Z') {
    165         c |= 32;
    166     } else if (c > 127) {
    167         c = latin_tolower(c);
    168     }
    169     return c;
    170 }
    171 
    172 } // namespace latinime
    173 
    174 #endif // LATINIME_DICTIONARY_H
    175