1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef LATINIME_DEFINES_H 18 #define LATINIME_DEFINES_H 19 20 #ifdef __GNUC__ 21 #define AK_FORCE_INLINE __attribute__((always_inline)) __inline__ 22 #else // __GNUC__ 23 #define AK_FORCE_INLINE inline 24 #endif // __GNUC__ 25 26 #if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 27 #undef AK_FORCE_INLINE 28 #define AK_FORCE_INLINE inline 29 #endif // defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 30 31 // Must be equal to Constants.Dictionary.MAX_WORD_LENGTH in Java 32 #define MAX_WORD_LENGTH 48 33 // Must be equal to BinaryDictionary.MAX_RESULTS in Java 34 #define MAX_RESULTS 18 35 // Must be equal to ProximityInfo.MAX_PROXIMITY_CHARS_SIZE in Java 36 #define MAX_PROXIMITY_CHARS_SIZE 16 37 #define ADDITIONAL_PROXIMITY_CHAR_DELIMITER_CODE 2 38 39 #if defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 40 #include <android/log.h> 41 #ifndef LOG_TAG 42 #define LOG_TAG "LatinIME: " 43 #endif // LOG_TAG 44 #define AKLOGE(fmt, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, fmt, ##__VA_ARGS__) 45 #define AKLOGI(fmt, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, fmt, ##__VA_ARGS__) 46 47 #define DUMP_RESULT(words, frequencies) do { dumpResult(words, frequencies); } while (0) 48 #define DUMP_WORD(word, length) do { dumpWord(word, length); } while (0) 49 #define INTS_TO_CHARS(input, length, output) do { \ 50 intArrayToCharArray(input, length, output); } while (0) 51 52 // TODO: Support full UTF-8 conversion 53 AK_FORCE_INLINE static int intArrayToCharArray(const int *source, const int sourceSize, 54 char *dest) { 55 int si = 0; 56 int di = 0; 57 while (si < sourceSize && di < MAX_WORD_LENGTH - 1 && 0 != source[si]) { 58 const int codePoint = source[si++]; 59 if (codePoint < 0x7F) { 60 dest[di++] = codePoint; 61 } else if (codePoint < 0x7FF) { 62 dest[di++] = 0xC0 + (codePoint >> 6); 63 dest[di++] = 0x80 + (codePoint & 0x3F); 64 } else if (codePoint < 0xFFFF) { 65 dest[di++] = 0xE0 + (codePoint >> 12); 66 dest[di++] = 0x80 + ((codePoint & 0xFC0) >> 6); 67 dest[di++] = 0x80 + (codePoint & 0x3F); 68 } 69 } 70 dest[di] = 0; 71 return di; 72 } 73 74 static inline void dumpWordInfo(const int *word, const int length, const int rank, 75 const int probability) { 76 static char charBuf[50]; 77 const int N = intArrayToCharArray(word, length, charBuf); 78 if (N > 1) { 79 AKLOGI("%2d [ %s ] (%d)", rank, charBuf, probability); 80 } 81 } 82 83 static inline void dumpResult(const int *outWords, const int *frequencies) { 84 AKLOGI("--- DUMP RESULT ---------"); 85 for (int i = 0; i < MAX_RESULTS; ++i) { 86 dumpWordInfo(&outWords[i * MAX_WORD_LENGTH], MAX_WORD_LENGTH, i, frequencies[i]); 87 } 88 AKLOGI("-------------------------"); 89 } 90 91 static AK_FORCE_INLINE void dumpWord(const int *word, const int length) { 92 static char charBuf[50]; 93 const int N = intArrayToCharArray(word, length, charBuf); 94 if (N > 1) { 95 AKLOGI("[ %s ]", charBuf); 96 } 97 } 98 99 #ifndef __ANDROID__ 100 #include <cassert> 101 #include <execinfo.h> 102 #include <stdlib.h> 103 104 #define DO_ASSERT_TEST 105 #define ASSERT(success) do { if (!(success)) { showStackTrace(); assert(success);} } while (0) 106 #define SHOW_STACK_TRACE do { showStackTrace(); } while (0) 107 108 static inline void showStackTrace() { 109 void *callstack[128]; 110 int i, frames = backtrace(callstack, 128); 111 char **strs = backtrace_symbols(callstack, frames); 112 for (i = 0; i < frames; ++i) { 113 if (i == 0) { 114 AKLOGI("=== Trace ==="); 115 continue; 116 } 117 AKLOGI("%s", strs[i]); 118 } 119 free(strs); 120 } 121 #else // __ANDROID__ 122 #include <cassert> 123 #define DO_ASSERT_TEST 124 #define ASSERT(success) assert(success) 125 #define SHOW_STACK_TRACE 126 #endif // __ANDROID__ 127 128 #else // defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 129 #define AKLOGE(fmt, ...) 130 #define AKLOGI(fmt, ...) 131 #define DUMP_RESULT(words, frequencies) 132 #define DUMP_WORD(word, length) 133 #undef DO_ASSERT_TEST 134 #define ASSERT(success) 135 #define SHOW_STACK_TRACE 136 #define INTS_TO_CHARS(input, length, output) 137 #endif // defined(FLAG_DO_PROFILE) || defined(FLAG_DBG) 138 139 #ifdef FLAG_DO_PROFILE 140 // Profiler 141 #include <time.h> 142 143 #define PROF_BUF_SIZE 100 144 static float profile_buf[PROF_BUF_SIZE]; 145 static float profile_old[PROF_BUF_SIZE]; 146 static unsigned int profile_counter[PROF_BUF_SIZE]; 147 148 #define PROF_RESET prof_reset() 149 #define PROF_COUNT(prof_buf_id) ++profile_counter[prof_buf_id] 150 #define PROF_OPEN do { PROF_RESET; PROF_START(PROF_BUF_SIZE - 1); } while (0) 151 #define PROF_START(prof_buf_id) do { \ 152 PROF_COUNT(prof_buf_id); profile_old[prof_buf_id] = (clock()); } while (0) 153 #define PROF_CLOSE do { PROF_END(PROF_BUF_SIZE - 1); PROF_OUTALL; } while (0) 154 #define PROF_END(prof_buf_id) profile_buf[prof_buf_id] += ((clock()) - profile_old[prof_buf_id]) 155 #define PROF_CLOCKOUT(prof_buf_id) \ 156 AKLOGI("%s : clock is %f", __FUNCTION__, (clock() - profile_old[prof_buf_id])) 157 #define PROF_OUTALL do { AKLOGI("--- %s ---", __FUNCTION__); prof_out(); } while (0) 158 159 static inline void prof_reset(void) { 160 for (int i = 0; i < PROF_BUF_SIZE; ++i) { 161 profile_buf[i] = 0; 162 profile_old[i] = 0; 163 profile_counter[i] = 0; 164 } 165 } 166 167 static inline void prof_out(void) { 168 if (profile_counter[PROF_BUF_SIZE - 1] != 1) { 169 AKLOGI("Error: You must call PROF_OPEN before PROF_CLOSE."); 170 } 171 AKLOGI("Total time is %6.3f ms.", 172 profile_buf[PROF_BUF_SIZE - 1] * 1000.0f / static_cast<float>(CLOCKS_PER_SEC)); 173 float all = 0.0f; 174 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { 175 all += profile_buf[i]; 176 } 177 if (all < 1.0f) all = 1.0f; 178 for (int i = 0; i < PROF_BUF_SIZE - 1; ++i) { 179 if (profile_buf[i] > 0.0f) { 180 AKLOGI("(%d): Used %4.2f%%, %8.4f ms. Called %d times.", 181 i, (profile_buf[i] * 100.0f / all), 182 profile_buf[i] * 1000.0f / static_cast<float>(CLOCKS_PER_SEC), 183 profile_counter[i]); 184 } 185 } 186 } 187 188 #else // FLAG_DO_PROFILE 189 #define PROF_BUF_SIZE 0 190 #define PROF_RESET 191 #define PROF_COUNT(prof_buf_id) 192 #define PROF_OPEN 193 #define PROF_START(prof_buf_id) 194 #define PROF_CLOSE 195 #define PROF_END(prof_buf_id) 196 #define PROF_CLOCK_OUT(prof_buf_id) 197 #define PROF_CLOCKOUT(prof_buf_id) 198 #define PROF_OUTALL 199 200 #endif // FLAG_DO_PROFILE 201 202 #ifdef FLAG_DBG 203 #define DEBUG_DICT true 204 #define DEBUG_DICT_FULL false 205 #define DEBUG_EDIT_DISTANCE false 206 #define DEBUG_SHOW_FOUND_WORD false 207 #define DEBUG_NODE DEBUG_DICT_FULL 208 #define DEBUG_TRACE DEBUG_DICT_FULL 209 #define DEBUG_PROXIMITY_INFO false 210 #define DEBUG_PROXIMITY_CHARS false 211 #define DEBUG_CORRECTION false 212 #define DEBUG_CORRECTION_FREQ false 213 #define DEBUG_WORDS_PRIORITY_QUEUE false 214 #define DEBUG_SAMPLING_POINTS false 215 #define DEBUG_POINTS_PROBABILITY false 216 #define DEBUG_DOUBLE_LETTER false 217 #define DEBUG_CACHE false 218 #define DEBUG_DUMP_ERROR false 219 #define DEBUG_EVALUATE_MOST_PROBABLE_STRING false 220 221 #ifdef FLAG_FULL_DBG 222 #define DEBUG_GEO_FULL true 223 #else 224 #define DEBUG_GEO_FULL false 225 #endif 226 227 #else // FLAG_DBG 228 229 #define DEBUG_DICT false 230 #define DEBUG_DICT_FULL false 231 #define DEBUG_EDIT_DISTANCE false 232 #define DEBUG_SHOW_FOUND_WORD false 233 #define DEBUG_NODE false 234 #define DEBUG_TRACE false 235 #define DEBUG_PROXIMITY_INFO false 236 #define DEBUG_PROXIMITY_CHARS false 237 #define DEBUG_CORRECTION false 238 #define DEBUG_CORRECTION_FREQ false 239 #define DEBUG_WORDS_PRIORITY_QUEUE false 240 #define DEBUG_SAMPLING_POINTS false 241 #define DEBUG_POINTS_PROBABILITY false 242 #define DEBUG_DOUBLE_LETTER false 243 #define DEBUG_CACHE false 244 #define DEBUG_DUMP_ERROR false 245 #define DEBUG_EVALUATE_MOST_PROBABLE_STRING false 246 247 #define DEBUG_GEO_FULL false 248 249 #endif // FLAG_DBG 250 251 #ifndef S_INT_MAX 252 #define S_INT_MAX 2147483647 // ((1 << 31) - 1) 253 #endif 254 #ifndef S_INT_MIN 255 // The literal constant -2147483648 does not work in C prior C90, because 256 // the compiler tries to fit the positive number into an int and then negate it. 257 // GCC warns about this. 258 #define S_INT_MIN (-2147483647 - 1) // -(1 << 31) 259 #endif 260 261 #define M_PI_F 3.14159265f 262 #define MAX_PERCENTILE 100 263 264 // Number of base-10 digits in the largest integer + 1 to leave room for a zero terminator. 265 // As such, this is the maximum number of characters will be needed to represent an int as a 266 // string, including the terminator; this is used as the size of a string buffer large enough to 267 // hold any value that is intended to fit in an integer, e.g. in the code that reads the header 268 // of the binary dictionary where a {key,value} string pair scheme is used. 269 #define LARGEST_INT_DIGIT_COUNT 11 270 271 // Define this to use mmap() for dictionary loading. Undefine to use malloc() instead of mmap(). 272 // We measured and compared performance of both, and found mmap() is fairly good in terms of 273 // loading time, and acceptable even for several initial lookups which involve page faults. 274 #define USE_MMAP_FOR_DICTIONARY 275 276 #define NOT_VALID_WORD (-99) 277 #define NOT_A_CODE_POINT (-1) 278 #define NOT_A_DISTANCE (-1) 279 #define NOT_A_COORDINATE (-1) 280 #define MATCH_CHAR_WITHOUT_DISTANCE_INFO (-2) 281 #define PROXIMITY_CHAR_WITHOUT_DISTANCE_INFO (-3) 282 #define ADDITIONAL_PROXIMITY_CHAR_DISTANCE_INFO (-4) 283 #define NOT_AN_INDEX (-1) 284 #define NOT_A_PROBABILITY (-1) 285 286 #define KEYCODE_SPACE ' ' 287 #define KEYCODE_SINGLE_QUOTE '\'' 288 #define KEYCODE_HYPHEN_MINUS '-' 289 290 #define CALIBRATE_SCORE_BY_TOUCH_COORDINATES true 291 #define SUGGEST_MULTIPLE_WORDS true 292 #define USE_SUGGEST_INTERFACE_FOR_TYPING true 293 #define SUGGEST_INTERFACE_OUTPUT_SCALE 1000000.0f 294 295 // The following "rate"s are used as a multiplier before dividing by 100, so they are in percent. 296 #define WORDS_WITH_MISSING_CHARACTER_DEMOTION_RATE 80 297 #define WORDS_WITH_MISSING_CHARACTER_DEMOTION_START_POS_10X 12 298 #define WORDS_WITH_MISSING_SPACE_CHARACTER_DEMOTION_RATE 58 299 #define WORDS_WITH_MISTYPED_SPACE_DEMOTION_RATE 50 300 #define WORDS_WITH_EXCESSIVE_CHARACTER_DEMOTION_RATE 75 301 #define WORDS_WITH_EXCESSIVE_CHARACTER_OUT_OF_PROXIMITY_DEMOTION_RATE 75 302 #define WORDS_WITH_TRANSPOSED_CHARACTERS_DEMOTION_RATE 70 303 #define FULL_MATCHED_WORDS_PROMOTION_RATE 120 304 #define WORDS_WITH_PROXIMITY_CHARACTER_DEMOTION_RATE 90 305 #define WORDS_WITH_ADDITIONAL_PROXIMITY_CHARACTER_DEMOTION_RATE 70 306 #define WORDS_WITH_MATCH_SKIP_PROMOTION_RATE 105 307 #define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_RATE 148 308 #define WORDS_WITH_JUST_ONE_CORRECTION_PROMOTION_MULTIPLIER 3 309 #define CORRECTION_COUNT_RATE_DEMOTION_RATE_BASE 45 310 #define INPUT_EXCEEDS_OUTPUT_DEMOTION_RATE 70 311 #define FIRST_CHAR_DIFFERENT_DEMOTION_RATE 96 312 #define TWO_WORDS_CAPITALIZED_DEMOTION_RATE 50 313 #define TWO_WORDS_CORRECTION_DEMOTION_BASE 80 314 #define TWO_WORDS_PLUS_OTHER_ERROR_CORRECTION_DEMOTION_DIVIDER 1 315 #define ZERO_DISTANCE_PROMOTION_RATE 110.0f 316 #define NEUTRAL_SCORE_SQUARED_RADIUS 8.0f 317 #define HALF_SCORE_SQUARED_RADIUS 32.0f 318 #define MAX_PROBABILITY 255 319 #define MAX_BIGRAM_ENCODED_PROBABILITY 15 320 321 // Assuming locale strings such as en_US, sr-Latn etc. 322 #define MAX_LOCALE_STRING_LENGTH 10 323 324 // Word limit for sub queues used in WordsPriorityQueuePool. Sub queues are temporary queues used 325 // for better performance. 326 // Holds up to 1 candidate for each word 327 #define SUB_QUEUE_MAX_WORDS 1 328 #define SUB_QUEUE_MAX_COUNT 10 329 #define SUB_QUEUE_MIN_WORD_LENGTH 4 330 // TODO: Extend this limitation 331 #define MULTIPLE_WORDS_SUGGESTION_MAX_WORDS 5 332 // TODO: Remove this limitation 333 #define MULTIPLE_WORDS_SUGGESTION_MAX_WORD_LENGTH 12 334 // TODO: Remove this limitation 335 #define MULTIPLE_WORDS_SUGGESTION_MAX_TOTAL_TRAVERSE_COUNT 45 336 #define MULTIPLE_WORDS_DEMOTION_RATE 80 337 #define MIN_INPUT_LENGTH_FOR_THREE_OR_MORE_WORDS_CORRECTION 6 338 339 #define TWO_WORDS_CORRECTION_WITH_OTHER_ERROR_THRESHOLD 0.35f 340 #define START_TWO_WORDS_CORRECTION_THRESHOLD 0.185f 341 /* heuristic... This should be changed if we change the unit of the probability. */ 342 #define SUPPRESS_SHORT_MULTIPLE_WORDS_THRESHOLD_FREQ (MAX_PROBABILITY * 58 / 100) 343 344 #define MAX_DEPTH_MULTIPLIER 3 345 #define FIRST_WORD_INDEX 0 346 347 // Max value for length, distance and probability which are used in weighting 348 // TODO: Remove 349 #define MAX_VALUE_FOR_WEIGHTING 10000000 350 351 // The max number of the keys in one keyboard layout 352 #define MAX_KEY_COUNT_IN_A_KEYBOARD 64 353 354 // TODO: Reduce this constant if possible; check the maximum number of digraphs in the same 355 // word in the dictionary for languages with digraphs, like German and French 356 #define DEFAULT_MAX_DIGRAPH_SEARCH_DEPTH 5 357 358 #define MIN_USER_TYPED_LENGTH_FOR_MULTIPLE_WORD_SUGGESTION 3 359 360 // TODO: Remove 361 #define MAX_POINTER_COUNT 1 362 #define MAX_POINTER_COUNT_G 2 363 364 // Size, in bytes, of the bloom filter index for bigrams 365 // 128 gives us 1024 buckets. The probability of false positive is (1 - e ** (-kn/m))**k, 366 // where k is the number of hash functions, n the number of bigrams, and m the number of 367 // bits we can test. 368 // At the moment 100 is the maximum number of bigrams for a word with the current 369 // dictionaries, so n = 100. 1024 buckets give us m = 1024. 370 // With 1 hash function, our false positive rate is about 9.3%, which should be enough for 371 // our uses since we are only using this to increase average performance. For the record, 372 // k = 2 gives 3.1% and k = 3 gives 1.6%. With k = 1, making m = 2048 gives 4.8%, 373 // and m = 4096 gives 2.4%. 374 #define BIGRAM_FILTER_BYTE_SIZE 128 375 // Must be smaller than BIGRAM_FILTER_BYTE_SIZE * 8, and preferably prime. 1021 is the largest 376 // prime under 128 * 8. 377 #define BIGRAM_FILTER_MODULO 1021 378 #if BIGRAM_FILTER_BYTE_SIZE * 8 < BIGRAM_FILTER_MODULO 379 #error "BIGRAM_FILTER_MODULO is larger than BIGRAM_FILTER_BYTE_SIZE" 380 #endif 381 382 // Max number of bigram maps (previous word contexts) to be cached. Increasing this number could 383 // improve bigram lookup speed for multi-word suggestions, but at the cost of more memory usage. 384 // Also, there are diminishing returns since the most frequently used bigrams are typically near 385 // the beginning of the input and are thus the first ones to be cached. Note that these bigrams 386 // are reset for each new composing word. 387 #define MAX_CACHED_PREV_WORDS_IN_BIGRAM_MAP 25 388 // Most common previous word contexts currently have 100 bigrams 389 #define DEFAULT_HASH_MAP_SIZE_FOR_EACH_BIGRAM_MAP 100 390 391 template<typename T> AK_FORCE_INLINE const T &min(const T &a, const T &b) { return a < b ? a : b; } 392 template<typename T> AK_FORCE_INLINE const T &max(const T &a, const T &b) { return a > b ? a : b; } 393 394 #define NELEMS(x) (sizeof(x) / sizeof((x)[0])) 395 396 // DEBUG 397 #define INPUTLENGTH_FOR_DEBUG (-1) 398 #define MIN_OUTPUT_INDEX_FOR_DEBUG (-1) 399 400 #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 401 TypeName(const TypeName&); \ 402 void operator=(const TypeName&) 403 404 #define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ 405 TypeName(); \ 406 DISALLOW_COPY_AND_ASSIGN(TypeName) 407 408 // Used as a return value for character comparison 409 typedef enum { 410 // Same char, possibly with different case or accent 411 MATCH_CHAR, 412 // It is a char located nearby on the keyboard 413 PROXIMITY_CHAR, 414 // Additional proximity char which can differ by language. 415 ADDITIONAL_PROXIMITY_CHAR, 416 // It is a substitution char 417 SUBSTITUTION_CHAR, 418 // It is an unrelated char 419 UNRELATED_CHAR, 420 } ProximityType; 421 422 typedef enum { 423 NOT_A_DOUBLE_LETTER, 424 A_DOUBLE_LETTER, 425 A_STRONG_DOUBLE_LETTER 426 } DoubleLetterLevel; 427 428 typedef enum { 429 // Correction for MATCH_CHAR 430 CT_MATCH, 431 // Correction for PROXIMITY_CHAR 432 CT_PROXIMITY, 433 // Correction for ADDITIONAL_PROXIMITY_CHAR 434 CT_ADDITIONAL_PROXIMITY, 435 // Correction for SUBSTITUTION_CHAR 436 CT_SUBSTITUTION, 437 // Skip one omitted letter 438 CT_OMISSION, 439 // Delete an unnecessarily inserted letter 440 CT_INSERTION, 441 // Swap the order of next two touch points 442 CT_TRANSPOSITION, 443 CT_COMPLETION, 444 CT_TERMINAL, 445 // Create new word with space omission 446 CT_NEW_WORD_SPACE_OMITTION, 447 // Create new word with space substitution 448 CT_NEW_WORD_SPACE_SUBSTITUTION, 449 } CorrectionType; 450 451 // ErrorType is mainly decided by CorrectionType but it is also depending on if 452 // the correction has really been performed or not. 453 typedef enum { 454 // Substitution, omission and transposition 455 ET_EDIT_CORRECTION, 456 // Proximity error 457 ET_PROXIMITY_CORRECTION, 458 // Completion 459 ET_COMPLETION, 460 // New word 461 // TODO: Remove. 462 // A new word error should be an edit correction error or a proximity correction error. 463 ET_NEW_WORD, 464 // Treat error as an intentional omission when the CorrectionType is omission and the node can 465 // be intentional omission. 466 ET_INTENTIONAL_OMISSION, 467 // Not treated as an error. Tracked for checking exact match 468 ET_NOT_AN_ERROR 469 } ErrorType; 470 #endif // LATINIME_DEFINES_H 471
