1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.inputmethod.latin; 18 19 import android.content.Context; 20 import android.text.TextUtils; 21 import android.util.Log; 22 import android.view.inputmethod.InputMethodSubtype; 23 24 import com.android.inputmethod.annotations.UsedForTesting; 25 import com.android.inputmethod.keyboard.ProximityInfo; 26 import com.android.inputmethod.latin.PrevWordsInfo.WordInfo; 27 import com.android.inputmethod.latin.SuggestedWords.SuggestedWordInfo; 28 import com.android.inputmethod.latin.personalization.ContextualDictionary; 29 import com.android.inputmethod.latin.personalization.PersonalizationDataChunk; 30 import com.android.inputmethod.latin.personalization.PersonalizationDictionary; 31 import com.android.inputmethod.latin.personalization.UserHistoryDictionary; 32 import com.android.inputmethod.latin.settings.SettingsValuesForSuggestion; 33 import com.android.inputmethod.latin.settings.SpacingAndPunctuations; 34 import com.android.inputmethod.latin.utils.DistracterFilter; 35 import com.android.inputmethod.latin.utils.DistracterFilterCheckingIsInDictionary; 36 import com.android.inputmethod.latin.utils.ExecutorUtils; 37 import com.android.inputmethod.latin.utils.LanguageModelParam; 38 import com.android.inputmethod.latin.utils.SuggestionResults; 39 40 import java.io.File; 41 import java.lang.reflect.InvocationTargetException; 42 import java.lang.reflect.Method; 43 import java.util.ArrayList; 44 import java.util.Arrays; 45 import java.util.HashMap; 46 import java.util.HashSet; 47 import java.util.List; 48 import java.util.Locale; 49 import java.util.Map; 50 import java.util.concurrent.ConcurrentHashMap; 51 import java.util.concurrent.CountDownLatch; 52 import java.util.concurrent.TimeUnit; 53 54 // TODO: Consolidate dictionaries in native code. 55 public class DictionaryFacilitator { 56 public static final String TAG = DictionaryFacilitator.class.getSimpleName(); 57 58 // HACK: This threshold is being used when adding a capitalized entry in the User History 59 // dictionary. 60 private static final int CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT = 140; 61 62 private Dictionaries mDictionaries = new Dictionaries(); 63 private boolean mIsUserDictEnabled = false; 64 private volatile CountDownLatch mLatchForWaitingLoadingMainDictionary = new CountDownLatch(0); 65 // To synchronize assigning mDictionaries to ensure closing dictionaries. 66 private final Object mLock = new Object(); 67 private final DistracterFilter mDistracterFilter; 68 69 private static final String[] DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS = 70 new String[] { 71 Dictionary.TYPE_MAIN, 72 Dictionary.TYPE_USER_HISTORY, 73 Dictionary.TYPE_PERSONALIZATION, 74 Dictionary.TYPE_USER, 75 Dictionary.TYPE_CONTACTS, 76 Dictionary.TYPE_CONTEXTUAL 77 }; 78 79 public static final Map<String, Class<? extends ExpandableBinaryDictionary>> 80 DICT_TYPE_TO_CLASS = new HashMap<>(); 81 82 static { 83 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER_HISTORY, UserHistoryDictionary.class); 84 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_PERSONALIZATION, PersonalizationDictionary.class); 85 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_USER, UserBinaryDictionary.class); 86 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTACTS, ContactsBinaryDictionary.class); 87 DICT_TYPE_TO_CLASS.put(Dictionary.TYPE_CONTEXTUAL, ContextualDictionary.class); 88 } 89 90 private static final String DICT_FACTORY_METHOD_NAME = "getDictionary"; 91 private static final Class<?>[] DICT_FACTORY_METHOD_ARG_TYPES = 92 new Class[] { Context.class, Locale.class, File.class, String.class }; 93 94 private static final String[] SUB_DICT_TYPES = 95 Arrays.copyOfRange(DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS, 1 /* start */, 96 DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS.length); 97 98 /** 99 * Class contains dictionaries for a locale. 100 */ 101 private static class Dictionaries { 102 public final Locale mLocale; 103 private Dictionary mMainDict; 104 public final ConcurrentHashMap<String, ExpandableBinaryDictionary> mSubDictMap = 105 new ConcurrentHashMap<>(); 106 107 public Dictionaries() { 108 mLocale = null; 109 } 110 111 public Dictionaries(final Locale locale, final Dictionary mainDict, 112 final Map<String, ExpandableBinaryDictionary> subDicts) { 113 mLocale = locale; 114 // Main dictionary can be asynchronously loaded. 115 setMainDict(mainDict); 116 for (final Map.Entry<String, ExpandableBinaryDictionary> entry : subDicts.entrySet()) { 117 setSubDict(entry.getKey(), entry.getValue()); 118 } 119 } 120 121 private void setSubDict(final String dictType, final ExpandableBinaryDictionary dict) { 122 if (dict != null) { 123 mSubDictMap.put(dictType, dict); 124 } 125 } 126 127 public void setMainDict(final Dictionary mainDict) { 128 // Close old dictionary if exists. Main dictionary can be assigned multiple times. 129 final Dictionary oldDict = mMainDict; 130 mMainDict = mainDict; 131 if (oldDict != null && mainDict != oldDict) { 132 oldDict.close(); 133 } 134 } 135 136 public Dictionary getDict(final String dictType) { 137 if (Dictionary.TYPE_MAIN.equals(dictType)) { 138 return mMainDict; 139 } else { 140 return getSubDict(dictType); 141 } 142 } 143 144 public ExpandableBinaryDictionary getSubDict(final String dictType) { 145 return mSubDictMap.get(dictType); 146 } 147 148 public boolean hasDict(final String dictType) { 149 if (Dictionary.TYPE_MAIN.equals(dictType)) { 150 return mMainDict != null; 151 } else { 152 return mSubDictMap.containsKey(dictType); 153 } 154 } 155 156 public void closeDict(final String dictType) { 157 final Dictionary dict; 158 if (Dictionary.TYPE_MAIN.equals(dictType)) { 159 dict = mMainDict; 160 } else { 161 dict = mSubDictMap.remove(dictType); 162 } 163 if (dict != null) { 164 dict.close(); 165 } 166 } 167 } 168 169 public interface DictionaryInitializationListener { 170 public void onUpdateMainDictionaryAvailability(boolean isMainDictionaryAvailable); 171 } 172 173 public DictionaryFacilitator() { 174 mDistracterFilter = DistracterFilter.EMPTY_DISTRACTER_FILTER; 175 } 176 177 public DictionaryFacilitator(final DistracterFilter distracterFilter) { 178 mDistracterFilter = distracterFilter; 179 } 180 181 public void updateEnabledSubtypes(final List<InputMethodSubtype> enabledSubtypes) { 182 mDistracterFilter.updateEnabledSubtypes(enabledSubtypes); 183 } 184 185 public Locale getLocale() { 186 return mDictionaries.mLocale; 187 } 188 189 private static ExpandableBinaryDictionary getSubDict(final String dictType, 190 final Context context, final Locale locale, final File dictFile, 191 final String dictNamePrefix) { 192 final Class<? extends ExpandableBinaryDictionary> dictClass = 193 DICT_TYPE_TO_CLASS.get(dictType); 194 if (dictClass == null) { 195 return null; 196 } 197 try { 198 final Method factoryMethod = dictClass.getMethod(DICT_FACTORY_METHOD_NAME, 199 DICT_FACTORY_METHOD_ARG_TYPES); 200 final Object dict = factoryMethod.invoke(null /* obj */, 201 new Object[] { context, locale, dictFile, dictNamePrefix }); 202 return (ExpandableBinaryDictionary) dict; 203 } catch (final NoSuchMethodException | SecurityException | IllegalAccessException 204 | IllegalArgumentException | InvocationTargetException e) { 205 Log.e(TAG, "Cannot create dictionary: " + dictType, e); 206 return null; 207 } 208 } 209 210 public void resetDictionaries(final Context context, final Locale newLocale, 211 final boolean useContactsDict, final boolean usePersonalizedDicts, 212 final boolean forceReloadMainDictionary, 213 final DictionaryInitializationListener listener) { 214 resetDictionariesWithDictNamePrefix(context, newLocale, useContactsDict, 215 usePersonalizedDicts, forceReloadMainDictionary, listener, "" /* dictNamePrefix */); 216 } 217 218 public void resetDictionariesWithDictNamePrefix(final Context context, final Locale newLocale, 219 final boolean useContactsDict, final boolean usePersonalizedDicts, 220 final boolean forceReloadMainDictionary, 221 final DictionaryInitializationListener listener, 222 final String dictNamePrefix) { 223 final boolean localeHasBeenChanged = !newLocale.equals(mDictionaries.mLocale); 224 // We always try to have the main dictionary. Other dictionaries can be unused. 225 final boolean reloadMainDictionary = localeHasBeenChanged || forceReloadMainDictionary; 226 // TODO: Make subDictTypesToUse configurable by resource or a static final list. 227 final HashSet<String> subDictTypesToUse = new HashSet<>(); 228 if (useContactsDict) { 229 subDictTypesToUse.add(Dictionary.TYPE_CONTACTS); 230 } 231 subDictTypesToUse.add(Dictionary.TYPE_USER); 232 if (usePersonalizedDicts) { 233 subDictTypesToUse.add(Dictionary.TYPE_USER_HISTORY); 234 subDictTypesToUse.add(Dictionary.TYPE_PERSONALIZATION); 235 subDictTypesToUse.add(Dictionary.TYPE_CONTEXTUAL); 236 } 237 238 final Dictionary newMainDict; 239 if (reloadMainDictionary) { 240 // The main dictionary will be asynchronously loaded. 241 newMainDict = null; 242 } else { 243 newMainDict = mDictionaries.getDict(Dictionary.TYPE_MAIN); 244 } 245 246 final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>(); 247 for (final String dictType : SUB_DICT_TYPES) { 248 if (!subDictTypesToUse.contains(dictType)) { 249 // This dictionary will not be used. 250 continue; 251 } 252 final ExpandableBinaryDictionary dict; 253 if (!localeHasBeenChanged && mDictionaries.hasDict(dictType)) { 254 // Continue to use current dictionary. 255 dict = mDictionaries.getSubDict(dictType); 256 } else { 257 // Start to use new dictionary. 258 dict = getSubDict(dictType, context, newLocale, null /* dictFile */, 259 dictNamePrefix); 260 } 261 subDicts.put(dictType, dict); 262 } 263 264 // Replace Dictionaries. 265 final Dictionaries newDictionaries = new Dictionaries(newLocale, newMainDict, subDicts); 266 final Dictionaries oldDictionaries; 267 synchronized (mLock) { 268 oldDictionaries = mDictionaries; 269 mDictionaries = newDictionaries; 270 mIsUserDictEnabled = UserBinaryDictionary.isEnabled(context); 271 if (reloadMainDictionary) { 272 asyncReloadMainDictionary(context, newLocale, listener); 273 } 274 } 275 if (listener != null) { 276 listener.onUpdateMainDictionaryAvailability(hasInitializedMainDictionary()); 277 } 278 // Clean up old dictionaries. 279 if (reloadMainDictionary) { 280 oldDictionaries.closeDict(Dictionary.TYPE_MAIN); 281 } 282 for (final String dictType : SUB_DICT_TYPES) { 283 if (localeHasBeenChanged || !subDictTypesToUse.contains(dictType)) { 284 oldDictionaries.closeDict(dictType); 285 } 286 } 287 oldDictionaries.mSubDictMap.clear(); 288 } 289 290 private void asyncReloadMainDictionary(final Context context, final Locale locale, 291 final DictionaryInitializationListener listener) { 292 final CountDownLatch latchForWaitingLoadingMainDictionary = new CountDownLatch(1); 293 mLatchForWaitingLoadingMainDictionary = latchForWaitingLoadingMainDictionary; 294 ExecutorUtils.getExecutor("InitializeBinaryDictionary").execute(new Runnable() { 295 @Override 296 public void run() { 297 final Dictionary mainDict = 298 DictionaryFactory.createMainDictionaryFromManager(context, locale); 299 synchronized (mLock) { 300 if (locale.equals(mDictionaries.mLocale)) { 301 mDictionaries.setMainDict(mainDict); 302 } else { 303 // Dictionary facilitator has been reset for another locale. 304 mainDict.close(); 305 } 306 } 307 if (listener != null) { 308 listener.onUpdateMainDictionaryAvailability(hasInitializedMainDictionary()); 309 } 310 latchForWaitingLoadingMainDictionary.countDown(); 311 } 312 }); 313 } 314 315 @UsedForTesting 316 public void resetDictionariesForTesting(final Context context, final Locale locale, 317 final ArrayList<String> dictionaryTypes, final HashMap<String, File> dictionaryFiles, 318 final Map<String, Map<String, String>> additionalDictAttributes) { 319 Dictionary mainDictionary = null; 320 final Map<String, ExpandableBinaryDictionary> subDicts = new HashMap<>(); 321 322 for (final String dictType : dictionaryTypes) { 323 if (dictType.equals(Dictionary.TYPE_MAIN)) { 324 mainDictionary = DictionaryFactory.createMainDictionaryFromManager(context, locale); 325 } else { 326 final File dictFile = dictionaryFiles.get(dictType); 327 final ExpandableBinaryDictionary dict = getSubDict( 328 dictType, context, locale, dictFile, "" /* dictNamePrefix */); 329 if (additionalDictAttributes.containsKey(dictType)) { 330 dict.clearAndFlushDictionaryWithAdditionalAttributes( 331 additionalDictAttributes.get(dictType)); 332 } 333 if (dict == null) { 334 throw new RuntimeException("Unknown dictionary type: " + dictType); 335 } 336 dict.reloadDictionaryIfRequired(); 337 dict.waitAllTasksForTests(); 338 subDicts.put(dictType, dict); 339 } 340 } 341 mDictionaries = new Dictionaries(locale, mainDictionary, subDicts); 342 } 343 344 public void closeDictionaries() { 345 final Dictionaries dictionaries; 346 synchronized (mLock) { 347 dictionaries = mDictionaries; 348 mDictionaries = new Dictionaries(); 349 } 350 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 351 dictionaries.closeDict(dictType); 352 } 353 mDistracterFilter.close(); 354 } 355 356 @UsedForTesting 357 public ExpandableBinaryDictionary getSubDictForTesting(final String dictName) { 358 return mDictionaries.getSubDict(dictName); 359 } 360 361 // The main dictionary could have been loaded asynchronously. Don't cache the return value 362 // of this method. 363 public boolean hasInitializedMainDictionary() { 364 final Dictionary mainDict = mDictionaries.getDict(Dictionary.TYPE_MAIN); 365 return mainDict != null && mainDict.isInitialized(); 366 } 367 368 public boolean hasPersonalizationDictionary() { 369 return mDictionaries.hasDict(Dictionary.TYPE_PERSONALIZATION); 370 } 371 372 public void flushPersonalizationDictionary() { 373 final ExpandableBinaryDictionary personalizationDict = 374 mDictionaries.getSubDict(Dictionary.TYPE_PERSONALIZATION); 375 if (personalizationDict != null) { 376 personalizationDict.asyncFlushBinaryDictionary(); 377 } 378 } 379 380 public void waitForLoadingMainDictionary(final long timeout, final TimeUnit unit) 381 throws InterruptedException { 382 mLatchForWaitingLoadingMainDictionary.await(timeout, unit); 383 } 384 385 @UsedForTesting 386 public void waitForLoadingDictionariesForTesting(final long timeout, final TimeUnit unit) 387 throws InterruptedException { 388 waitForLoadingMainDictionary(timeout, unit); 389 final Map<String, ExpandableBinaryDictionary> dictMap = mDictionaries.mSubDictMap; 390 for (final ExpandableBinaryDictionary dict : dictMap.values()) { 391 dict.waitAllTasksForTests(); 392 } 393 } 394 395 public boolean isUserDictionaryEnabled() { 396 return mIsUserDictEnabled; 397 } 398 399 public void addWordToUserDictionary(final Context context, final String word) { 400 final Locale locale = getLocale(); 401 if (locale == null) { 402 return; 403 } 404 UserBinaryDictionary.addWordToUserDictionary(context, locale, word); 405 } 406 407 public void addToUserHistory(final String suggestion, final boolean wasAutoCapitalized, 408 final PrevWordsInfo prevWordsInfo, final int timeStampInSeconds, 409 final boolean blockPotentiallyOffensive) { 410 final Dictionaries dictionaries = mDictionaries; 411 final String[] words = suggestion.split(Constants.WORD_SEPARATOR); 412 PrevWordsInfo prevWordsInfoForCurrentWord = prevWordsInfo; 413 for (int i = 0; i < words.length; i++) { 414 final String currentWord = words[i]; 415 final boolean wasCurrentWordAutoCapitalized = (i == 0) ? wasAutoCapitalized : false; 416 addWordToUserHistory(dictionaries, prevWordsInfoForCurrentWord, currentWord, 417 wasCurrentWordAutoCapitalized, timeStampInSeconds, blockPotentiallyOffensive); 418 prevWordsInfoForCurrentWord = 419 prevWordsInfoForCurrentWord.getNextPrevWordsInfo(new WordInfo(currentWord)); 420 } 421 } 422 423 private void addWordToUserHistory(final Dictionaries dictionaries, 424 final PrevWordsInfo prevWordsInfo, final String word, final boolean wasAutoCapitalized, 425 final int timeStampInSeconds, final boolean blockPotentiallyOffensive) { 426 final ExpandableBinaryDictionary userHistoryDictionary = 427 dictionaries.getSubDict(Dictionary.TYPE_USER_HISTORY); 428 if (userHistoryDictionary == null) { 429 return; 430 } 431 final int maxFreq = getFrequency(word); 432 if (maxFreq == 0 && blockPotentiallyOffensive) { 433 return; 434 } 435 final String lowerCasedWord = word.toLowerCase(dictionaries.mLocale); 436 final String secondWord; 437 if (wasAutoCapitalized) { 438 if (isValidWord(word, false /* ignoreCase */) 439 && !isValidWord(lowerCasedWord, false /* ignoreCase */)) { 440 // If the word was auto-capitalized and exists only as a capitalized word in the 441 // dictionary, then we must not downcase it before registering it. For example, 442 // the name of the contacts in start-of-sentence position would come here with the 443 // wasAutoCapitalized flag: if we downcase it, we'd register a lower-case version 444 // of that contact's name which would end up popping in suggestions. 445 secondWord = word; 446 } else { 447 // If however the word is not in the dictionary, or exists as a lower-case word 448 // only, then we consider that was a lower-case word that had been auto-capitalized. 449 secondWord = lowerCasedWord; 450 } 451 } else { 452 // HACK: We'd like to avoid adding the capitalized form of common words to the User 453 // History dictionary in order to avoid suggesting them until the dictionary 454 // consolidation is done. 455 // TODO: Remove this hack when ready. 456 final int lowerCaseFreqInMainDict = dictionaries.hasDict(Dictionary.TYPE_MAIN) ? 457 dictionaries.getDict(Dictionary.TYPE_MAIN).getFrequency(lowerCasedWord) : 458 Dictionary.NOT_A_PROBABILITY; 459 if (maxFreq < lowerCaseFreqInMainDict 460 && lowerCaseFreqInMainDict >= CAPITALIZED_FORM_MAX_PROBABILITY_FOR_INSERT) { 461 // Use lower cased word as the word can be a distracter of the popular word. 462 secondWord = lowerCasedWord; 463 } else { 464 secondWord = word; 465 } 466 } 467 // We demote unrecognized words (frequency < 0, below) by specifying them as "invalid". 468 // We don't add words with 0-frequency (assuming they would be profanity etc.). 469 final boolean isValid = maxFreq > 0; 470 UserHistoryDictionary.addToDictionary(userHistoryDictionary, prevWordsInfo, secondWord, 471 isValid, timeStampInSeconds, 472 new DistracterFilterCheckingIsInDictionary( 473 mDistracterFilter, userHistoryDictionary)); 474 } 475 476 private void removeWord(final String dictName, final String word) { 477 final ExpandableBinaryDictionary dictionary = mDictionaries.getSubDict(dictName); 478 if (dictionary != null) { 479 dictionary.removeUnigramEntryDynamically(word); 480 } 481 } 482 483 public void removeWordFromPersonalizedDicts(final String word) { 484 removeWord(Dictionary.TYPE_USER_HISTORY, word); 485 removeWord(Dictionary.TYPE_PERSONALIZATION, word); 486 removeWord(Dictionary.TYPE_CONTEXTUAL, word); 487 } 488 489 // TODO: Revise the way to fusion suggestion results. 490 public SuggestionResults getSuggestionResults(final WordComposer composer, 491 final PrevWordsInfo prevWordsInfo, final ProximityInfo proximityInfo, 492 final SettingsValuesForSuggestion settingsValuesForSuggestion, final int sessionId) { 493 final Dictionaries dictionaries = mDictionaries; 494 final SuggestionResults suggestionResults = new SuggestionResults( 495 dictionaries.mLocale, SuggestedWords.MAX_SUGGESTIONS, 496 prevWordsInfo.mPrevWordsInfo[0].mIsBeginningOfSentence); 497 final float[] languageWeight = new float[] { Dictionary.NOT_A_LANGUAGE_WEIGHT }; 498 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 499 final Dictionary dictionary = dictionaries.getDict(dictType); 500 if (null == dictionary) continue; 501 final ArrayList<SuggestedWordInfo> dictionarySuggestions = 502 dictionary.getSuggestions(composer, prevWordsInfo, proximityInfo, 503 settingsValuesForSuggestion, sessionId, languageWeight); 504 if (null == dictionarySuggestions) continue; 505 suggestionResults.addAll(dictionarySuggestions); 506 if (null != suggestionResults.mRawSuggestions) { 507 suggestionResults.mRawSuggestions.addAll(dictionarySuggestions); 508 } 509 } 510 return suggestionResults; 511 } 512 513 public boolean isValidWord(final String word, final boolean ignoreCase) { 514 if (TextUtils.isEmpty(word)) { 515 return false; 516 } 517 final Dictionaries dictionaries = mDictionaries; 518 if (dictionaries.mLocale == null) { 519 return false; 520 } 521 final String lowerCasedWord = word.toLowerCase(dictionaries.mLocale); 522 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 523 final Dictionary dictionary = dictionaries.getDict(dictType); 524 // Ideally the passed map would come out of a {@link java.util.concurrent.Future} and 525 // would be immutable once it's finished initializing, but concretely a null test is 526 // probably good enough for the time being. 527 if (null == dictionary) continue; 528 if (dictionary.isValidWord(word) 529 || (ignoreCase && dictionary.isValidWord(lowerCasedWord))) { 530 return true; 531 } 532 } 533 return false; 534 } 535 536 private int getFrequencyInternal(final String word, 537 final boolean isGettingMaxFrequencyOfExactMatches) { 538 if (TextUtils.isEmpty(word)) { 539 return Dictionary.NOT_A_PROBABILITY; 540 } 541 int maxFreq = Dictionary.NOT_A_PROBABILITY; 542 final Dictionaries dictionaries = mDictionaries; 543 for (final String dictType : DICT_TYPES_ORDERED_TO_GET_SUGGESTIONS) { 544 final Dictionary dictionary = dictionaries.getDict(dictType); 545 if (dictionary == null) continue; 546 final int tempFreq; 547 if (isGettingMaxFrequencyOfExactMatches) { 548 tempFreq = dictionary.getMaxFrequencyOfExactMatches(word); 549 } else { 550 tempFreq = dictionary.getFrequency(word); 551 } 552 if (tempFreq >= maxFreq) { 553 maxFreq = tempFreq; 554 } 555 } 556 return maxFreq; 557 } 558 559 public int getFrequency(final String word) { 560 return getFrequencyInternal(word, false /* isGettingMaxFrequencyOfExactMatches */); 561 } 562 563 public int getMaxFrequencyOfExactMatches(final String word) { 564 return getFrequencyInternal(word, true /* isGettingMaxFrequencyOfExactMatches */); 565 } 566 567 private void clearSubDictionary(final String dictName) { 568 final ExpandableBinaryDictionary dictionary = mDictionaries.getSubDict(dictName); 569 if (dictionary != null) { 570 dictionary.clear(); 571 } 572 } 573 574 public void clearUserHistoryDictionary() { 575 clearSubDictionary(Dictionary.TYPE_USER_HISTORY); 576 } 577 578 // This method gets called only when the IME receives a notification to remove the 579 // personalization dictionary. 580 public void clearPersonalizationDictionary() { 581 clearSubDictionary(Dictionary.TYPE_PERSONALIZATION); 582 } 583 584 public void clearContextualDictionary() { 585 clearSubDictionary(Dictionary.TYPE_CONTEXTUAL); 586 } 587 588 public void addEntriesToPersonalizationDictionary( 589 final PersonalizationDataChunk personalizationDataChunk, 590 final SpacingAndPunctuations spacingAndPunctuations, 591 final ExpandableBinaryDictionary.AddMultipleDictionaryEntriesCallback callback) { 592 final ExpandableBinaryDictionary personalizationDict = 593 mDictionaries.getSubDict(Dictionary.TYPE_PERSONALIZATION); 594 if (personalizationDict == null) { 595 if (callback != null) { 596 callback.onFinished(); 597 } 598 return; 599 } 600 final ArrayList<LanguageModelParam> languageModelParams = 601 LanguageModelParam.createLanguageModelParamsFrom( 602 personalizationDataChunk.mTokens, 603 personalizationDataChunk.mTimestampInSeconds, 604 this /* dictionaryFacilitator */, spacingAndPunctuations, 605 new DistracterFilterCheckingIsInDictionary( 606 mDistracterFilter, personalizationDict)); 607 if (languageModelParams == null || languageModelParams.isEmpty()) { 608 if (callback != null) { 609 callback.onFinished(); 610 } 611 return; 612 } 613 personalizationDict.addMultipleDictionaryEntriesDynamically(languageModelParams, callback); 614 } 615 616 public void addPhraseToContextualDictionary(final String[] phrase, final int probability, 617 final int bigramProbabilityForWords, final int bigramProbabilityForPhrases) { 618 final ExpandableBinaryDictionary contextualDict = 619 mDictionaries.getSubDict(Dictionary.TYPE_CONTEXTUAL); 620 if (contextualDict == null) { 621 return; 622 } 623 PrevWordsInfo prevWordsInfo = PrevWordsInfo.BEGINNING_OF_SENTENCE; 624 for (int i = 0; i < phrase.length; i++) { 625 final String[] subPhrase = Arrays.copyOfRange(phrase, i /* start */, phrase.length); 626 final String subPhraseStr = TextUtils.join(Constants.WORD_SEPARATOR, subPhrase); 627 contextualDict.addUnigramEntryWithCheckingDistracter( 628 subPhraseStr, probability, null /* shortcutTarget */, 629 Dictionary.NOT_A_PROBABILITY /* shortcutFreq */, 630 false /* isNotAWord */, false /* isBlacklisted */, 631 BinaryDictionary.NOT_A_VALID_TIMESTAMP, 632 DistracterFilter.EMPTY_DISTRACTER_FILTER); 633 contextualDict.addNgramEntry(prevWordsInfo, subPhraseStr, 634 bigramProbabilityForPhrases, BinaryDictionary.NOT_A_VALID_TIMESTAMP); 635 636 if (i < phrase.length - 1) { 637 contextualDict.addUnigramEntryWithCheckingDistracter( 638 phrase[i], probability, null /* shortcutTarget */, 639 Dictionary.NOT_A_PROBABILITY /* shortcutFreq */, 640 false /* isNotAWord */, false /* isBlacklisted */, 641 BinaryDictionary.NOT_A_VALID_TIMESTAMP, 642 DistracterFilter.EMPTY_DISTRACTER_FILTER); 643 contextualDict.addNgramEntry(prevWordsInfo, phrase[i], 644 bigramProbabilityForWords, BinaryDictionary.NOT_A_VALID_TIMESTAMP); 645 } 646 prevWordsInfo = 647 prevWordsInfo.getNextPrevWordsInfo(new PrevWordsInfo.WordInfo(phrase[i])); 648 } 649 } 650 651 public void dumpDictionaryForDebug(final String dictName) { 652 final ExpandableBinaryDictionary dictToDump = mDictionaries.getSubDict(dictName); 653 if (dictToDump == null) { 654 Log.e(TAG, "Cannot dump " + dictName + ". " 655 + "The dictionary is not being used for suggestion or cannot be dumped."); 656 return; 657 } 658 dictToDump.dumpAllWordsForDebug(); 659 } 660 } 661
