Home | History | Annotate | Download | only in personalization 
      1 /*
      2  * Copyright (C) 2013 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.inputmethod.latin.personalization;
     18 
     19 import android.content.Context;
     20 import android.content.SharedPreferences;
     21 import android.util.Log;
     22 
     23 import com.android.inputmethod.annotations.UsedForTesting;
     24 import com.android.inputmethod.latin.Constants;
     25 import com.android.inputmethod.latin.Dictionary;
     26 import com.android.inputmethod.latin.ExpandableBinaryDictionary;
     27 import com.android.inputmethod.latin.LatinImeLogger;
     28 import com.android.inputmethod.latin.makedict.DictDecoder;
     29 import com.android.inputmethod.latin.makedict.FormatSpec;
     30 import com.android.inputmethod.latin.settings.Settings;
     31 import com.android.inputmethod.latin.utils.CollectionUtils;
     32 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils;
     33 import com.android.inputmethod.latin.utils.UserHistoryDictIOUtils.OnAddWordListener;
     34 
     35 import java.io.File;
     36 import java.io.IOException;
     37 import java.util.ArrayList;
     38 import java.util.HashMap;
     39 import java.util.Map;
     40 
     41 /**
     42  * This class is a base class of a dictionary that supports decaying for the personalized language
     43  * model.
     44  */
     45 public abstract class DecayingExpandableBinaryDictionaryBase extends ExpandableBinaryDictionary {
     46     private static final String TAG = DecayingExpandableBinaryDictionaryBase.class.getSimpleName();
     47     public static final boolean DBG_SAVE_RESTORE = false;
     48     private static final boolean DBG_STRESS_TEST = false;
     49     private static final boolean PROFILE_SAVE_RESTORE = LatinImeLogger.sDBG;
     50 
     51     /** Any pair being typed or picked */
     52     public static final int FREQUENCY_FOR_TYPED = 2;
     53 
     54     public static final int FREQUENCY_FOR_WORDS_IN_DICTS = FREQUENCY_FOR_TYPED;
     55     public static final int FREQUENCY_FOR_WORDS_NOT_IN_DICTS = Dictionary.NOT_A_PROBABILITY;
     56 
     57     /** Locale for which this user history dictionary is storing words */
     58     private final String mLocale;
     59 
     60     private final String mFileName;
     61 
     62     private final SharedPreferences mPrefs;
     63 
     64     private final ArrayList<PersonalizationDictionaryUpdateSession> mSessions =
     65             CollectionUtils.newArrayList();
     66 
     67     // Should always be false except when we use this class for test
     68     @UsedForTesting boolean mIsTest = false;
     69 
     70     /* package */ DecayingExpandableBinaryDictionaryBase(final Context context,
     71             final String locale, final SharedPreferences sp, final String dictionaryType,
     72             final String fileName) {
     73         super(context, fileName, dictionaryType, true);
     74         mLocale = locale;
     75         mFileName = fileName;
     76         mPrefs = sp;
     77         if (mLocale != null && mLocale.length() > 1) {
     78             asyncLoadDictionaryToMemory();
     79             reloadDictionaryIfRequired();
     80         }
     81     }
     82 
     83     @Override
     84     public void close() {
     85         if (!ExpandableBinaryDictionary.ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE) {
     86             closeBinaryDictionary();
     87         }
     88         // Flush pending writes.
     89         // TODO: Remove after this class become to use a dynamic binary dictionary.
     90         asyncFlashAllBinaryDictionary();
     91         Settings.writeLastUserHistoryWriteTime(mPrefs, mLocale);
     92     }
     93 
     94     @Override
     95     protected Map<String, String> getHeaderAttributeMap() {
     96         HashMap<String, String> attributeMap = new HashMap<String, String>();
     97         attributeMap.put(FormatSpec.FileHeader.SUPPORTS_DYNAMIC_UPDATE_ATTRIBUTE,
     98                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
     99         attributeMap.put(FormatSpec.FileHeader.USES_FORGETTING_CURVE_ATTRIBUTE,
    100                 FormatSpec.FileHeader.ATTRIBUTE_VALUE_TRUE);
    101         attributeMap.put(FormatSpec.FileHeader.DICTIONARY_ID_ATTRIBUTE, mFileName);
    102         attributeMap.put(FormatSpec.FileHeader.DICTIONARY_LOCALE_ATTRIBUTE, mLocale);
    103         return attributeMap;
    104     }
    105 
    106     @Override
    107     protected boolean hasContentChanged() {
    108         return false;
    109     }
    110 
    111     @Override
    112     protected boolean needsToReloadBeforeWriting() {
    113         return false;
    114     }
    115 
    116     /**
    117      * Pair will be added to the decaying dictionary.
    118      *
    119      * The first word may be null. That means we don't know the context, in other words,
    120      * it's only a unigram. The first word may also be an empty string : this means start
    121      * context, as in beginning of a sentence for example.
    122      * The second word may not be null (a NullPointerException would be thrown).
    123      */
    124     public void addToDictionary(final String word0, final String word1, final boolean isValid) {
    125         if (word1.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH ||
    126                 (word0 != null && word0.length() >= Constants.DICTIONARY_MAX_WORD_LENGTH)) {
    127             return;
    128         }
    129         final int frequency = ENABLE_BINARY_DICTIONARY_DYNAMIC_UPDATE ?
    130                 (isValid ? FREQUENCY_FOR_WORDS_IN_DICTS : FREQUENCY_FOR_WORDS_NOT_IN_DICTS) :
    131                         FREQUENCY_FOR_TYPED;
    132         addWordDynamically(word1, null /* shortcutTarget */, frequency, 0 /* shortcutFreq */,
    133                 false /* isNotAWord */);
    134         // Do not insert a word as a bigram of itself
    135         if (word1.equals(word0)) {
    136             return;
    137         }
    138         if (null != word0) {
    139             addBigramDynamically(word0, word1, frequency, isValid);
    140         }
    141     }
    142 
    143     public void cancelAddingUserHistory(final String word0, final String word1) {
    144         removeBigramDynamically(word0, word1);
    145     }
    146 
    147     @Override
    148     protected void loadDictionaryAsync() {
    149         final int[] profTotalCount = { 0 };
    150         final String locale = getLocale();
    151         if (DBG_STRESS_TEST) {
    152             try {
    153                 Log.w(TAG, "Start stress in loading: " + locale);
    154                 Thread.sleep(15000);
    155                 Log.w(TAG, "End stress in loading");
    156             } catch (InterruptedException e) {
    157             }
    158         }
    159         final long last = Settings.readLastUserHistoryWriteTime(mPrefs, locale);
    160         final long now = System.currentTimeMillis();
    161         final ExpandableBinaryDictionary dictionary = this;
    162         final OnAddWordListener listener = new OnAddWordListener() {
    163             @Override
    164             public void setUnigram(final String word, final String shortcutTarget,
    165                     final int frequency, final int shortcutFreq) {
    166                 if (DBG_SAVE_RESTORE) {
    167                     Log.d(TAG, "load unigram: " + word + "," + frequency);
    168                 }
    169                 addWord(word, shortcutTarget, frequency, shortcutFreq, false /* isNotAWord */);
    170                 ++profTotalCount[0];
    171             }
    172 
    173             @Override
    174             public void setBigram(final String word0, final String word1, final int frequency) {
    175                 if (word0.length() < Constants.DICTIONARY_MAX_WORD_LENGTH
    176                         && word1.length() < Constants.DICTIONARY_MAX_WORD_LENGTH) {
    177                     if (DBG_SAVE_RESTORE) {
    178                         Log.d(TAG, "load bigram: " + word0 + "," + word1 + "," + frequency);
    179                     }
    180                     ++profTotalCount[0];
    181                     addBigram(word0, word1, frequency, last);
    182                 }
    183             }
    184         };
    185 
    186         // Load the dictionary from binary file
    187         final File dictFile = new File(mContext.getFilesDir(), mFileName);
    188         final DictDecoder dictDecoder = FormatSpec.getDictDecoder(dictFile,
    189                 DictDecoder.USE_BYTEARRAY);
    190         if (dictDecoder == null) {
    191             // This is an expected condition: we don't have a user history dictionary for this
    192             // language yet. It will be created sometime later.
    193             return;
    194         }
    195 
    196         try {
    197             dictDecoder.openDictBuffer();
    198             UserHistoryDictIOUtils.readDictionaryBinary(dictDecoder, listener);
    199         } catch (IOException e) {
    200             Log.d(TAG, "IOException on opening a bytebuffer", e);
    201         } finally {
    202             if (PROFILE_SAVE_RESTORE) {
    203                 final long diff = System.currentTimeMillis() - now;
    204                 Log.d(TAG, "PROF: Load UserHistoryDictionary: "
    205                         + locale + ", " + diff + "ms. load " + profTotalCount[0] + "entries.");
    206             }
    207         }
    208     }
    209 
    210     protected String getLocale() {
    211         return mLocale;
    212     }
    213 
    214     public void registerUpdateSession(PersonalizationDictionaryUpdateSession session) {
    215         session.setPredictionDictionary(this);
    216         mSessions.add(session);
    217         session.onDictionaryReady();
    218     }
    219 
    220     public void unRegisterUpdateSession(PersonalizationDictionaryUpdateSession session) {
    221         mSessions.remove(session);
    222     }
    223 
    224     @UsedForTesting
    225     public void clearAndFlushDictionary() {
    226         // Clear the node structure on memory
    227         clear();
    228         // Then flush the cleared state of the dictionary on disk.
    229         asyncFlashAllBinaryDictionary();
    230     }
    231 
    232     /* package */ void decayIfNeeded() {
    233         runGCIfRequired(false /* mindsBlockByGC */);
    234     }
    235 }
    236