1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.inputmethod.latin.makedict; 18 19 import com.android.inputmethod.latin.Constants; 20 import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; 21 22 /** 23 * Dictionary File Format Specification. 24 */ 25 public final class FormatSpec { 26 27 /* 28 * Array of Node(FusionDictionary.Node) layout is as follows: 29 * 30 * g | 31 * r | the number of groups, 1 or 2 bytes. 32 * o | 1 byte = bbbbbbbb match 33 * u | case 1xxxxxxx => xxxxxxx << 8 + next byte 34 * p | otherwise => bbbbbbbb 35 * c | 36 * ount 37 * 38 * g | 39 * r | sequence of groups, 40 * o | the layout of each group is described below. 41 * u | 42 * ps 43 * 44 * f | 45 * o | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) 46 * r | forward link address, 3byte 47 * w | 1 byte = bbbbbbbb match 48 * a | case 1xxxxxxx => -((xxxxxxx << 16) + (next byte << 8) + next byte) 49 * r | otherwise => (xxxxxxx << 16) + (next byte << 8) + next byte 50 * d | 51 * linkaddress 52 */ 53 54 /* Node(CharGroup) layout is as follows: 55 * | IF !SUPPORTS_DYNAMIC_UPDATE 56 * | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE 57 * | 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS 58 * f | 01 = 1 byte : FLAG_GROUP_ADDRESS_TYPE_ONEBYTE 59 * l | 10 = 2 bytes : FLAG_GROUP_ADDRESS_TYPE_TWOBYTES 60 * a | 11 = 3 bytes : FLAG_GROUP_ADDRESS_TYPE_THREEBYTES 61 * g | ELSE 62 * s | is moved ? 2 bits, 11 = no : FLAG_IS_NOT_MOVED 63 * | This must be the same as FLAG_GROUP_ADDRESS_TYPE_THREEBYTES 64 * | 01 = yes : FLAG_IS_MOVED 65 * | the new address is stored in the same place as the parent address 66 * | is deleted? 10 = yes : FLAG_IS_DELETED 67 * | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS 68 * | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL 69 * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS 70 * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS 71 * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD 72 * | is blacklisted ? 1 bit, 1 = yes, 0 = no : FLAG_IS_BLACKLISTED 73 * 74 * p | 75 * a | IF SUPPORTS_DYNAMIC_UPDATE (defined in the file header) 76 * r | parent address, 3byte 77 * e | 1 byte = bbbbbbbb match 78 * n | case 1xxxxxxx => -((0xxxxxxx << 16) + (next byte << 8) + next byte) 79 * t | otherwise => (bbbbbbbb << 16) + (next byte << 8) + next byte 80 * a | 81 * ddress 82 * 83 * c | IF FLAG_HAS_MULTIPLE_CHARS 84 * h | char, char, char, char n * (1 or 3 bytes) : use CharGroupInfo for i/o helpers 85 * a | end 1 byte, = 0 86 * r | ELSE 87 * s | char 1 or 3 bytes 88 * | END 89 * 90 * f | 91 * r | IF FLAG_IS_TERMINAL 92 * e | frequency 1 byte 93 * q | 94 * 95 * c | IF 00 = FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = addressType 96 * h | // nothing 97 * i | ELSIF 01 = FLAG_GROUP_ADDRESS_TYPE_ONEBYTE == addressType 98 * l | children address, 1 byte 99 * d | ELSIF 10 = FLAG_GROUP_ADDRESS_TYPE_TWOBYTES == addressType 100 * r | children address, 2 bytes 101 * e | ELSE // 11 = FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = addressType 102 * n | children address, 3 bytes 103 * A | END 104 * d 105 * dress 106 * 107 * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS 108 * | shortcut string list 109 * | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS 110 * | bigrams address list 111 * 112 * Char format is: 113 * 1 byte = bbbbbbbb match 114 * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte 115 * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because 116 * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with 117 * 00011111 would be outside unicode. 118 * else: iso-latin-1 code 119 * This allows for the whole unicode range to be encoded, including chars outside of 120 * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control 121 * characters which should never happen anyway (and still work, but take 3 bytes). 122 * 123 * bigram address list is: 124 * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT 125 * | addressSign = 1 bit, : FLAG_ATTRIBUTE_OFFSET_NEGATIVE 126 * | 1 = must take -address, 0 = must take +address 127 * | xx : mask with MASK_ATTRIBUTE_ADDRESS_TYPE 128 * | addressFormat = 2 bits, 00 = unused : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE 129 * | 01 = 1 byte : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE 130 * | 10 = 2 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES 131 * | 11 = 3 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES 132 * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY 133 * <address> | IF (01 == FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE == addressFormat) 134 * | read 1 byte, add top 4 bits 135 * | ELSIF (10 == FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES == addressFormat) 136 * | read 2 bytes, add top 4 bits 137 * | ELSE // 11 == FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES == addressFormat 138 * | read 3 bytes, add top 4 bits 139 * | END 140 * | if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE) then address = -address 141 * if (FLAG_ATTRIBUTE_HAS_NEXT) goto bigram_and_shortcut_address_list_is 142 * 143 * shortcut string list is: 144 * <byte size> = GROUP_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. 145 * <flags> = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT 146 * | reserved = 3 bits, must be 0 147 * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY 148 * <shortcut> = | string of characters at the char format described above, with the terminator 149 * | used to signal the end of the string. 150 * if (FLAG_ATTRIBUTE_HAS_NEXT goto flags 151 */ 152 153 static final int VERSION_1_MAGIC_NUMBER = 0x78B1; 154 public static final int VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; 155 static final int MINIMUM_SUPPORTED_VERSION = 1; 156 static final int MAXIMUM_SUPPORTED_VERSION = 3; 157 static final int NOT_A_VERSION_NUMBER = -1; 158 static final int FIRST_VERSION_WITH_HEADER_SIZE = 2; 159 static final int FIRST_VERSION_WITH_DYNAMIC_UPDATE = 3; 160 161 // These options need to be the same numeric values as the one in the native reading code. 162 static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; 163 // TODO: Make the native reading code read this variable. 164 static final int SUPPORTS_DYNAMIC_UPDATE = 0x2; 165 static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; 166 static final int CONTAINS_BIGRAMS_FLAG = 0x8; 167 168 // TODO: Make this value adaptative to content data, store it in the header, and 169 // use it in the reading code. 170 static final int MAX_WORD_LENGTH = Constants.Dictionary.MAX_WORD_LENGTH; 171 172 static final int PARENT_ADDRESS_SIZE = 3; 173 static final int FORWARD_LINK_ADDRESS_SIZE = 3; 174 175 // These flags are used only in the static dictionary. 176 static final int MASK_GROUP_ADDRESS_TYPE = 0xC0; 177 static final int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; 178 static final int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; 179 static final int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; 180 static final int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; 181 182 static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; 183 184 static final int FLAG_IS_TERMINAL = 0x10; 185 static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08; 186 static final int FLAG_HAS_BIGRAMS = 0x04; 187 static final int FLAG_IS_NOT_A_WORD = 0x02; 188 static final int FLAG_IS_BLACKLISTED = 0x01; 189 190 // These flags are used only in the dynamic dictionary. 191 static final int MASK_MOVE_AND_DELETE_FLAG = 0xC0; 192 static final int FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE = 0x40; 193 static final int FLAG_IS_MOVED = 0x00 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE; 194 static final int FLAG_IS_NOT_MOVED = 0x80 | FIXED_BIT_OF_DYNAMIC_UPDATE_MOVE; 195 static final int FLAG_IS_DELETED = 0x80; 196 197 static final int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; 198 static final int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; 199 static final int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; 200 static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; 201 static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; 202 static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; 203 static final int FLAG_ATTRIBUTE_FREQUENCY = 0x0F; 204 205 static final int GROUP_CHARACTERS_TERMINATOR = 0x1F; 206 207 static final int GROUP_TERMINATOR_SIZE = 1; 208 static final int GROUP_FLAGS_SIZE = 1; 209 static final int GROUP_FREQUENCY_SIZE = 1; 210 static final int GROUP_MAX_ADDRESS_SIZE = 3; 211 static final int GROUP_ATTRIBUTE_FLAGS_SIZE = 1; 212 static final int GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; 213 static final int GROUP_SHORTCUT_LIST_SIZE_SIZE = 2; 214 215 static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; 216 static final int NO_PARENT_ADDRESS = 0; 217 static final int NO_FORWARD_LINK_ADDRESS = 0; 218 static final int INVALID_CHARACTER = -1; 219 220 static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127 221 static final int MAX_CHARGROUPS_IN_A_NODE = 0x7FFF; // 32767 222 static final int MAX_BIGRAMS_IN_A_GROUP = 10000; 223 224 static final int MAX_TERMINAL_FREQUENCY = 255; 225 static final int MAX_BIGRAM_FREQUENCY = 15; 226 227 public static final int SHORTCUT_WHITELIST_FREQUENCY = 15; 228 229 // This option needs to be the same numeric value as the one in binary_format.h. 230 static final int NOT_VALID_WORD = -99; 231 static final int SIGNED_CHILDREN_ADDRESS_SIZE = 3; 232 233 /** 234 * Options about file format. 235 */ 236 public static final class FormatOptions { 237 public final int mVersion; 238 public final boolean mSupportsDynamicUpdate; 239 public FormatOptions(final int version) { 240 this(version, false); 241 } 242 public FormatOptions(final int version, final boolean supportsDynamicUpdate) { 243 mVersion = version; 244 if (version < FIRST_VERSION_WITH_DYNAMIC_UPDATE && supportsDynamicUpdate) { 245 throw new RuntimeException("Dynamic updates are only supported with versions " 246 + FIRST_VERSION_WITH_DYNAMIC_UPDATE + " and ulterior."); 247 } 248 mSupportsDynamicUpdate = supportsDynamicUpdate; 249 } 250 } 251 252 /** 253 * Class representing file header. 254 */ 255 public static final class FileHeader { 256 public final int mHeaderSize; 257 public final DictionaryOptions mDictionaryOptions; 258 public final FormatOptions mFormatOptions; 259 private static final String DICTIONARY_VERSION_ATTRIBUTE = "version"; 260 private static final String DICTIONARY_LOCALE_ATTRIBUTE = "locale"; 261 private static final String DICTIONARY_ID_ATTRIBUTE = "dictionary"; 262 private static final String DICTIONARY_DESCRIPTION_ATTRIBUTE = "description"; 263 public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions, 264 final FormatOptions formatOptions) { 265 mHeaderSize = headerSize; 266 mDictionaryOptions = dictionaryOptions; 267 mFormatOptions = formatOptions; 268 } 269 270 // Helper method to get the locale as a String 271 public String getLocaleString() { 272 return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_LOCALE_ATTRIBUTE); 273 } 274 275 // Helper method to get the version String 276 public String getVersion() { 277 return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_VERSION_ATTRIBUTE); 278 } 279 280 // Helper method to get the dictionary ID as a String 281 public String getId() { 282 return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_ID_ATTRIBUTE); 283 } 284 285 // Helper method to get the description 286 public String getDescription() { 287 // TODO: Right now each dictionary file comes with a description in its own language. 288 // It will display as is no matter the device's locale. It should be internationalized. 289 return mDictionaryOptions.mAttributes.get(FileHeader.DICTIONARY_DESCRIPTION_ATTRIBUTE); 290 } 291 } 292 293 private FormatSpec() { 294 // This utility class is not publicly instantiable. 295 } 296 } 297
