1 // 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html#License 3 /* 4 ******************************************************************************* 5 * Copyright (C) 2006-2015, International Business Machines Corporation and 6 * others. All Rights Reserved. 7 ******************************************************************************* 8 */ 9 10 package com.ibm.icu.charset; 11 12 import java.io.IOException; 13 import java.nio.ByteBuffer; 14 import java.nio.CharBuffer; 15 import java.nio.IntBuffer; 16 17 import com.ibm.icu.charset.CharsetMBCS.MBCSHeader; 18 import com.ibm.icu.charset.CharsetMBCS.MBCSToUFallback; 19 import com.ibm.icu.charset.CharsetMBCS.UConverterMBCSTable; 20 import com.ibm.icu.impl.ICUBinary; 21 import com.ibm.icu.impl.InvalidFormatException; 22 23 /** 24 * ucnvmbcs.h 25 * 26 * ICU conversion (.cnv) data file structure, following the usual UDataInfo 27 * header. 28 * 29 * Format version: 6.2 30 * 31 * struct UConverterStaticData -- struct containing the converter name, IBM CCSID, 32 * min/max bytes per character, etc. 33 * see ucnv_bld.h 34 * 35 * -------------------- 36 * 37 * The static data is followed by conversionType-specific data structures. 38 * At the moment, there are only variations of MBCS converters. They all have 39 * the same toUnicode structures, while the fromUnicode structures for SBCS 40 * differ from those for other MBCS-style converters. 41 * 42 * _MBCSHeader.version 4.2 adds an optional conversion extension data structure. 43 * If it is present, then an ICU version reading header versions 4.0 or 4.1 44 * will be able to use the base table and ignore the extension. 45 * 46 * The unicodeMask in the static data is part of the base table data structure. 47 * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the 48 * fromUnicode stage 1 array. 49 * The static data unicodeMask refers only to the base table's properties if 50 * a base table is included. 51 * In an extension-only file, the static data unicodeMask is 0. 52 * The extension data indexes have a separate field with the unicodeMask flags. 53 * 54 * MBCS-style data structure following the static data. 55 * Offsets are counted in bytes from the beginning of the MBCS header structure. 56 * Details about usage in comments in ucnvmbcs.c. 57 * 58 * struct _MBCSHeader (see the definition in this header file below) 59 * contains 32-bit fields as follows: 60 * 8 values: 61 * 0 uint8_t[4] MBCS version in UVersionInfo format (currently 4.2.0.0) 62 * 1 uint32_t countStates 63 * 2 uint32_t countToUFallbacks 64 * 3 uint32_t offsetToUCodeUnits 65 * 4 uint32_t offsetFromUTable 66 * 5 uint32_t offsetFromUBytes 67 * 6 uint32_t flags, bits: 68 * 31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher 69 * 0 for older versions and if 70 * there is not extension structure 71 * 7.. 0 outputType 72 * 7 uint32_t fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher 73 * counts bytes in fromUBytes[] 74 * 75 * if(outputType==MBCS_OUTPUT_EXT_ONLY) { 76 * -- base table name for extension-only table 77 * char baseTableName[variable]; -- with NUL plus padding for 4-alignment 78 * 79 * -- all _MBCSHeader fields except for version and flags are 0 80 * } else { 81 * -- normal base table with optional extension 82 * 83 * int32_t stateTable[countStates][256]; 84 * 85 * struct _MBCSToUFallback { (fallbacks are sorted by offset) 86 * uint32_t offset; 87 * UChar32 codePoint; 88 * } toUFallbacks[countToUFallbacks]; 89 * 90 * uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2]; 91 * (padded to an even number of units) 92 * 93 * -- stage 1 tables 94 * if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) { 95 * -- stage 1 table for all of Unicode 96 * uint16_t fromUTable[0x440]; (32-bit-aligned) 97 * } else { 98 * -- BMP-only tables have a smaller stage 1 table 99 * uint16_t fromUTable[0x40]; (32-bit-aligned) 100 * } 101 * 102 * -- stage 2 tables 103 * length determined by top of stage 1 and bottom of stage 3 tables 104 * if(outputType==MBCS_OUTPUT_1) { 105 * -- SBCS: pure indexes 106 * uint16_t stage 2 indexes[?]; 107 * } else { 108 * -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes 109 * uint32_t stage 2 flags and indexes[?]; 110 * } 111 * 112 * -- stage 3 tables with byte results 113 * if(outputType==MBCS_OUTPUT_1) { 114 * -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c 115 * uint16_t fromUBytes[fromUBytesLength/2]; 116 * } else { 117 * -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c 118 * uint8_t fromUBytes[fromUBytesLength]; or 119 * uint16_t fromUBytes[fromUBytesLength/2]; or 120 * uint32_t fromUBytes[fromUBytesLength/4]; 121 * } 122 * } 123 * 124 * -- extension table, details see ucnv_ext.h 125 * int32_t indexes[>=32]; ... 126 */ 127 /* 128 * ucnv_ext.h 129 * 130 * See icuhtml/design/conversion/conversion_extensions.html 131 * 132 * Conversion extensions serve two purposes: 133 * 1. They support m:n mappings. 134 * 2. They support extension-only conversion files that are used together 135 * with the regular conversion data in base files. 136 * 137 * A base file may contain an extension table (explicitly requested or 138 * implicitly generated for m:n mappings), but its extension table is not 139 * used when an extension-only file is used. 140 * 141 * It is an error if a base file contains any regular (not extension) mapping 142 * from the same sequence as a mapping in the extension file 143 * because the base mapping would hide the extension mapping. 144 * 145 * 146 * Data for conversion extensions: 147 * 148 * One set of data structures per conversion direction (to/from Unicode). 149 * The data structures are sorted by input units to allow for binary search. 150 * Input sequences of more than one unit are handled like contraction tables 151 * in collation: 152 * The lookup value of a unit points to another table that is to be searched 153 * for the next unit, recursively. 154 * 155 * For conversion from Unicode, the initial code point is looked up in 156 * a 3-stage trie for speed, 157 * with an additional table of unique results to save space. 158 * 159 * Long output strings are stored in separate arrays, with length and index 160 * in the lookup tables. 161 * Output results also include a flag distinguishing roundtrip from 162 * (reverse) fallback mappings. 163 * 164 * Input Unicode strings must not begin or end with unpaired surrogates 165 * to avoid problems with matches on parts of surrogate pairs. 166 * 167 * Mappings from multiple characters (code points or codepage state 168 * table sequences) must be searched preferring the longest match. 169 * For this to work and be efficient, the variable-width table must contain 170 * all mappings that contain prefixes of the multiple characters. 171 * If an extension table is built on top of a base table in another file 172 * and a base table entry is a prefix of a multi-character mapping, then 173 * this is an error. 174 * 175 * 176 * Implementation note: 177 * 178 * Currently, the parser and several checks in the code limit the number 179 * of UChars or bytes in a mapping to 180 * UCNV_EXT_MAX_UCHARS and UCNV_EXT_MAX_BYTES, respectively, 181 * which are output value limits in the data structure. 182 * 183 * For input, this is not strictly necessary - it is a hard limit only for the 184 * buffers in UConverter that are used to store partial matches. 185 * 186 * Input sequences could otherwise be arbitrarily long if partial matches 187 * need not be stored (i.e., if a sequence does not span several buffers with too 188 * many units before the last buffer), although then results would differ 189 * depending on whether partial matches exceed the limits or not, 190 * which depends on the pattern of buffer sizes. 191 * 192 * 193 * Data structure: 194 * 195 * int32_t indexes[>=32]; 196 * 197 * Array of indexes and lengths etc. The length of the array is at least 32. 198 * The actual length is stored in indexes[0] to be forward compatible. 199 * 200 * Each index to another array is the number of bytes from indexes[]. 201 * Each length of an array is the number of array base units in that array. 202 * 203 * Some of the structures may not be present, in which case their indexes 204 * and lengths are 0. 205 * 206 * Usage of indexes[i]: 207 * [0] length of indexes[] 208 * 209 * // to Unicode table 210 * [1] index of toUTable[] (array of uint32_t) 211 * [2] length of toUTable[] 212 * [3] index of toUUChars[] (array of UChar) 213 * [4] length of toUUChars[] 214 * 215 * // from Unicode table, not for the initial code point 216 * [5] index of fromUTableUChars[] (array of UChar) 217 * [6] index of fromUTableValues[] (array of uint32_t) 218 * [7] length of fromUTableUChars[] and fromUTableValues[] 219 * [8] index of fromUBytes[] (array of char) 220 * [9] length of fromUBytes[] 221 * 222 * // from Unicode trie for initial-code point lookup 223 * [10] index of fromUStage12[] (combined array of uint16_t for stages 1 & 2) 224 * [11] length of stage 1 portion of fromUStage12[] 225 * [12] length of fromUStage12[] 226 * [13] index of fromUStage3[] (array of uint16_t indexes into fromUStage3b[]) 227 * [14] length of fromUStage3[] 228 * [15] index of fromUStage3b[] (array of uint32_t like fromUTableValues[]) 229 * [16] length of fromUStage3b[] 230 * 231 * [17] Bit field containing numbers of bytes: 232 * 31..24 reserved, 0 233 * 23..16 maximum input bytes 234 * 15.. 8 maximum output bytes 235 * 7.. 0 maximum bytes per UChar 236 * 237 * [18] Bit field containing numbers of UChars: 238 * 31..24 reserved, 0 239 * 23..16 maximum input UChars 240 * 15.. 8 maximum output UChars 241 * 7.. 0 maximum UChars per byte 242 * 243 * [19] Bit field containing flags: 244 * (extension table unicodeMask) 245 * 1 UCNV_HAS_SURROGATES flag for the extension table 246 * 0 UCNV_HAS_SUPPLEMENTARY flag for the extension table 247 * 248 * [20]..[30] reserved, 0 249 * [31] number of bytes for the entire extension structure 250 * [>31] reserved; there are indexes[0] indexes 251 * 252 * 253 * uint32_t toUTable[]; 254 * 255 * Array of byte/value pairs for lookups for toUnicode conversion. 256 * The array is partitioned into sections like collation contraction tables. 257 * Each section contains one word with the number of following words and 258 * a default value for when the lookup in this section yields no match. 259 * 260 * A section is sorted in ascending order of input bytes, 261 * allowing for fast linear or binary searches. 262 * The builder may store entries for a contiguous range of byte values 263 * (compare difference between the first and last one with count), 264 * which then allows for direct array access. 265 * The builder should always do this for the initial table section. 266 * 267 * Entries may have 0 values, see below. 268 * No two entries in a section have the same byte values. 269 * 270 * Each uint32_t contains an input byte value in bits 31..24 and the 271 * corresponding lookup value in bits 23..0. 272 * Interpret the value as follows: 273 * if(value==0) { 274 * no match, see below 275 * } else if(value<0x1f0000) { 276 * partial match - use value as index to the next toUTable section 277 * and match the next unit; (value indexes toUTable[value]) 278 * } else { 279 * if(bit 23 set) { 280 * roundtrip; 281 * } else { 282 * fallback; 283 * } 284 * unset value bit 23; 285 * if(value<=0x2fffff) { 286 * (value-0x1f0000) is a code point; (BMP: value<=0x1fffff) 287 * } else { 288 * bits 17..0 (value&0x3ffff) is an index to 289 * the result UChars in toUUChars[]; (0 indexes toUUChars[0]) 290 * length of the result=((value>>18)-12); (length=0..19) 291 * } 292 * } 293 * 294 * The first word in a section contains the number of following words in the 295 * input byte position (bits 31..24, number=1..0xff). 296 * The value of the initial word is used when the current byte is not found 297 * in this section. 298 * If the value is not 0, then it represents a result as above. 299 * If the value is 0, then the search has to return a shorter match with an 300 * earlier default value as the result, or result in "unmappable" even for the 301 * initial bytes. 302 * If the value is 0 for the initial toUTable entry, then the initial byte 303 * does not start any mapping input. 304 * 305 * 306 * UChar toUUChars[]; 307 * 308 * Contains toUnicode mapping results, stored as sequences of UChars. 309 * Indexes and lengths stored in the toUTable[]. 310 * 311 * 312 * UChar fromUTableUChars[]; 313 * uint32_t fromUTableValues[]; 314 * 315 * The fromUTable is split into two arrays, but works otherwise much like 316 * the toUTable. The array is partitioned into sections like collation 317 * contraction tables and toUTable. 318 * A row in the table consists of same-index entries in fromUTableUChars[] 319 * and fromUTableValues[]. 320 * 321 * Interpret a value as follows: 322 * if(value==0) { 323 * no match, see below 324 * } else if(value<=0xffffff) { (bits 31..24 are 0) 325 * partial match - use value as index to the next fromUTable section 326 * and match the next unit; (value indexes fromUTable[value]) 327 * } else { 328 * if(value==0x80000001) { 329 * return no mapping, but request for <subchar1>; 330 * } 331 * if(bit 31 set) { 332 * roundtrip; 333 * } else { 334 * fallback; 335 * } 336 * // bits 30..29 reserved, 0 337 * length=(value>>24)&0x1f; (bits 28..24) 338 * if(length==1..3) { 339 * bits 23..0 contain 1..3 bytes, padded with 00s on the left; 340 * } else { 341 * bits 23..0 (value&0xffffff) is an index to 342 * the result bytes in fromUBytes[]; (0 indexes fromUBytes[0]) 343 * } 344 * } 345 * 346 * The first pair in a section contains the number of following pairs in the 347 * UChar position (16 bits, number=1..0xffff). 348 * The value of the initial pair is used when the current UChar is not found 349 * in this section. 350 * If the value is not 0, then it represents a result as above. 351 * If the value is 0, then the search has to return a shorter match with an 352 * earlier default value as the result, or result in "unmappable" even for the 353 * initial UChars. 354 * 355 * If the from Unicode trie is present, then the from Unicode search tables 356 * are not used for initial code points. 357 * In this case, the first entries (index 0) in the tables are not used 358 * (reserved, set to 0) because a value of 0 is used in trie results 359 * to indicate no mapping. 360 * 361 * 362 * uint16_t fromUStage12[]; 363 * 364 * Stages 1 & 2 of a trie that maps an initial code point. 365 * Indexes in stage 1 are all offset by the length of stage 1 so that the 366 * same array pointer can be used for both stages. 367 * If (c>>10)>=(length of stage 1) then c does not start any mapping. 368 * Same bit distribution as for regular conversion tries. 369 * 370 * 371 * uint16_t fromUStage3[]; 372 * uint32_t fromUStage3b[]; 373 * 374 * Stage 3 of the trie. The first array simply contains indexes to the second, 375 * which contains words in the same format as fromUTableValues[]. 376 * Use a stage 3 granularity of 4, which allows for 256k stage 3 entries, 377 * and 16-bit entries in stage 3 allow for 64k stage 3b entries. 378 * The stage 3 granularity means that the stage 2 entry needs to be left-shifted. 379 * 380 * Two arrays are used because it is expected that more than half of the stage 3 381 * entries will be zero. The 16-bit index stage 3 array saves space even 382 * considering storing a total of 6 bytes per non-zero entry in both arrays 383 * together. 384 * Using a stage 3 granularity of >1 diminishes the compactability in that stage 385 * but provides a larger effective addressing space in stage 2. 386 * All but the final result stage use 16-bit entries to save space. 387 * 388 * fromUStage3b[] contains a zero for "no mapping" at its index 0, 389 * and may contain UCNV_EXT_FROM_U_SUBCHAR1 at index 1 for "<subchar1> SUB mapping" 390 * (i.e., "no mapping" with preference for <subchar1> rather than <subchar>), 391 * and all other items are unique non-zero results. 392 * 393 * The default value of a fromUTableValues[] section that is referenced 394 * _directly_ from a fromUStage3b[] item may also be UCNV_EXT_FROM_U_SUBCHAR1, 395 * but this value must not occur anywhere else in fromUTableValues[] 396 * because "no mapping" is always a property of a single code point, 397 * never of multiple. 398 * 399 * 400 * char fromUBytes[]; 401 * 402 * Contains fromUnicode mapping results, stored as sequences of chars. 403 * Indexes and lengths stored in the fromUTableValues[]. 404 */ 405 406 final class UConverterDataReader { 407 //private final static boolean debug = ICUDebug.enabled("UConverterDataReader"); 408 409 private static final class IsAcceptable implements ICUBinary.Authenticate { 410 // @Override when we switch to Java 6 411 @Override 412 public boolean isDataVersionAcceptable(byte formatVersion[]) { 413 return formatVersion[0] == 6; 414 } 415 } 416 private static final IsAcceptable IS_ACCEPTABLE = new IsAcceptable(); 417 418 /* 419 * UConverterDataReader(UConverterDataReader r) 420 { 421 byteBuffer = ICUBinary.getByteBufferFromInputStreamAndCloseStream(r.byteBuffer); 422 unicodeVersion = r.unicodeVersion; 423 } 424 */ 425 /** The buffer position after the static data. */ 426 private int posAfterStaticData; 427 428 /** 429 * <p>Protected constructor.</p> 430 * @param bytes ICU conversion data file 431 * @exception IOException throw if data file fails authentication 432 */ 433 protected UConverterDataReader(ByteBuffer bytes) 434 throws IOException{ 435 //if(debug) System.out.println("Bytes in buffer " + bytes.remaining()); 436 437 byteBuffer = bytes; 438 /*unicodeVersion = */ICUBinary.readHeader(byteBuffer, DATA_FORMAT_ID, IS_ACCEPTABLE); 439 440 //if(debug) System.out.println("Bytes left in byteBuffer " + byteBuffer.remaining()); 441 } 442 443 // protected methods ------------------------------------------------- 444 445 protected void readStaticData(UConverterStaticData sd) throws IOException 446 { 447 sd.structSize = byteBuffer.getInt(); 448 byte[] name = new byte[UConverterConstants.MAX_CONVERTER_NAME_LENGTH]; 449 byteBuffer.get(name); 450 sd.name = new String(name, "US-ASCII"); 451 sd.codepage = byteBuffer.getInt(); 452 sd.platform = byteBuffer.get(); 453 sd.conversionType = byteBuffer.get(); 454 sd.minBytesPerChar = byteBuffer.get(); 455 sd.maxBytesPerChar = byteBuffer.get(); 456 byteBuffer.get(sd.subChar); 457 sd.subCharLen = byteBuffer.get(); 458 sd.hasToUnicodeFallback = byteBuffer.get(); 459 sd.hasFromUnicodeFallback = byteBuffer.get(); 460 sd.unicodeMask = (short)(byteBuffer.get() & 0xff); 461 sd.subChar1 = byteBuffer.get(); 462 byteBuffer.get(sd.reserved); 463 posAfterStaticData = byteBuffer.position(); 464 } 465 466 int bytesReadAfterStaticData() { 467 return byteBuffer.position() - posAfterStaticData; 468 } 469 470 protected void readMBCSHeader(CharsetMBCS.MBCSHeader h) throws IOException 471 { 472 byteBuffer.get(h.version); 473 h.countStates = byteBuffer.getInt(); 474 h.countToUFallbacks = byteBuffer.getInt(); 475 h.offsetToUCodeUnits = byteBuffer.getInt(); 476 h.offsetFromUTable = byteBuffer.getInt(); 477 h.offsetFromUBytes = byteBuffer.getInt(); 478 h.flags = byteBuffer.getInt(); 479 h.fromUBytesLength = byteBuffer.getInt(); 480 if (h.version[0] == 5 && h.version[1] >= 3) { 481 h.options = byteBuffer.getInt(); 482 if ((h.options & CharsetMBCS.MBCS_OPT_NO_FROM_U) != 0) { 483 h.fullStage2Length = byteBuffer.getInt(); 484 } 485 } 486 } 487 488 protected void readMBCSTable(MBCSHeader header, UConverterMBCSTable mbcsTable) throws IOException 489 { 490 IntBuffer intBuffer = byteBuffer.asIntBuffer(); 491 mbcsTable.countStates = (byte) header.countStates; 492 mbcsTable.stateTable = new int[header.countStates][256]; 493 int i; 494 for(i = 0; i < header.countStates; ++i) { 495 intBuffer.get(mbcsTable.stateTable[i]); 496 } 497 498 mbcsTable.countToUFallbacks = header.countToUFallbacks; 499 mbcsTable.toUFallbacks = new MBCSToUFallback[header.countToUFallbacks]; 500 for(i = 0; i < header.countToUFallbacks; ++i) { 501 int offset = intBuffer.get(); 502 int codePoint = intBuffer.get(); 503 mbcsTable.toUFallbacks[i] = new MBCSToUFallback(offset, codePoint); 504 } 505 // Skip as many bytes as we have read from the IntBuffer. 506 int length = intBuffer.position() * 4; 507 ICUBinary.skipBytes(byteBuffer, length); 508 509 // Consider leaving some large arrays as CharBuffer/IntBuffer rather than 510 // reading them into Java arrays, to reduce initialization time and memory usage, 511 // at the cost of some performance. 512 // For example: unicodeCodeUnits, fromUnicodeTable, fromUnicodeInts. 513 // Take care not to modify the buffer contents for swaplfnl. 514 CharBuffer charBuffer = byteBuffer.asCharBuffer(); 515 length = header.offsetFromUTable - header.offsetToUCodeUnits; 516 assert (length & 1) == 0; 517 mbcsTable.unicodeCodeUnits = new char[length / 2]; 518 charBuffer.get(mbcsTable.unicodeCodeUnits); 519 // Skip as many bytes as we have read from the CharBuffer. 520 ICUBinary.skipBytes(byteBuffer, length); 521 522 length = header.offsetFromUBytes - header.offsetFromUTable; 523 assert (length & 1) == 0; 524 int fromUTableCharsLength; 525 if (mbcsTable.outputType == CharsetMBCS.MBCS_OUTPUT_1) { 526 // single-byte table stage1 + stage2 527 fromUTableCharsLength = length / 2; 528 } else if (mbcsTable.hasSupplementary()) { 529 // stage1 for Unicode limit 0x110000 >> 10 530 fromUTableCharsLength = 0x440; 531 } else { 532 // stage1 for BMP limit 0x10000 >> 10 533 fromUTableCharsLength = 0x40; 534 } 535 mbcsTable.fromUnicodeTable = new char[fromUTableCharsLength]; 536 charBuffer.get(mbcsTable.fromUnicodeTable); 537 if (mbcsTable.outputType != CharsetMBCS.MBCS_OUTPUT_1) { 538 // Read both stage1 and stage2 together into an int[] array. 539 // Keeping the short stage1 in the array avoids offsetting at runtime. 540 // The stage1 part of this array will not be used. 541 assert (length & 3) == 0; 542 mbcsTable.fromUnicodeTableInts = new int[length / 4]; 543 byteBuffer.asIntBuffer().get(mbcsTable.fromUnicodeTableInts); 544 } 545 // Skip as many bytes as are in stage1 + stage2. 546 ICUBinary.skipBytes(byteBuffer, length); 547 548 mbcsTable.fromUBytesLength = header.fromUBytesLength; 549 boolean noFromU = ((header.options & CharsetMBCS.MBCS_OPT_NO_FROM_U) != 0); 550 if (!noFromU) { 551 switch (mbcsTable.outputType) { 552 case CharsetMBCS.MBCS_OUTPUT_1: 553 case CharsetMBCS.MBCS_OUTPUT_2: 554 case CharsetMBCS.MBCS_OUTPUT_2_SISO: 555 case CharsetMBCS.MBCS_OUTPUT_3_EUC: 556 mbcsTable.fromUnicodeChars = ICUBinary.getChars( 557 byteBuffer, header.fromUBytesLength / 2, 0); 558 break; 559 case CharsetMBCS.MBCS_OUTPUT_3: 560 case CharsetMBCS.MBCS_OUTPUT_4_EUC: 561 mbcsTable.fromUnicodeBytes = new byte[header.fromUBytesLength]; 562 byteBuffer.get(mbcsTable.fromUnicodeBytes); 563 break; 564 case CharsetMBCS.MBCS_OUTPUT_4: 565 mbcsTable.fromUnicodeInts = ICUBinary.getInts( 566 byteBuffer, header.fromUBytesLength / 4, 0); 567 break; 568 default: 569 // Cannot occur, caller checked already. 570 assert false; 571 } 572 } else { 573 // Optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher. 574 // Needed for reconstituting omitted data. 575 mbcsTable.mbcsIndex = byteBuffer.asCharBuffer(); 576 } 577 } 578 579 protected String readBaseTableName() throws IOException 580 { 581 char c; 582 StringBuilder name = new StringBuilder(); 583 while((c = (char)byteBuffer.get()) != 0){ 584 name.append(c); 585 } 586 return name.toString(); 587 } 588 589 //protected int[] readExtIndexes(int skip) throws IOException 590 protected ByteBuffer readExtIndexes(int skip) throws IOException, InvalidFormatException 591 { 592 ICUBinary.skipBytes(byteBuffer, skip); 593 ByteBuffer b = ICUBinary.sliceWithOrder(byteBuffer); 594 int lengthOfIndexes = b.getInt(0); 595 if (lengthOfIndexes < 32) { 596 throw new InvalidFormatException(); 597 } 598 int numBytesExtensionStructure = b.getInt(31 * 4); 599 b.limit(numBytesExtensionStructure); 600 ICUBinary.skipBytes(byteBuffer, numBytesExtensionStructure); 601 return b; 602 } 603 604 /** 605 * Data formatVersion 6.1 and higher has a unicodeMask. 606 */ 607 boolean dataFormatHasUnicodeMask() { 608 int formatVersion0 = byteBuffer.get(16) & 0xff; 609 return formatVersion0 > 6 || (formatVersion0 == 6 && byteBuffer.get(17) != 0); 610 } 611 612 // private data members ------------------------------------------------- 613 614 /** 615 * ICU data file input stream 616 */ 617 private ByteBuffer byteBuffer; 618 619 // private VersionInfo unicodeVersion; 620 621 /** 622 * File format version that this class understands. 623 * No guarantees are made if a older version is used 624 * see store.c of gennorm for more information and values 625 */ 626 // DATA_FORMAT_ID_ values taken from icu4c isCnvAcceptable (ucnv_bld.c) 627 private static final int DATA_FORMAT_ID = 0x636e7674; // dataFormat="cnvt" 628 } 629
