Home | History | Annotate | Download | only in common
      1 /*
      2 ******************************************************************************
      3 *
      4 *   Copyright (C) 2000-2012, International Business Machines
      5 *   Corporation and others.  All Rights Reserved.
      6 *
      7 ******************************************************************************
      8 *   file name:  ucnvmbcs.h
      9 *   encoding:   US-ASCII
     10 *   tab size:   8 (not used)
     11 *   indentation:4
     12 *
     13 *   created on: 2000jul07
     14 *   created by: Markus W. Scherer
     15 */
     16 
     17 #ifndef __UCNVMBCS_H__
     18 #define __UCNVMBCS_H__
     19 
     20 #include "unicode/utypes.h"
     21 
     22 #if !UCONFIG_NO_CONVERSION
     23 
     24 #include "unicode/ucnv.h"
     25 #include "ucnv_cnv.h"
     26 #include "ucnv_ext.h"
     27 
     28 /**
     29  * ICU conversion (.cnv) data file structure, following the usual UDataInfo
     30  * header.
     31  *
     32  * Format version: 6.2
     33  *
     34  * struct UConverterStaticData -- struct containing the converter name, IBM CCSID,
     35  *                                min/max bytes per character, etc.
     36  *                                see ucnv_bld.h
     37  *
     38  * --------------------
     39  *
     40  * The static data is followed by conversionType-specific data structures.
     41  * At the moment, there are only variations of MBCS converters. They all have
     42  * the same toUnicode structures, while the fromUnicode structures for SBCS
     43  * differ from those for other MBCS-style converters.
     44  *
     45  * _MBCSHeader.version 5 is optional and not backward-compatible
     46  * (as usual for changes in the major version field).
     47  *
     48  * Versions 5.m work like versions 4.m except:
     49  * - The _MBCSHeader has variable length (and is always longer than in version 4).
     50  *   See the struct _MBCSHeader further description below.
     51  * - There is a set of flags which indicate further incompatible changes.
     52  *   (Reader code must reject the file if it does not recognize them all.)
     53  * - In particular, one of these flags indicates that most of the fromUnicode
     54  *   data is missing and must be reconstituted from the toUnicode data
     55  *   and from the utf8Friendly mbcsIndex at load time.
     56  *   (This only works with a utf8Friendly table.)
     57  *   In this case, makeconv may increase maxFastUChar automatically to U+FFFF.
     58  *
     59  * The first of these versions is 5.3, which is like 4.3 except for the differences above.
     60  *
     61  * When possible, makeconv continues to generate version 4.m files.
     62  *
     63  * _MBCSHeader.version 4.3 optionally modifies the fromUnicode data structures
     64  * slightly and optionally adds a table for conversion to MBCS (non-SBCS)
     65  * charsets.
     66  *
     67  * The modifications are to make the data utf8Friendly. Not every 4.3 file
     68  * file contains utf8Friendly data.
     69  * It is utf8Friendly if _MBCSHeader.version[2]!=0.
     70  * In this case, the data structures are utf8Friendly up to the code point
     71  *   maxFastUChar=((_MBCSHeader.version[2]<<8)|0xff)
     72  *
     73  * A utf8Friendly file has fromUnicode stage 3 entries for code points up to
     74  * maxFastUChar allocated in blocks of 64 for indexing with the 6 bits from
     75  * a UTF-8 trail byte. ASCII is allocated linearly with 128 contiguous entries.
     76  *
     77  * In addition, a utf8Friendly MBCS file contains an additional
     78  *   uint16_t mbcsIndex[(maxFastUChar+1)>>6];
     79  * which replaces the stage 1 and 2 tables for indexing with bits from the
     80  * UTF-8 lead byte and middle trail byte. Unlike the older MBCS stage 2 table,
     81  * the mbcsIndex does not contain roundtrip flags. Therefore, all fallbacks
     82  * from code points up to maxFastUChar (and roundtrips to 0x00) are moved to
     83  * the extension data structure. This also allows for faster roundtrip
     84  * conversion from UTF-16.
     85  *
     86  * SBCS files do not contain an additional sbcsIndex[] array because the
     87  * proportional size increase would be noticeable, but the runtime
     88  * code builds one for the code point range for which the runtime conversion
     89  * code is optimized.
     90  *
     91  * For SBCS, maxFastUChar should be at least U+0FFF. The initial makeconv
     92  * implementation sets it to U+1FFF. Because the sbcsIndex is not stored in
     93  * the file, a larger maxFastUChar only affects stage 3 block allocation size
     94  * and is free in empty blocks. (Larger blocks with sparse contents cause larger
     95  * files.) U+1FFF includes almost all of the small scripts.
     96  * U+0FFF covers UTF-8 two-byte sequences and three-byte sequences starting with
     97  * 0xe0. This includes most scripts with legacy SBCS charsets.
     98  * The initial runtime implementation using 4.3 files only builds an sbcsIndex
     99  * for code points up to U+0FFF.
    100  *
    101  * For MBCS, maxFastUChar should be at least U+D7FF (=initial value).
    102  * This boundary is convenient because practically all of the commonly used
    103  * characters are below it, and because it is the boundary to surrogate
    104  * code points, above which special handling is necessary anyway.
    105  * (Surrogate pair assembly for UTF-16, validity checking for UTF-8.)
    106  *
    107  * maxFastUChar could be up to U+FFFF to cover the whole BMP, which could be
    108  * useful especially for conversion from UTF-8 when the input can be assumed
    109  * to be valid, because the surrogate range would then not have to be
    110  * checked.
    111  * (With maxFastUChar=0xffff, makeconv would have to check for mbcsIndex value
    112  * overflow because with the all-unassigned block 0 and nearly full mappings
    113  * from the BMP it is theoretically possible that an index into stage 3
    114  * exceeds 16 bits.)
    115  *
    116  * _MBCSHeader.version 4.2 adds an optional conversion extension data structure.
    117  * If it is present, then an ICU version reading header versions 4.0 or 4.1
    118  * will be able to use the base table and ignore the extension.
    119  *
    120  * The unicodeMask in the static data is part of the base table data structure.
    121  * Especially, the UCNV_HAS_SUPPLEMENTARY flag determines the length of the
    122  * fromUnicode stage 1 array.
    123  * The static data unicodeMask refers only to the base table's properties if
    124  * a base table is included.
    125  * In an extension-only file, the static data unicodeMask is 0.
    126  * The extension data indexes have a separate field with the unicodeMask flags.
    127  *
    128  * MBCS-style data structure following the static data.
    129  * Offsets are counted in bytes from the beginning of the MBCS header structure.
    130  * Details about usage in comments in ucnvmbcs.c.
    131  *
    132  * struct _MBCSHeader (see the definition in this header file below)
    133  * contains 32-bit fields as follows:
    134  * 8 values:
    135  *  0   uint8_t[4]  MBCS version in UVersionInfo format (currently 4.3.x.0)
    136  *  1   uint32_t    countStates
    137  *  2   uint32_t    countToUFallbacks
    138  *  3   uint32_t    offsetToUCodeUnits
    139  *  4   uint32_t    offsetFromUTable
    140  *  5   uint32_t    offsetFromUBytes
    141  *  6   uint32_t    flags, bits:
    142  *                      31.. 8 offsetExtension -- _MBCSHeader.version 4.2 (ICU 2.8) and higher
    143  *                                                0 for older versions and if
    144  *                                                there is not extension structure
    145  *                       7.. 0 outputType
    146  *  7   uint32_t    fromUBytesLength -- _MBCSHeader.version 4.1 (ICU 2.4) and higher
    147  *                  counts bytes in fromUBytes[]
    148  *
    149  * New and required in version 5:
    150  *  8   uint32_t    options, bits:
    151  *                      31..16 reserved for flags that can be added without breaking
    152  *                                 backward compatibility
    153  *                      15.. 6 reserved for flags whose addition will break
    154  *                                 backward compatibility
    155  *                           6 MBCS_OPT_FROM_U -- if set,
    156  *                                 then most of the fromUnicode data is omitted;
    157  *                                 fullStage2Length is present and the missing
    158  *                                 bottom part of stage 2 must be reconstituted from
    159  *                                 the toUnicode data;
    160  *                                 stage 3 is missing completely as well;
    161  *                                 not used for SBCS tables
    162  *                       5.. 0 length of the _MBCSHeader (number of uint32_t)
    163  *
    164  * New and optional in version 5:
    165  *  9   uint32_t    fullStage2Length: used if MBCS_OPT_FROM_U is set
    166  *                                 specifies the full length of stage 2
    167  *                                 including the omitted part
    168  *
    169  * if(outputType==MBCS_OUTPUT_EXT_ONLY) {
    170  *     -- base table name for extension-only table
    171  *     char baseTableName[variable]; -- with NUL plus padding for 4-alignment
    172  *
    173  *     -- all _MBCSHeader fields except for version and flags are 0
    174  * } else {
    175  *     -- normal base table with optional extension
    176  *
    177  *     int32_t stateTable[countStates][256];
    178  *
    179  *     struct _MBCSToUFallback { (fallbacks are sorted by offset)
    180  *         uint32_t offset;
    181  *         UChar32 codePoint;
    182  *     } toUFallbacks[countToUFallbacks];
    183  *
    184  *     uint16_t unicodeCodeUnits[(offsetFromUTable-offsetToUCodeUnits)/2];
    185  *                  (padded to an even number of units)
    186  *
    187  *     -- stage 1 tables
    188  *     if(staticData.unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
    189  *         -- stage 1 table for all of Unicode
    190  *         uint16_t fromUTable[0x440]; (32-bit-aligned)
    191  *     } else {
    192  *         -- BMP-only tables have a smaller stage 1 table
    193  *         uint16_t fromUTable[0x40]; (32-bit-aligned)
    194  *     }
    195  *
    196  *     -- stage 2 tables
    197  *        length determined by top of stage 1 and bottom of stage 3 tables
    198  *     if(outputType==MBCS_OUTPUT_1) {
    199  *         -- SBCS: pure indexes
    200  *         uint16_t stage 2 indexes[?];
    201  *     } else {
    202  *         -- DBCS, MBCS, EBCDIC_STATEFUL, ...: roundtrip flags and indexes
    203  *         uint32_t stage 2 flags and indexes[?];
    204  *         if(options&MBCS_OPT_NO_FROM_U) {
    205  *             stage 2 really has length fullStage2Length
    206  *             and the omitted lower part must be reconstituted from
    207  *             the toUnicode data
    208  *         }
    209  *     }
    210  *
    211  *     -- stage 3 tables with byte results
    212  *     if(outputType==MBCS_OUTPUT_1) {
    213  *         -- SBCS: each 16-bit result contains flags and the result byte, see ucnvmbcs.c
    214  *         uint16_t fromUBytes[fromUBytesLength/2];
    215  *     } else if(!(options&MBCS_OPT_NO_FROM_U)) {
    216  *         -- DBCS, MBCS, EBCDIC_STATEFUL, ... 2/3/4 bytes result, see ucnvmbcs.c
    217  *         uint8_t fromUBytes[fromUBytesLength]; or
    218  *         uint16_t fromUBytes[fromUBytesLength/2]; or
    219  *         uint32_t fromUBytes[fromUBytesLength/4];
    220  *     } else {
    221  *         fromUBytes[] must be reconstituted from the toUnicode data
    222  *     }
    223  *
    224  *     -- optional utf8Friendly mbcsIndex -- _MBCSHeader.version 4.3 (ICU 3.8) and higher
    225  *     if(outputType!=MBCS_OUTPUT_1 &&
    226  *        _MBCSHeader.version[1]>=3 &&
    227  *        (maxFastUChar=_MBCSHeader.version[2])!=0
    228  *     ) {
    229  *         maxFastUChar=(maxFastUChar<<8)|0xff;
    230  *         uint16_t mbcsIndex[(maxFastUChar+1)>>6];
    231  *     }
    232  * }
    233  *
    234  * -- extension table, details see ucnv_ext.h
    235  * int32_t indexes[>=32]; ...
    236  */
    237 
    238 /* MBCS converter data and state -------------------------------------------- */
    239 
    240 enum {
    241     MBCS_MAX_STATE_COUNT=128
    242 };
    243 
    244 /**
    245  * MBCS action codes for conversions to Unicode.
    246  * These values are in bits 23..20 of the state table entries.
    247  */
    248 enum {
    249     MBCS_STATE_VALID_DIRECT_16,
    250     MBCS_STATE_VALID_DIRECT_20,
    251 
    252     MBCS_STATE_FALLBACK_DIRECT_16,
    253     MBCS_STATE_FALLBACK_DIRECT_20,
    254 
    255     MBCS_STATE_VALID_16,
    256     MBCS_STATE_VALID_16_PAIR,
    257 
    258     MBCS_STATE_UNASSIGNED,
    259     MBCS_STATE_ILLEGAL,
    260 
    261     MBCS_STATE_CHANGE_ONLY
    262 };
    263 
    264 /* Macros for state table entries */
    265 #define MBCS_ENTRY_TRANSITION(state, offset) (int32_t)(((int32_t)(state)<<24L)|(offset))
    266 #define MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, offset) (int32_t)(((entry)&0xff000000)|(offset))
    267 #define MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, offset) (int32_t)((entry)+(offset))
    268 
    269 #define MBCS_ENTRY_FINAL(state, action, value) (int32_t)(0x80000000|((int32_t)(state)<<24L)|((action)<<20L)|(value))
    270 #define MBCS_ENTRY_SET_FINAL(entry) (int32_t)((entry)|0x80000000)
    271 #define MBCS_ENTRY_FINAL_SET_ACTION(entry, action) (int32_t)(((entry)&0xff0fffff)|((int32_t)(action)<<20L))
    272 #define MBCS_ENTRY_FINAL_SET_VALUE(entry, value) (int32_t)(((entry)&0xfff00000)|(value))
    273 #define MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, action, value) (int32_t)(((entry)&0xff000000)|((int32_t)(action)<<20L)|(value))
    274 
    275 #define MBCS_ENTRY_SET_STATE(entry, state) (int32_t)(((entry)&0x80ffffff)|((int32_t)(state)<<24L))
    276 
    277 #define MBCS_ENTRY_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
    278 
    279 #define MBCS_ENTRY_IS_TRANSITION(entry) ((entry)>=0)
    280 #define MBCS_ENTRY_IS_FINAL(entry) ((entry)<0)
    281 
    282 #define MBCS_ENTRY_TRANSITION_STATE(entry) (((uint32_t)entry)>>24)
    283 #define MBCS_ENTRY_TRANSITION_OFFSET(entry) ((entry)&0xffffff)
    284 
    285 #define MBCS_ENTRY_FINAL_STATE(entry) ((((uint32_t)entry)>>24)&0x7f)
    286 #define MBCS_ENTRY_FINAL_IS_VALID_DIRECT_16(entry) ((entry)<(int32_t)0x80100000)
    287 #define MBCS_ENTRY_FINAL_ACTION(entry) ((((uint32_t)entry)>>20)&0xf)
    288 #define MBCS_ENTRY_FINAL_VALUE(entry) ((entry)&0xfffff)
    289 #define MBCS_ENTRY_FINAL_VALUE_16(entry) (uint16_t)(entry)
    290 
    291 #define IS_ASCII_ROUNDTRIP(b, asciiRoundtrips) (((asciiRoundtrips) & (1<<((b)>>2)))!=0)
    292 
    293 /* single-byte fromUnicode: get the 16-bit result word */
    294 #define MBCS_SINGLE_RESULT_FROM_U(table, results, c) (results)[ (table)[ (table)[(c)>>10] +(((c)>>4)&0x3f) ] +((c)&0xf) ]
    295 
    296 /* single-byte fromUnicode using the sbcsIndex */
    297 #define SBCS_RESULT_FROM_LOW_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
    298 
    299 /* single-byte fromUTF8 using the sbcsIndex; l and t must be masked externally; can be l=0 and t<=0x7f */
    300 #define SBCS_RESULT_FROM_UTF8(table, results, l, t) (results)[ (table)[l] +(t) ]
    301 
    302 /* multi-byte fromUnicode: get the 32-bit stage 2 entry */
    303 #define MBCS_STAGE_2_FROM_U(table, c) ((const uint32_t *)(table))[ (table)[(c)>>10] +(((c)>>4)&0x3f) ]
    304 #define MBCS_FROM_U_IS_ROUNDTRIP(stage2Entry, c) ( ((stage2Entry) & ((uint32_t)1<< (16+((c)&0xf)) )) !=0)
    305 
    306 #define MBCS_VALUE_2_FROM_STAGE_2(bytes, stage2Entry, c) ((uint16_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
    307 #define MBCS_VALUE_4_FROM_STAGE_2(bytes, stage2Entry, c) ((uint32_t *)(bytes))[16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf)]
    308 
    309 #define MBCS_POINTER_3_FROM_STAGE_2(bytes, stage2Entry, c) ((bytes)+(16*(uint32_t)(uint16_t)(stage2Entry)+((c)&0xf))*3)
    310 
    311 /* double-byte fromUnicode using the mbcsIndex */
    312 #define DBCS_RESULT_FROM_MOST_BMP(table, results, c) (results)[ (table)[(c)>>6] +((c)&0x3f) ]
    313 
    314 /* double-byte fromUTF8 using the mbcsIndex; l and t1 combined into lt1; lt1 and t2 must be masked externally */
    315 #define DBCS_RESULT_FROM_UTF8(table, results, lt1, t2) (results)[ (table)[lt1] +(t2) ]
    316 
    317 
    318 /**
    319  * MBCS output types for conversions from Unicode.
    320  * These per-converter types determine the storage method in stage 3 of the lookup table,
    321  * mostly how many bytes are stored per entry.
    322  */
    323 enum {
    324     MBCS_OUTPUT_1,          /* 0 */
    325     MBCS_OUTPUT_2,          /* 1 */
    326     MBCS_OUTPUT_3,          /* 2 */
    327     MBCS_OUTPUT_4,          /* 3 */
    328 
    329     MBCS_OUTPUT_3_EUC=8,    /* 8 */
    330     MBCS_OUTPUT_4_EUC,      /* 9 */
    331 
    332     MBCS_OUTPUT_2_SISO=12,  /* c */
    333     MBCS_OUTPUT_2_HZ,       /* d */
    334 
    335     MBCS_OUTPUT_EXT_ONLY,   /* e */
    336 
    337     MBCS_OUTPUT_COUNT,
    338 
    339     MBCS_OUTPUT_DBCS_ONLY=0xdb  /* runtime-only type for DBCS-only handling of SISO tables */
    340 };
    341 
    342 /**
    343  * Fallbacks to Unicode are stored outside the normal state table and code point structures
    344  * in a vector of items of this type. They are sorted by offset.
    345  */
    346 typedef struct {
    347     uint32_t offset;
    348     UChar32 codePoint;
    349 } _MBCSToUFallback;
    350 
    351 /** Constants for fast and UTF-8-friendly conversion. */
    352 enum {
    353     SBCS_FAST_MAX=0x0fff,               /* maximum code point with UTF-8-friendly SBCS runtime code, see makeconv SBCS_UTF8_MAX */
    354     SBCS_FAST_LIMIT=SBCS_FAST_MAX+1,    /* =0x1000 */
    355     MBCS_FAST_MAX=0xd7ff,               /* maximum code point with UTF-8-friendly MBCS runtime code, see makeconv MBCS_UTF8_MAX */
    356     MBCS_FAST_LIMIT=MBCS_FAST_MAX+1     /* =0xd800 */
    357 };
    358 
    359 /**
    360  * This is the MBCS part of the UConverterTable union (a runtime data structure).
    361  * It keeps all the per-converter data and points into the loaded mapping tables.
    362  *
    363  * utf8Friendly data structures added with _MBCSHeader.version 4.3
    364  */
    365 typedef struct UConverterMBCSTable {
    366     /* toUnicode */
    367     uint8_t countStates, dbcsOnlyState, stateTableOwned;
    368     uint32_t countToUFallbacks;
    369 
    370     const int32_t (*stateTable)/*[countStates]*/[256];
    371     int32_t (*swapLFNLStateTable)/*[countStates]*/[256]; /* for swaplfnl */
    372     const uint16_t *unicodeCodeUnits/*[countUnicodeResults]*/;
    373     const _MBCSToUFallback *toUFallbacks;
    374 
    375     /* fromUnicode */
    376     const uint16_t *fromUnicodeTable;
    377     const uint16_t *mbcsIndex;              /* for fast conversion from most of BMP to MBCS (utf8Friendly data) */
    378     uint16_t sbcsIndex[SBCS_FAST_LIMIT>>6]; /* for fast conversion from low BMP to SBCS (utf8Friendly data) */
    379     const uint8_t *fromUnicodeBytes;
    380     uint8_t *swapLFNLFromUnicodeBytes;      /* for swaplfnl */
    381     uint32_t fromUBytesLength;
    382     uint8_t outputType, unicodeMask;
    383     UBool utf8Friendly;                     /* for utf8Friendly data */
    384     UChar maxFastUChar;                     /* for utf8Friendly data */
    385 
    386     /* roundtrips */
    387     uint32_t asciiRoundtrips;
    388 
    389     /* reconstituted data that was omitted from the .cnv file */
    390     uint8_t *reconstitutedData;
    391 
    392     /* converter name for swaplfnl */
    393     char *swapLFNLName;
    394 
    395     /* extension data */
    396     struct UConverterSharedData *baseSharedData;
    397     const int32_t *extIndexes;
    398 } UConverterMBCSTable;
    399 
    400 #define UCNV_MBCS_TABLE_INITIALIZER { \
    401     /* toUnicode */ \
    402     0, 0, 0, \
    403     0, \
    404      \
    405     NULL, \
    406     NULL, \
    407     NULL, \
    408     NULL, \
    409      \
    410     /* fromUnicode */ \
    411     NULL, \
    412     NULL, \
    413     { 0 }, \
    414     NULL, \
    415     NULL, \
    416     0, \
    417     0, 0, \
    418     FALSE, \
    419     0, \
    420      \
    421     /* roundtrips */ \
    422     0, \
    423      \
    424     /* reconstituted data that was omitted from the .cnv file */ \
    425     NULL, \
    426      \
    427     /* converter name for swaplfnl */ \
    428     NULL, \
    429      \
    430     /* extension data */ \
    431     NULL, \
    432     NULL \
    433 }
    434 
    435 enum {
    436     MBCS_OPT_LENGTH_MASK=0x3f,
    437     MBCS_OPT_NO_FROM_U=0x40,
    438     /*
    439      * If any of the following options bits are set,
    440      * then the file must be rejected.
    441      */
    442     MBCS_OPT_INCOMPATIBLE_MASK=0xffc0,
    443     /*
    444      * Remove bits from this mask as more options are recognized
    445      * by all implementations that use this constant.
    446      */
    447     MBCS_OPT_UNKNOWN_INCOMPATIBLE_MASK=0xff80
    448 };
    449 
    450 enum {
    451     MBCS_HEADER_V4_LENGTH=8,
    452     MBCS_HEADER_V5_MIN_LENGTH=9
    453 };
    454 
    455 /**
    456  * MBCS data header. See data format description above.
    457  */
    458 typedef struct {
    459     UVersionInfo version;
    460     uint32_t countStates,
    461              countToUFallbacks,
    462              offsetToUCodeUnits,
    463              offsetFromUTable,
    464              offsetFromUBytes,
    465              flags,
    466              fromUBytesLength;
    467 
    468     /* new and required in version 5 */
    469     uint32_t options;
    470 
    471     /* new and optional in version 5; used if options&MBCS_OPT_NO_FROM_U */
    472     uint32_t fullStage2Length;  /* number of 32-bit units */
    473 } _MBCSHeader;
    474 
    475 #define UCNV_MBCS_HEADER_INITIALIZER { { 0 },  0, 0, 0, 0, 0, 0, 0,  0,  0 }
    476 
    477 /*
    478  * This is a simple version of _MBCSGetNextUChar() that is used
    479  * by other converter implementations.
    480  * It only returns an "assigned" result if it consumes the entire input.
    481  * It does not use state from the converter, nor error codes.
    482  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
    483  * It handles conversion extensions but not GB 18030.
    484  *
    485  * Return value:
    486  * U+fffe   unassigned
    487  * U+ffff   illegal
    488  * otherwise the Unicode code point
    489  */
    490 U_CFUNC UChar32
    491 ucnv_MBCSSimpleGetNextUChar(UConverterSharedData *sharedData,
    492                         const char *source, int32_t length,
    493                         UBool useFallback);
    494 
    495 /**
    496  * This version of _MBCSSimpleGetNextUChar() is optimized for single-byte, single-state codepages.
    497  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
    498  * It does not handle conversion extensions (_extToU()).
    499  */
    500 U_CFUNC UChar32
    501 ucnv_MBCSSingleSimpleGetNextUChar(UConverterSharedData *sharedData,
    502                               uint8_t b, UBool useFallback);
    503 
    504 /**
    505  * This macro version of _MBCSSingleSimpleGetNextUChar() gets a code point from a byte.
    506  * It works for single-byte, single-state codepages that only map
    507  * to and from BMP code points, and it always
    508  * returns fallback values.
    509  */
    510 #define _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(sharedData, b) \
    511     (UChar)MBCS_ENTRY_FINAL_VALUE_16((sharedData)->mbcs.stateTable[0][(uint8_t)(b)])
    512 
    513 /**
    514  * This is an internal function that allows other converter implementations
    515  * to check whether a byte is a lead byte.
    516  */
    517 U_CFUNC UBool
    518 ucnv_MBCSIsLeadByte(UConverterSharedData *sharedData, char byte);
    519 
    520 /** This is a macro version of _MBCSIsLeadByte(). */
    521 #define _MBCS_IS_LEAD_BYTE(sharedData, byte) \
    522     (UBool)MBCS_ENTRY_IS_TRANSITION((sharedData)->mbcs.stateTable[0][(uint8_t)(byte)])
    523 
    524 /*
    525  * This is another simple conversion function for internal use by other
    526  * conversion implementations.
    527  * It does not use the converter state nor call callbacks.
    528  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
    529  * It handles conversion extensions but not GB 18030.
    530  *
    531  * It converts one single Unicode code point into codepage bytes, encoded
    532  * as one 32-bit value. The function returns the number of bytes in *pValue:
    533  * 1..4 the number of bytes in *pValue
    534  * 0    unassigned (*pValue undefined)
    535  * -1   illegal (currently not used, *pValue undefined)
    536  *
    537  * *pValue will contain the resulting bytes with the last byte in bits 7..0,
    538  * the second to last byte in bits 15..8, etc.
    539  * Currently, the function assumes but does not check that 0<=c<=0x10ffff.
    540  */
    541 U_CFUNC int32_t
    542 ucnv_MBCSFromUChar32(UConverterSharedData *sharedData,
    543                  UChar32 c, uint32_t *pValue,
    544                  UBool useFallback);
    545 
    546 /**
    547  * This version of _MBCSFromUChar32() is optimized for single-byte codepages.
    548  * It does not handle the EBCDIC swaplfnl option (set in UConverter).
    549  *
    550  * It returns the codepage byte for the code point, or -1 if it is unassigned.
    551  */
    552 U_CFUNC int32_t
    553 ucnv_MBCSSingleFromUChar32(UConverterSharedData *sharedData,
    554                        UChar32 c,
    555                        UBool useFallback);
    556 
    557 /**
    558  * SBCS, DBCS, and EBCDIC_STATEFUL are replaced by MBCS, but
    559  * we cheat a little about the type, returning the old types if appropriate.
    560  */
    561 U_CFUNC UConverterType
    562 ucnv_MBCSGetType(const UConverter* converter);
    563 
    564 U_CFUNC void
    565 ucnv_MBCSFromUnicodeWithOffsets(UConverterFromUnicodeArgs *pArgs,
    566                             UErrorCode *pErrorCode);
    567 U_CFUNC void
    568 ucnv_MBCSToUnicodeWithOffsets(UConverterToUnicodeArgs *pArgs,
    569                           UErrorCode *pErrorCode);
    570 
    571 /*
    572  * Internal function returning a UnicodeSet for toUnicode() conversion.
    573  * Currently only used for ISO-2022-CN, and only handles roundtrip mappings.
    574  * In the future, if we add support for fallback sets, this function
    575  * needs to be updated.
    576  * Handles extensions.
    577  * Does not empty the set first.
    578  */
    579 U_CFUNC void
    580 ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
    581                                  const USetAdder *sa,
    582                                  UConverterUnicodeSet which,
    583                                  UErrorCode *pErrorCode);
    584 
    585 /*
    586  * Same as ucnv_MBCSGetUnicodeSetForUnicode() but
    587  * the set can be filtered by encoding scheme.
    588  * Used by stateful converters which share regular conversion tables
    589  * but only use a subset of their mappings.
    590  */
    591 U_CFUNC void
    592 ucnv_MBCSGetFilteredUnicodeSetForUnicode(const UConverterSharedData *sharedData,
    593                                          const USetAdder *sa,
    594                                          UConverterUnicodeSet which,
    595                                          UConverterSetFilter filter,
    596                                          UErrorCode *pErrorCode);
    597 
    598 #endif
    599 
    600 #endif
    601