Home | History | Annotate | Download | only in common
      1 /*
      2 **********************************************************************
      3 *   Copyright (C) 2000-2011, International Business Machines
      4 *   Corporation and others.  All Rights Reserved.
      5 **********************************************************************
      6 *   file name:  ucnv_lmb.cpp
      7 *   encoding:   US-ASCII
      8 *   tab size:   4 (not used)
      9 *   indentation:4
     10 *
     11 *   created on: 2000feb09
     12 *   created by: Brendan Murray
     13 *   extensively hacked up by: Jim Snyder-Grant
     14 *
     15 * Modification History:
     16 *
     17 *   Date        Name             Description
     18 *
     19 *   06/20/2000  helena           OS/400 port changes; mostly typecast.
     20 *   06/27/2000  Jim Snyder-Grant Deal with partial characters and small buffers.
     21 *                                Add comments to document LMBCS format and implementation
     22 *                                restructured order & breakdown of functions
     23 *   06/28/2000  helena           Major rewrite for the callback API changes.
     24 */
     25 
     26 #include "unicode/utypes.h"
     27 
     28 #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION
     29 
     30 #include "unicode/ucnv_err.h"
     31 #include "unicode/ucnv.h"
     32 #include "unicode/uset.h"
     33 #include "cmemory.h"
     34 #include "cstring.h"
     35 #include "uassert.h"
     36 #include "ucnv_imp.h"
     37 #include "ucnv_bld.h"
     38 #include "ucnv_cnv.h"
     39 
     40 #ifdef EBCDIC_RTL
     41     #include "ascii_a.h"
     42 #endif
     43 
     44 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
     45 
     46 /*
     47   LMBCS
     48 
     49   (Lotus Multi-Byte Character Set)
     50 
     51   LMBCS was invented in the late 1980's and is primarily used in Lotus Notes
     52   databases and in Lotus 1-2-3 files. Programmers who work with the APIs
     53   into these products will sometimes need to deal with strings in this format.
     54 
     55   The code in this file provides an implementation for an ICU converter of
     56   LMBCS to and from Unicode.
     57 
     58   Since the LMBCS character set is only sparsely documented in existing
     59   printed or online material, we have added  extensive annotation to this
     60   file to serve as a guide to understanding LMBCS.
     61 
     62   LMBCS was originally designed with these four sometimes-competing design goals:
     63 
     64   -Provide encodings for the characters in 12 existing national standards
     65    (plus a few other characters)
     66   -Minimal memory footprint
     67   -Maximal speed of conversion into the existing national character sets
     68   -No need to track a changing state as you interpret a string.
     69 
     70 
     71   All of the national character sets LMBCS was trying to encode are 'ANSI'
     72   based, in that the bytes from 0x20 - 0x7F are almost exactly the
     73   same common Latin unaccented characters and symbols in all character sets.
     74 
     75   So, in order to help meet the speed & memory design goals, the common ANSI
     76   bytes from 0x20-0x7F are represented by the same single-byte values in LMBCS.
     77 
     78   The general LMBCS code unit is from 1-3 bytes. We can describe the 3 bytes as
     79   follows:
     80 
     81   [G] D1 [D2]
     82 
     83   That is, a sometimes-optional 'group' byte, followed by 1 and sometimes 2
     84   data bytes. The maximum size of a LMBCS chjaracter is 3 bytes:
     85 */
     86 #define ULMBCS_CHARSIZE_MAX      3
     87 /*
     88   The single-byte values from 0x20 to 0x7F are examples of single D1 bytes.
     89   We often have to figure out if byte values are below or above this, so we
     90   use the ANSI nomenclature 'C0' and 'C1' to refer to the range of control
     91   characters just above & below the common lower-ANSI  range */
     92 #define ULMBCS_C0END           0x1F
     93 #define ULMBCS_C1START         0x80
     94 /*
     95   Since LMBCS is always dealing in byte units. we create a local type here for
     96   dealing with these units of LMBCS code units:
     97 
     98 */
     99 typedef uint8_t ulmbcs_byte_t;
    100 
    101 /*
    102    Most of the values less than 0x20 are reserved in LMBCS to announce
    103    which national  character standard is being used for the 'D' bytes.
    104    In the comments we show the common name and the IBM character-set ID
    105    for these character-set announcers:
    106 */
    107 
    108 #define ULMBCS_GRP_L1         0x01   /* Latin-1    :ibm-850  */
    109 #define ULMBCS_GRP_GR         0x02   /* Greek      :ibm-851  */
    110 #define ULMBCS_GRP_HE         0x03   /* Hebrew     :ibm-1255 */
    111 #define ULMBCS_GRP_AR         0x04   /* Arabic     :ibm-1256 */
    112 #define ULMBCS_GRP_RU         0x05   /* Cyrillic   :ibm-1251 */
    113 #define ULMBCS_GRP_L2         0x06   /* Latin-2    :ibm-852  */
    114 #define ULMBCS_GRP_TR         0x08   /* Turkish    :ibm-1254 */
    115 #define ULMBCS_GRP_TH         0x0B   /* Thai       :ibm-874  */
    116 #define ULMBCS_GRP_JA         0x10   /* Japanese   :ibm-943  */
    117 #define ULMBCS_GRP_KO         0x11   /* Korean     :ibm-1261 */
    118 #define ULMBCS_GRP_TW         0x12   /* Chinese SC :ibm-950  */
    119 #define ULMBCS_GRP_CN         0x13   /* Chinese TC :ibm-1386 */
    120 
    121 /*
    122    So, the beginning of understanding LMBCS is that IF the first byte of a LMBCS
    123    character is one of those 12 values, you can interpret the remaining bytes of
    124    that character as coming from one of those character sets. Since the lower
    125    ANSI bytes already are represented in single bytes, using one of the character
    126    set announcers is used to announce a character that starts with a byte of
    127    0x80 or greater.
    128 
    129    The character sets are  arranged so that the single byte sets all appear
    130    before the multi-byte character sets. When we need to tell whether a
    131    group byte is for a single byte char set or not we use this define: */
    132 
    133 #define ULMBCS_DOUBLEOPTGROUP_START  0x10
    134 
    135 /*
    136 However, to fully understand LMBCS, you must also understand a series of
    137 exceptions & optimizations made in service of the design goals.
    138 
    139 First, those of you who are character set mavens may have noticed that
    140 the 'double-byte' character sets are actually multi-byte character sets
    141 that can have 1 or two bytes, even in the upper-ascii range. To force
    142 each group byte to introduce a fixed-width encoding (to make it faster to
    143 count characters), we use a convention of doubling up on the group byte
    144 to introduce any single-byte character > 0x80 in an otherwise double-byte
    145 character set. So, for example, the LMBCS sequence x10 x10 xAE is the
    146 same as '0xAE' in the Japanese code page 943.
    147 
    148 Next, you will notice that the list of group bytes has some gaps.
    149 These are used in various ways.
    150 
    151 We reserve a few special single byte values for common control
    152 characters. These are in the same place as their ANSI eqivalents for speed.
    153 */
    154 
    155 #define ULMBCS_HT    0x09   /* Fixed control char - Horizontal Tab */
    156 #define ULMBCS_LF    0x0A   /* Fixed control char - Line Feed */
    157 #define ULMBCS_CR    0x0D   /* Fixed control char - Carriage Return */
    158 
    159 /* Then, 1-2-3 reserved a special single-byte character to put at the
    160 beginning of internal 'system' range names: */
    161 
    162 #define ULMBCS_123SYSTEMRANGE  0x19
    163 
    164 /* Then we needed a place to put all the other ansi control characters
    165 that must be moved to different values because LMBCS reserves those
    166 values for other purposes. To represent the control characters, we start
    167 with a first byte of 0xF & add the control chaarcter value as the
    168 second byte */
    169 #define ULMBCS_GRP_CTRL       0x0F
    170 
    171 /* For the C0 controls (less than 0x20), we add 0x20 to preserve the
    172 useful doctrine that any byte less than 0x20 in a LMBCS char must be
    173 the first byte of a character:*/
    174 #define ULMBCS_CTRLOFFSET      0x20
    175 
    176 /*
    177 Where to put the characters that aren't part of any of the 12 national
    178 character sets? The first thing that was done, in the earlier years of
    179 LMBCS, was to use up the spaces of the form
    180 
    181   [G] D1,
    182 
    183  where  'G' was one of the single-byte character groups, and
    184  D1 was less than 0x80. These sequences are gathered together
    185  into a Lotus-invented doublebyte character set to represent a
    186  lot of stray values. Internally, in this implementation, we track this
    187  as group '0', as a place to tuck this exceptions list.*/
    188 
    189 #define ULMBCS_GRP_EXCEPT     0x00
    190 /*
    191  Finally, as the durability and usefulness of UNICODE became clear,
    192  LOTUS added a new group 0x14 to hold Unicode values not otherwise
    193  represented in LMBCS: */
    194 #define ULMBCS_GRP_UNICODE    0x14
    195 /* The two bytes appearing after a 0x14 are intrepreted as UFT-16 BE
    196 (Big-Endian) characters. The exception comes when the UTF16
    197 representation would have a zero as the second byte. In that case,
    198 'F6' is used in its place, and the bytes are swapped. (This prevents
    199 LMBCS from encoding any Unicode values of the form U+F6xx, but that's OK:
    200 0xF6xx is in the middle of the Private Use Area.)*/
    201 #define ULMBCS_UNICOMPATZERO   0xF6
    202 
    203 /* It is also useful in our code to have a constant for the size of
    204 a LMBCS char that holds a literal Unicode value */
    205 #define ULMBCS_UNICODE_SIZE      3
    206 
    207 /*
    208 To squish the LMBCS representations down even further, and to make
    209 translations even faster,sometimes the optimization group byte can be dropped
    210 from a LMBCS character. This is decided on a process-by-process basis. The
    211 group byte that is dropped is called the 'optimization group'.
    212 
    213 For Notes, the optimzation group is always 0x1.*/
    214 #define ULMBCS_DEFAULTOPTGROUP 0x1
    215 /* For 1-2-3 files, the optimzation group is stored in the header of the 1-2-3
    216 file.
    217 
    218  In any case, when using ICU, you either pass in the
    219 optimization group as part of the name of the converter (LMBCS-1, LMBCS-2,
    220 etc.). Using plain 'LMBCS' as the name of the converter will give you
    221 LMBCS-1.
    222 
    223 
    224 *** Implementation strategy ***
    225 
    226 
    227 Because of the extensive use of other character sets, the LMBCS converter
    228 keeps a mapping between optimization groups and IBM character sets, so that
    229 ICU converters can be created and used as needed. */
    230 
    231 /* As you can see, even though any byte below 0x20 could be an optimization
    232 byte, only those at 0x13 or below can map to an actual converter. To limit
    233 some loops and searches, we define a value for that last group converter:*/
    234 
    235 #define ULMBCS_GRP_LAST       0x13   /* last LMBCS group that has a converter */
    236 
    237 static const char * const OptGroupByteToCPName[ULMBCS_GRP_LAST + 1] = {
    238    /* 0x0000 */ "lmb-excp", /* internal home for the LOTUS exceptions list */
    239    /* 0x0001 */ "ibm-850",
    240    /* 0x0002 */ "ibm-851",
    241    /* 0x0003 */ "windows-1255",
    242    /* 0x0004 */ "windows-1256",
    243    /* 0x0005 */ "windows-1251",
    244    /* 0x0006 */ "ibm-852",
    245    /* 0x0007 */ NULL,      /* Unused */
    246    /* 0x0008 */ "windows-1254",
    247    /* 0x0009 */ NULL,      /* Control char HT */
    248    /* 0x000A */ NULL,      /* Control char LF */
    249    /* 0x000B */ "windows-874",
    250    /* 0x000C */ NULL,      /* Unused */
    251    /* 0x000D */ NULL,      /* Control char CR */
    252    /* 0x000E */ NULL,      /* Unused */
    253    /* 0x000F */ NULL,      /* Control chars: 0x0F20 + C0/C1 character: algorithmic */
    254    /* 0x0010 */ "windows-932",
    255    /* 0x0011 */ "windows-949",
    256    /* 0x0012 */ "windows-950",
    257    /* 0x0013 */ "windows-936"
    258 
    259    /* The rest are null, including the 0x0014 Unicode compatibility region
    260    and 0x0019, the 1-2-3 system range control char */
    261 };
    262 
    263 
    264 /* That's approximately all the data that's needed for translating
    265   LMBCS to Unicode.
    266 
    267 
    268 However, to translate Unicode to LMBCS, we need some more support.
    269 
    270 That's because there are often more than one possible mappings from a Unicode
    271 code point back into LMBCS. The first thing we do is look up into a table
    272 to figure out if there are more than one possible mappings. This table,
    273 arranged by Unicode values (including ranges) either lists which group
    274 to use, or says that it could go into one or more of the SBCS sets, or
    275 into one or more of the DBCS sets.  (If the character exists in both DBCS &
    276 SBCS, the table will place it in the SBCS sets, to make the LMBCS code point
    277 length as small as possible. Here's the two special markers we use to indicate
    278 ambiguous mappings: */
    279 
    280 #define ULMBCS_AMBIGUOUS_SBCS   0x80   /* could fit in more than one
    281                                           LMBCS sbcs native encoding
    282                                           (example: most accented latin) */
    283 #define ULMBCS_AMBIGUOUS_MBCS   0x81   /* could fit in more than one
    284                                           LMBCS mbcs native encoding
    285                                           (example: Unihan) */
    286 #define ULMBCS_AMBIGUOUS_ALL   0x82
    287 /* And here's a simple way to see if a group falls in an appropriate range */
    288 #define ULMBCS_AMBIGUOUS_MATCH(agroup, xgroup) \
    289                   ((((agroup) == ULMBCS_AMBIGUOUS_SBCS) && \
    290                   (xgroup) < ULMBCS_DOUBLEOPTGROUP_START) || \
    291                   (((agroup) == ULMBCS_AMBIGUOUS_MBCS) && \
    292                   (xgroup) >= ULMBCS_DOUBLEOPTGROUP_START)) || \
    293                   ((agroup) == ULMBCS_AMBIGUOUS_ALL)
    294 
    295 
    296 /* The table & some code to use it: */
    297 
    298 
    299 static const struct _UniLMBCSGrpMap
    300 {
    301    const UChar uniStartRange;
    302    const UChar uniEndRange;
    303    const ulmbcs_byte_t  GrpType;
    304 } UniLMBCSGrpMap[]
    305 =
    306 {
    307 
    308     {0x0001, 0x001F,  ULMBCS_GRP_CTRL},
    309     {0x0080, 0x009F,  ULMBCS_GRP_CTRL},
    310     {0x00A0, 0x00A6,  ULMBCS_AMBIGUOUS_SBCS},
    311     {0x00A7, 0x00A8,  ULMBCS_AMBIGUOUS_ALL},
    312     {0x00A9, 0x00AF,  ULMBCS_AMBIGUOUS_SBCS},
    313     {0x00B0, 0x00B1,  ULMBCS_AMBIGUOUS_ALL},
    314     {0x00B2, 0x00B3,  ULMBCS_AMBIGUOUS_SBCS},
    315     {0x00B4, 0x00B4,  ULMBCS_AMBIGUOUS_ALL},
    316     {0x00B5, 0x00B5,  ULMBCS_AMBIGUOUS_SBCS},
    317     {0x00B6, 0x00B6,  ULMBCS_AMBIGUOUS_ALL},
    318     {0x00B7, 0x00D6,  ULMBCS_AMBIGUOUS_SBCS},
    319     {0x00D7, 0x00D7,  ULMBCS_AMBIGUOUS_ALL},
    320     {0x00D8, 0x00F6,  ULMBCS_AMBIGUOUS_SBCS},
    321     {0x00F7, 0x00F7,  ULMBCS_AMBIGUOUS_ALL},
    322     {0x00F8, 0x01CD,  ULMBCS_AMBIGUOUS_SBCS},
    323     {0x01CE, 0x01CE,  ULMBCS_GRP_TW },
    324     {0x01CF, 0x02B9,  ULMBCS_AMBIGUOUS_SBCS},
    325     {0x02BA, 0x02BA,  ULMBCS_GRP_CN},
    326     {0x02BC, 0x02C8,  ULMBCS_AMBIGUOUS_SBCS},
    327     {0x02C9, 0x02D0,  ULMBCS_AMBIGUOUS_MBCS},
    328     {0x02D8, 0x02DD,  ULMBCS_AMBIGUOUS_SBCS},
    329     {0x0384, 0x0390,  ULMBCS_AMBIGUOUS_SBCS},
    330     {0x0391, 0x03A9,  ULMBCS_AMBIGUOUS_ALL},
    331     {0x03AA, 0x03B0,  ULMBCS_AMBIGUOUS_SBCS},
    332     {0x03B1, 0x03C9,  ULMBCS_AMBIGUOUS_ALL},
    333     {0x03CA, 0x03CE,  ULMBCS_AMBIGUOUS_SBCS},
    334     {0x0400, 0x0400,  ULMBCS_GRP_RU},
    335     {0x0401, 0x0401,  ULMBCS_AMBIGUOUS_ALL},
    336     {0x0402, 0x040F,  ULMBCS_GRP_RU},
    337     {0x0410, 0x0431,  ULMBCS_AMBIGUOUS_ALL},
    338     {0x0432, 0x044E,  ULMBCS_GRP_RU},
    339     {0x044F, 0x044F,  ULMBCS_AMBIGUOUS_ALL},
    340     {0x0450, 0x0491,  ULMBCS_GRP_RU},
    341     {0x05B0, 0x05F2,  ULMBCS_GRP_HE},
    342     {0x060C, 0x06AF,  ULMBCS_GRP_AR},
    343     {0x0E01, 0x0E5B,  ULMBCS_GRP_TH},
    344     {0x200C, 0x200F,  ULMBCS_AMBIGUOUS_SBCS},
    345     {0x2010, 0x2010,  ULMBCS_AMBIGUOUS_MBCS},
    346     {0x2013, 0x2014,  ULMBCS_AMBIGUOUS_SBCS},
    347     {0x2015, 0x2015,  ULMBCS_AMBIGUOUS_MBCS},
    348     {0x2016, 0x2016,  ULMBCS_AMBIGUOUS_MBCS},
    349     {0x2017, 0x2017,  ULMBCS_AMBIGUOUS_SBCS},
    350     {0x2018, 0x2019,  ULMBCS_AMBIGUOUS_ALL},
    351     {0x201A, 0x201B,  ULMBCS_AMBIGUOUS_SBCS},
    352     {0x201C, 0x201D,  ULMBCS_AMBIGUOUS_ALL},
    353     {0x201E, 0x201F,  ULMBCS_AMBIGUOUS_SBCS},
    354     {0x2020, 0x2021,  ULMBCS_AMBIGUOUS_ALL},
    355     {0x2022, 0x2024,  ULMBCS_AMBIGUOUS_SBCS},
    356     {0x2025, 0x2025,  ULMBCS_AMBIGUOUS_MBCS},
    357     {0x2026, 0x2026,  ULMBCS_AMBIGUOUS_ALL},
    358     {0x2027, 0x2027,  ULMBCS_GRP_TW},
    359     {0x2030, 0x2030,  ULMBCS_AMBIGUOUS_ALL},
    360     {0x2031, 0x2031,  ULMBCS_AMBIGUOUS_SBCS},
    361     {0x2032, 0x2033,  ULMBCS_AMBIGUOUS_MBCS},
    362     {0x2035, 0x2035,  ULMBCS_AMBIGUOUS_MBCS},
    363     {0x2039, 0x203A,  ULMBCS_AMBIGUOUS_SBCS},
    364     {0x203B, 0x203B,  ULMBCS_AMBIGUOUS_MBCS},
    365     {0x203C, 0x203C,  ULMBCS_GRP_EXCEPT},
    366     {0x2074, 0x2074,  ULMBCS_GRP_KO},
    367     {0x207F, 0x207F,  ULMBCS_GRP_EXCEPT},
    368     {0x2081, 0x2084,  ULMBCS_GRP_KO},
    369     {0x20A4, 0x20AC,  ULMBCS_AMBIGUOUS_SBCS},
    370     {0x2103, 0x2109,  ULMBCS_AMBIGUOUS_MBCS},
    371     {0x2111, 0x2120,  ULMBCS_AMBIGUOUS_SBCS},
    372     /*zhujin: upgrade, for regressiont test, spr HKIA4YHTSU*/
    373     {0x2121, 0x2121,  ULMBCS_AMBIGUOUS_MBCS},
    374     {0x2122, 0x2126,  ULMBCS_AMBIGUOUS_SBCS},
    375     {0x212B, 0x212B,  ULMBCS_AMBIGUOUS_MBCS},
    376     {0x2135, 0x2135,  ULMBCS_AMBIGUOUS_SBCS},
    377     {0x2153, 0x2154,  ULMBCS_GRP_KO},
    378     {0x215B, 0x215E,  ULMBCS_GRP_EXCEPT},
    379     {0x2160, 0x2179,  ULMBCS_AMBIGUOUS_MBCS},
    380     {0x2190, 0x2193,  ULMBCS_AMBIGUOUS_ALL},
    381     {0x2194, 0x2195,  ULMBCS_GRP_EXCEPT},
    382     {0x2196, 0x2199,  ULMBCS_AMBIGUOUS_MBCS},
    383     {0x21A8, 0x21A8,  ULMBCS_GRP_EXCEPT},
    384     {0x21B8, 0x21B9,  ULMBCS_GRP_CN},
    385     {0x21D0, 0x21D1,  ULMBCS_GRP_EXCEPT},
    386     {0x21D2, 0x21D2,  ULMBCS_AMBIGUOUS_MBCS},
    387     {0x21D3, 0x21D3,  ULMBCS_GRP_EXCEPT},
    388     {0x21D4, 0x21D4,  ULMBCS_AMBIGUOUS_MBCS},
    389     {0x21D5, 0x21D5,  ULMBCS_GRP_EXCEPT},
    390     {0x21E7, 0x21E7,  ULMBCS_GRP_CN},
    391     {0x2200, 0x2200,  ULMBCS_AMBIGUOUS_MBCS},
    392     {0x2201, 0x2201,  ULMBCS_GRP_EXCEPT},
    393     {0x2202, 0x2202,  ULMBCS_AMBIGUOUS_MBCS},
    394     {0x2203, 0x2203,  ULMBCS_AMBIGUOUS_MBCS},
    395     {0x2204, 0x2206,  ULMBCS_GRP_EXCEPT},
    396     {0x2207, 0x2208,  ULMBCS_AMBIGUOUS_MBCS},
    397     {0x2209, 0x220A,  ULMBCS_GRP_EXCEPT},
    398     {0x220B, 0x220B,  ULMBCS_AMBIGUOUS_MBCS},
    399     {0x220F, 0x2215,  ULMBCS_AMBIGUOUS_MBCS},
    400     {0x2219, 0x2219,  ULMBCS_GRP_EXCEPT},
    401     {0x221A, 0x221A,  ULMBCS_AMBIGUOUS_MBCS},
    402     {0x221B, 0x221C,  ULMBCS_GRP_EXCEPT},
    403     {0x221D, 0x221E,  ULMBCS_AMBIGUOUS_MBCS},
    404     {0x221F, 0x221F,  ULMBCS_GRP_EXCEPT},
    405     {0x2220, 0x2220,  ULMBCS_AMBIGUOUS_MBCS},
    406     {0x2223, 0x222A,  ULMBCS_AMBIGUOUS_MBCS},
    407     {0x222B, 0x223D,  ULMBCS_AMBIGUOUS_MBCS},
    408     {0x2245, 0x2248,  ULMBCS_GRP_EXCEPT},
    409     {0x224C, 0x224C,  ULMBCS_GRP_TW},
    410     {0x2252, 0x2252,  ULMBCS_AMBIGUOUS_MBCS},
    411     {0x2260, 0x2261,  ULMBCS_AMBIGUOUS_MBCS},
    412     {0x2262, 0x2265,  ULMBCS_GRP_EXCEPT},
    413     {0x2266, 0x226F,  ULMBCS_AMBIGUOUS_MBCS},
    414     {0x2282, 0x2283,  ULMBCS_AMBIGUOUS_MBCS},
    415     {0x2284, 0x2285,  ULMBCS_GRP_EXCEPT},
    416     {0x2286, 0x2287,  ULMBCS_AMBIGUOUS_MBCS},
    417     {0x2288, 0x2297,  ULMBCS_GRP_EXCEPT},
    418     {0x2299, 0x22BF,  ULMBCS_AMBIGUOUS_MBCS},
    419     {0x22C0, 0x22C0,  ULMBCS_GRP_EXCEPT},
    420     {0x2310, 0x2310,  ULMBCS_GRP_EXCEPT},
    421     {0x2312, 0x2312,  ULMBCS_AMBIGUOUS_MBCS},
    422     {0x2318, 0x2321,  ULMBCS_GRP_EXCEPT},
    423     {0x2318, 0x2321,  ULMBCS_GRP_CN},
    424     {0x2460, 0x24E9,  ULMBCS_AMBIGUOUS_MBCS},
    425     {0x2500, 0x2500,  ULMBCS_AMBIGUOUS_SBCS},
    426     {0x2501, 0x2501,  ULMBCS_AMBIGUOUS_MBCS},
    427     {0x2502, 0x2502,  ULMBCS_AMBIGUOUS_ALL},
    428     {0x2503, 0x2503,  ULMBCS_AMBIGUOUS_MBCS},
    429     {0x2504, 0x2505,  ULMBCS_GRP_TW},
    430     {0x2506, 0x2665,  ULMBCS_AMBIGUOUS_ALL},
    431     {0x2666, 0x2666,  ULMBCS_GRP_EXCEPT},
    432     {0x2667, 0x2669,  ULMBCS_AMBIGUOUS_SBCS},
    433     {0x266A, 0x266A,  ULMBCS_AMBIGUOUS_ALL},
    434     {0x266B, 0x266C,  ULMBCS_AMBIGUOUS_SBCS},
    435     {0x266D, 0x266D,  ULMBCS_AMBIGUOUS_MBCS},
    436     {0x266E, 0x266E,  ULMBCS_AMBIGUOUS_SBCS},
    437     {0x266F, 0x266F,  ULMBCS_GRP_JA},
    438     {0x2670, 0x2E7F,  ULMBCS_AMBIGUOUS_SBCS},
    439     {0x2E80, 0xF861,  ULMBCS_AMBIGUOUS_MBCS},
    440     {0xF862, 0xF8FF,  ULMBCS_GRP_EXCEPT},
    441     {0xF900, 0xFA2D,  ULMBCS_AMBIGUOUS_MBCS},
    442     {0xFB00, 0xFEFF,  ULMBCS_AMBIGUOUS_SBCS},
    443     {0xFF01, 0xFFEE,  ULMBCS_AMBIGUOUS_MBCS},
    444     {0xFFFF, 0xFFFF,  ULMBCS_GRP_UNICODE}
    445 };
    446 
    447 static ulmbcs_byte_t
    448 FindLMBCSUniRange(UChar uniChar)
    449 {
    450    const struct _UniLMBCSGrpMap * pTable = UniLMBCSGrpMap;
    451 
    452    while (uniChar > pTable->uniEndRange)
    453    {
    454       pTable++;
    455    }
    456 
    457    if (uniChar >= pTable->uniStartRange)
    458    {
    459       return pTable->GrpType;
    460    }
    461    return ULMBCS_GRP_UNICODE;
    462 }
    463 
    464 /*
    465 We also ask the creator of a converter to send in a preferred locale
    466 that we can use in resolving ambiguous mappings. They send the locale
    467 in as a string, and we map it, if possible, to one of the
    468 LMBCS groups. We use this table, and the associated code, to
    469 do the lookup: */
    470 
    471 /**************************************************
    472   This table maps locale ID's to LMBCS opt groups.
    473   The default return is group 0x01. Note that for
    474   performance reasons, the table is sorted in
    475   increasing alphabetic order, with the notable
    476   exception of zhTW. This is to force the check
    477   for Traditonal Chinese before dropping back to
    478   Simplified.
    479 
    480   Note too that the Latin-1 groups have been
    481   commented out because it's the default, and
    482   this shortens the table, allowing a serial
    483   search to go quickly.
    484  *************************************************/
    485 
    486 static const struct _LocaleLMBCSGrpMap
    487 {
    488    const char    *LocaleID;
    489    const ulmbcs_byte_t OptGroup;
    490 } LocaleLMBCSGrpMap[] =
    491 {
    492     {"ar", ULMBCS_GRP_AR},
    493     {"be", ULMBCS_GRP_RU},
    494     {"bg", ULMBCS_GRP_L2},
    495    /* {"ca", ULMBCS_GRP_L1}, */
    496     {"cs", ULMBCS_GRP_L2},
    497    /* {"da", ULMBCS_GRP_L1}, */
    498    /* {"de", ULMBCS_GRP_L1}, */
    499     {"el", ULMBCS_GRP_GR},
    500    /* {"en", ULMBCS_GRP_L1}, */
    501    /* {"es", ULMBCS_GRP_L1}, */
    502    /* {"et", ULMBCS_GRP_L1}, */
    503    /* {"fi", ULMBCS_GRP_L1}, */
    504    /* {"fr", ULMBCS_GRP_L1}, */
    505     {"he", ULMBCS_GRP_HE},
    506     {"hu", ULMBCS_GRP_L2},
    507    /* {"is", ULMBCS_GRP_L1}, */
    508    /* {"it", ULMBCS_GRP_L1}, */
    509     {"iw", ULMBCS_GRP_HE},
    510     {"ja", ULMBCS_GRP_JA},
    511     {"ko", ULMBCS_GRP_KO},
    512    /* {"lt", ULMBCS_GRP_L1}, */
    513    /* {"lv", ULMBCS_GRP_L1}, */
    514     {"mk", ULMBCS_GRP_RU},
    515    /* {"nl", ULMBCS_GRP_L1}, */
    516    /* {"no", ULMBCS_GRP_L1}, */
    517     {"pl", ULMBCS_GRP_L2},
    518    /* {"pt", ULMBCS_GRP_L1}, */
    519     {"ro", ULMBCS_GRP_L2},
    520     {"ru", ULMBCS_GRP_RU},
    521     {"sh", ULMBCS_GRP_L2},
    522     {"sk", ULMBCS_GRP_L2},
    523     {"sl", ULMBCS_GRP_L2},
    524     {"sq", ULMBCS_GRP_L2},
    525     {"sr", ULMBCS_GRP_RU},
    526    /* {"sv", ULMBCS_GRP_L1}, */
    527     {"th", ULMBCS_GRP_TH},
    528     {"tr", ULMBCS_GRP_TR},
    529     {"uk", ULMBCS_GRP_RU},
    530    /* {"vi", ULMBCS_GRP_L1}, */
    531     {"zhTW", ULMBCS_GRP_TW},
    532     {"zh", ULMBCS_GRP_CN},
    533     {NULL, ULMBCS_GRP_L1}
    534 };
    535 
    536 
    537 static ulmbcs_byte_t
    538 FindLMBCSLocale(const char *LocaleID)
    539 {
    540    const struct _LocaleLMBCSGrpMap *pTable = LocaleLMBCSGrpMap;
    541 
    542    if ((!LocaleID) || (!*LocaleID))
    543    {
    544       return 0;
    545    }
    546 
    547    while (pTable->LocaleID)
    548    {
    549       if (*pTable->LocaleID == *LocaleID) /* Check only first char for speed */
    550       {
    551          /* First char matches - check whole name, for entry-length */
    552          if (uprv_strncmp(pTable->LocaleID, LocaleID, strlen(pTable->LocaleID)) == 0)
    553             return pTable->OptGroup;
    554       }
    555       else
    556       if (*pTable->LocaleID > *LocaleID) /* Sorted alphabetically - exit */
    557          break;
    558       pTable++;
    559    }
    560    return ULMBCS_GRP_L1;
    561 }
    562 
    563 
    564 /*
    565   Before we get to the main body of code, here's how we hook up to the rest
    566   of ICU. ICU converters are required to define a structure that includes
    567   some function pointers, and some common data, in the style of a C++
    568   vtable. There is also room in there for converter-specific data. LMBCS
    569   uses that converter-specific data to keep track of the 12 subconverters
    570   we use, the optimization group, and the group (if any) that matches the
    571   locale. We have one structure instantiated for each of the 12 possible
    572   optimization groups. To avoid typos & to avoid boring the reader, we
    573   put the declarations of these structures and functions into macros. To see
    574   the definitions of these structures, see unicode\ucnv_bld.h
    575 */
    576 
    577 typedef struct
    578   {
    579     UConverterSharedData *OptGrpConverter[ULMBCS_GRP_LAST+1];    /* Converter per Opt. grp. */
    580     uint8_t    OptGroup;                  /* default Opt. grp. for this LMBCS session */
    581     uint8_t    localeConverterIndex;      /* reasonable locale match for index */
    582   }
    583 UConverterDataLMBCS;
    584 
    585 static void _LMBCSClose(UConverter * _this);
    586 
    587 #define DECLARE_LMBCS_DATA(n) \
    588 static const UConverterImpl _LMBCSImpl##n={\
    589     UCNV_LMBCS_##n,\
    590     NULL,NULL,\
    591     _LMBCSOpen##n,\
    592     _LMBCSClose,\
    593     NULL,\
    594     _LMBCSToUnicodeWithOffsets,\
    595     _LMBCSToUnicodeWithOffsets,\
    596     _LMBCSFromUnicode,\
    597     _LMBCSFromUnicode,\
    598     NULL,\
    599     NULL,\
    600     NULL,\
    601     NULL,\
    602     _LMBCSSafeClone,\
    603     ucnv_getCompleteUnicodeSet\
    604 };\
    605 static const UConverterStaticData _LMBCSStaticData##n={\
    606   sizeof(UConverterStaticData),\
    607  "LMBCS-"  #n,\
    608     0, UCNV_IBM, UCNV_LMBCS_##n, 1, 3,\
    609     { 0x3f, 0, 0, 0 },1,FALSE,FALSE,0,0,{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0} \
    610 };\
    611 const UConverterSharedData _LMBCSData##n={\
    612     sizeof(UConverterSharedData), ~((uint32_t) 0),\
    613     NULL, NULL, &_LMBCSStaticData##n, FALSE, &_LMBCSImpl##n, \
    614     0 \
    615 };
    616 
    617  /* The only function we needed to duplicate 12 times was the 'open'
    618 function, which will do basically the same thing except set a  different
    619 optimization group. So, we put the common stuff into a worker function,
    620 and set up another macro to stamp out the 12 open functions:*/
    621 #define DEFINE_LMBCS_OPEN(n) \
    622 static void \
    623    _LMBCSOpen##n(UConverter* _this, UConverterLoadArgs* pArgs, UErrorCode* err) \
    624 { _LMBCSOpenWorker(_this, pArgs, err, n); }
    625 
    626 
    627 
    628 /* Here's the open worker & the common close function */
    629 static void
    630 _LMBCSOpenWorker(UConverter*  _this,
    631                  UConverterLoadArgs *pArgs,
    632                  UErrorCode*  err,
    633                  ulmbcs_byte_t OptGroup)
    634 {
    635     UConverterDataLMBCS * extraInfo = _this->extraInfo =
    636         (UConverterDataLMBCS*)uprv_malloc (sizeof (UConverterDataLMBCS));
    637     if(extraInfo != NULL)
    638     {
    639         UConverterNamePieces stackPieces;
    640         UConverterLoadArgs stackArgs={ (int32_t)sizeof(UConverterLoadArgs) };
    641         ulmbcs_byte_t i;
    642 
    643         uprv_memset(extraInfo, 0, sizeof(UConverterDataLMBCS));
    644 
    645         stackArgs.onlyTestIsLoadable = pArgs->onlyTestIsLoadable;
    646 
    647         for (i=0; i <= ULMBCS_GRP_LAST && U_SUCCESS(*err); i++)
    648         {
    649             if(OptGroupByteToCPName[i] != NULL) {
    650                 extraInfo->OptGrpConverter[i] = ucnv_loadSharedData(OptGroupByteToCPName[i], &stackPieces, &stackArgs, err);
    651             }
    652         }
    653 
    654         if(U_FAILURE(*err) || pArgs->onlyTestIsLoadable) {
    655             _LMBCSClose(_this);
    656             return;
    657         }
    658         extraInfo->OptGroup = OptGroup;
    659         extraInfo->localeConverterIndex = FindLMBCSLocale(pArgs->locale);
    660     }
    661     else
    662     {
    663         *err = U_MEMORY_ALLOCATION_ERROR;
    664     }
    665 }
    666 
    667 static void
    668 _LMBCSClose(UConverter *   _this)
    669 {
    670     if (_this->extraInfo != NULL)
    671     {
    672         ulmbcs_byte_t Ix;
    673         UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) _this->extraInfo;
    674 
    675         for (Ix=0; Ix <= ULMBCS_GRP_LAST; Ix++)
    676         {
    677            if (extraInfo->OptGrpConverter[Ix] != NULL)
    678               ucnv_unloadSharedDataIfReady(extraInfo->OptGrpConverter[Ix]);
    679         }
    680         if (!_this->isExtraLocal) {
    681             uprv_free (_this->extraInfo);
    682             _this->extraInfo = NULL;
    683         }
    684     }
    685 }
    686 
    687 typedef struct LMBCSClone {
    688     UConverter cnv;
    689     UConverterDataLMBCS lmbcs;
    690 } LMBCSClone;
    691 
    692 static UConverter *
    693 _LMBCSSafeClone(const UConverter *cnv,
    694                 void *stackBuffer,
    695                 int32_t *pBufferSize,
    696                 UErrorCode *status) {
    697     LMBCSClone *newLMBCS;
    698     UConverterDataLMBCS *extraInfo;
    699     int32_t i;
    700 
    701     if(*pBufferSize<=0) {
    702         *pBufferSize=(int32_t)sizeof(LMBCSClone);
    703         return NULL;
    704     }
    705 
    706     extraInfo=(UConverterDataLMBCS *)cnv->extraInfo;
    707     newLMBCS=(LMBCSClone *)stackBuffer;
    708 
    709     /* ucnv.c/ucnv_safeClone() copied the main UConverter already */
    710 
    711     uprv_memcpy(&newLMBCS->lmbcs, extraInfo, sizeof(UConverterDataLMBCS));
    712 
    713     /* share the subconverters */
    714     for(i = 0; i <= ULMBCS_GRP_LAST; ++i) {
    715         if(extraInfo->OptGrpConverter[i] != NULL) {
    716             ucnv_incrementRefCount(extraInfo->OptGrpConverter[i]);
    717         }
    718     }
    719 
    720     newLMBCS->cnv.extraInfo = &newLMBCS->lmbcs;
    721     newLMBCS->cnv.isExtraLocal = TRUE;
    722     return &newLMBCS->cnv;
    723 }
    724 
    725 /*
    726  * There used to be a _LMBCSGetUnicodeSet() function here (up to svn revision 20117)
    727  * which added all code points except for U+F6xx
    728  * because those cannot be represented in the Unicode group.
    729  * However, it turns out that windows-950 has roundtrips for all of U+F6xx
    730  * which means that LMBCS can convert all Unicode code points after all.
    731  * We now simply use ucnv_getCompleteUnicodeSet().
    732  *
    733  * This may need to be looked at again as Lotus uses _LMBCSGetUnicodeSet(). (091216)
    734  */
    735 
    736 /*
    737    Here's the basic helper function that we use when converting from
    738    Unicode to LMBCS, and we suspect that a Unicode character will fit into
    739    one of the 12 groups. The return value is the number of bytes written
    740    starting at pStartLMBCS (if any).
    741 */
    742 
    743 static size_t
    744 LMBCSConversionWorker (
    745    UConverterDataLMBCS * extraInfo,    /* subconverters, opt & locale groups */
    746    ulmbcs_byte_t group,                /* The group to try */
    747    ulmbcs_byte_t  * pStartLMBCS,              /* where to put the results */
    748    UChar * pUniChar,                   /* The input unicode character */
    749    ulmbcs_byte_t * lastConverterIndex, /* output: track last successful group used */
    750    UBool * groups_tried                /* output: track any unsuccessful groups */
    751 )
    752 {
    753    ulmbcs_byte_t  * pLMBCS = pStartLMBCS;
    754    UConverterSharedData * xcnv = extraInfo->OptGrpConverter[group];
    755 
    756    int bytesConverted;
    757    uint32_t value;
    758    ulmbcs_byte_t firstByte;
    759 
    760    U_ASSERT(xcnv);
    761    U_ASSERT(group<ULMBCS_GRP_UNICODE);
    762 
    763    bytesConverted = ucnv_MBCSFromUChar32(xcnv, *pUniChar, &value, FALSE);
    764 
    765    /* get the first result byte */
    766    if(bytesConverted > 0) {
    767       firstByte = (ulmbcs_byte_t)(value >> ((bytesConverted - 1) * 8));
    768    } else {
    769       /* most common failure mode is an unassigned character */
    770       groups_tried[group] = TRUE;
    771       return 0;
    772    }
    773 
    774    *lastConverterIndex = group;
    775 
    776    /* All initial byte values in lower ascii range should have been caught by now,
    777       except with the exception group.
    778     */
    779    U_ASSERT((firstByte <= ULMBCS_C0END) || (firstByte >= ULMBCS_C1START) || (group == ULMBCS_GRP_EXCEPT));
    780 
    781    /* use converted data: first write 0, 1 or two group bytes */
    782    if (group != ULMBCS_GRP_EXCEPT && extraInfo->OptGroup != group)
    783    {
    784       *pLMBCS++ = group;
    785       if (bytesConverted == 1 && group >= ULMBCS_DOUBLEOPTGROUP_START)
    786       {
    787          *pLMBCS++ = group;
    788       }
    789    }
    790 
    791   /* don't emit control chars */
    792    if ( bytesConverted == 1 && firstByte < 0x20 )
    793       return 0;
    794 
    795 
    796    /* then move over the converted data */
    797    switch(bytesConverted)
    798    {
    799    case 4:
    800       *pLMBCS++ = (ulmbcs_byte_t)(value >> 24);
    801    case 3: /*fall through*/
    802       *pLMBCS++ = (ulmbcs_byte_t)(value >> 16);
    803    case 2: /*fall through*/
    804       *pLMBCS++ = (ulmbcs_byte_t)(value >> 8);
    805    case 1: /*fall through*/
    806       *pLMBCS++ = (ulmbcs_byte_t)value;
    807    default:
    808       /* will never occur */
    809       break;
    810    }
    811 
    812    return (pLMBCS - pStartLMBCS);
    813 }
    814 
    815 
    816 /* This is a much simpler version of above, when we
    817 know we are writing LMBCS using the Unicode group
    818 */
    819 static size_t
    820 LMBCSConvertUni(ulmbcs_byte_t * pLMBCS, UChar uniChar)
    821 {
    822      /* encode into LMBCS Unicode range */
    823    uint8_t LowCh =   (uint8_t)(uniChar & 0x00FF);
    824    uint8_t HighCh  = (uint8_t)(uniChar >> 8);
    825 
    826    *pLMBCS++ = ULMBCS_GRP_UNICODE;
    827 
    828    if (LowCh == 0)
    829    {
    830       *pLMBCS++ = ULMBCS_UNICOMPATZERO;
    831       *pLMBCS++ = HighCh;
    832    }
    833    else
    834    {
    835       *pLMBCS++ = HighCh;
    836       *pLMBCS++ = LowCh;
    837    }
    838    return ULMBCS_UNICODE_SIZE;
    839 }
    840 
    841 
    842 
    843 /* The main Unicode to LMBCS conversion function */
    844 static void
    845 _LMBCSFromUnicode(UConverterFromUnicodeArgs*     args,
    846                   UErrorCode*     err)
    847 {
    848    ulmbcs_byte_t lastConverterIndex = 0;
    849    UChar uniChar;
    850    ulmbcs_byte_t  LMBCS[ULMBCS_CHARSIZE_MAX];
    851    ulmbcs_byte_t  * pLMBCS;
    852    int32_t bytes_written;
    853    UBool groups_tried[ULMBCS_GRP_LAST+1];
    854    UConverterDataLMBCS * extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo;
    855    int sourceIndex = 0;
    856 
    857    /* Basic strategy: attempt to fill in local LMBCS 1-char buffer.(LMBCS)
    858       If that succeeds, see if it will all fit into the target & copy it over
    859       if it does.
    860 
    861       We try conversions in the following order:
    862 
    863       1. Single-byte ascii & special fixed control chars (&null)
    864       2. Look up group in table & try that (could be
    865             A) Unicode group
    866             B) control group,
    867             C) national encoding,
    868                or ambiguous SBCS or MBCS group (on to step 4...)
    869 
    870       3. If its ambiguous, try this order:
    871          A) The optimization group
    872          B) The locale group
    873          C) The last group that succeeded with this string.
    874          D) every other group that's relevent (single or double)
    875          E) If its single-byte ambiguous, try the exceptions group
    876 
    877       4. And as a grand fallback: Unicode
    878    */
    879 
    880     /*Fix for SPR#DJOE66JFN3 (Lotus)*/
    881     ulmbcs_byte_t OldConverterIndex = 0;
    882 
    883    while (args->source < args->sourceLimit && !U_FAILURE(*err))
    884    {
    885       /*Fix for SPR#DJOE66JFN3 (Lotus)*/
    886       OldConverterIndex = extraInfo->localeConverterIndex;
    887 
    888       if (args->target >= args->targetLimit)
    889       {
    890          *err = U_BUFFER_OVERFLOW_ERROR;
    891          break;
    892       }
    893       uniChar = *(args->source);
    894       bytes_written = 0;
    895       pLMBCS = LMBCS;
    896 
    897       /* check cases in rough order of how common they are, for speed */
    898 
    899       /* single byte matches: strategy 1 */
    900       /*Fix for SPR#DJOE66JFN3 (Lotus)*/
    901       if((uniChar>=0x80) && (uniChar<=0xff)
    902       /*Fix for SPR#JUYA6XAERU and TSAO7GL5NK (Lotus)*/ &&(uniChar!=0xB1) &&(uniChar!=0xD7) &&(uniChar!=0xF7)
    903         &&(uniChar!=0xB0) &&(uniChar!=0xB4) &&(uniChar!=0xB6) &&(uniChar!=0xA7) &&(uniChar!=0xA8))
    904       {
    905             extraInfo->localeConverterIndex = ULMBCS_GRP_L1;
    906       }
    907       if (((uniChar > ULMBCS_C0END) && (uniChar < ULMBCS_C1START)) ||
    908           uniChar == 0 || uniChar == ULMBCS_HT || uniChar == ULMBCS_CR ||
    909           uniChar == ULMBCS_LF || uniChar == ULMBCS_123SYSTEMRANGE
    910           )
    911       {
    912          *pLMBCS++ = (ulmbcs_byte_t ) uniChar;
    913          bytes_written = 1;
    914       }
    915 
    916 
    917       if (!bytes_written)
    918       {
    919          /* Check by UNICODE range (Strategy 2) */
    920          ulmbcs_byte_t group = FindLMBCSUniRange(uniChar);
    921 
    922          if (group == ULMBCS_GRP_UNICODE)  /* (Strategy 2A) */
    923          {
    924             pLMBCS += LMBCSConvertUni(pLMBCS,uniChar);
    925 
    926             bytes_written = (int32_t)(pLMBCS - LMBCS);
    927          }
    928          else if (group == ULMBCS_GRP_CTRL)  /* (Strategy 2B) */
    929          {
    930             /* Handle control characters here */
    931             if (uniChar <= ULMBCS_C0END)
    932             {
    933                *pLMBCS++ = ULMBCS_GRP_CTRL;
    934                *pLMBCS++ = (ulmbcs_byte_t)(ULMBCS_CTRLOFFSET + uniChar);
    935             }
    936             else if (uniChar >= ULMBCS_C1START && uniChar <= ULMBCS_C1START + ULMBCS_CTRLOFFSET)
    937             {
    938                *pLMBCS++ = ULMBCS_GRP_CTRL;
    939                *pLMBCS++ = (ulmbcs_byte_t ) (uniChar & 0x00FF);
    940             }
    941             bytes_written = (int32_t)(pLMBCS - LMBCS);
    942          }
    943          else if (group < ULMBCS_GRP_UNICODE)  /* (Strategy 2C) */
    944          {
    945             /* a specific converter has been identified - use it */
    946             bytes_written = (int32_t)LMBCSConversionWorker (
    947                               extraInfo, group, pLMBCS, &uniChar,
    948                               &lastConverterIndex, groups_tried);
    949          }
    950          if (!bytes_written)    /* the ambiguous group cases  (Strategy 3) */
    951          {
    952             uprv_memset(groups_tried, 0, sizeof(groups_tried));
    953 
    954             /* check for non-default optimization group (Strategy 3A )*/
    955             if ((extraInfo->OptGroup != 1) && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->OptGroup)))
    956             {
    957                 /*zhujin: upgrade, merge #39299 here (Lotus) */
    958                 /*To make R5 compatible translation, look for exceptional group first for non-DBCS*/
    959 
    960                 if(extraInfo->localeConverterIndex < ULMBCS_DOUBLEOPTGROUP_START)
    961                 {
    962                   bytes_written = LMBCSConversionWorker (extraInfo,
    963                      ULMBCS_GRP_L1, pLMBCS, &uniChar,
    964                      &lastConverterIndex, groups_tried);
    965 
    966                   if(!bytes_written)
    967                   {
    968                      bytes_written = LMBCSConversionWorker (extraInfo,
    969                          ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar,
    970                          &lastConverterIndex, groups_tried);
    971                   }
    972                   if(!bytes_written)
    973                   {
    974                       bytes_written = LMBCSConversionWorker (extraInfo,
    975                           extraInfo->localeConverterIndex, pLMBCS, &uniChar,
    976                           &lastConverterIndex, groups_tried);
    977                   }
    978                 }
    979                 else
    980                 {
    981                      bytes_written = LMBCSConversionWorker (extraInfo,
    982                          extraInfo->localeConverterIndex, pLMBCS, &uniChar,
    983                          &lastConverterIndex, groups_tried);
    984                 }
    985             }
    986             /* check for locale optimization group (Strategy 3B) */
    987             if (!bytes_written && (extraInfo->localeConverterIndex) && (ULMBCS_AMBIGUOUS_MATCH(group, extraInfo->localeConverterIndex)))
    988             {
    989                 bytes_written = (int32_t)LMBCSConversionWorker (extraInfo,
    990                         extraInfo->localeConverterIndex, pLMBCS, &uniChar, &lastConverterIndex, groups_tried);
    991             }
    992             /* check for last optimization group used for this string (Strategy 3C) */
    993             if (!bytes_written && (lastConverterIndex) && (ULMBCS_AMBIGUOUS_MATCH(group, lastConverterIndex)))
    994             {
    995                 bytes_written = (int32_t)LMBCSConversionWorker (extraInfo,
    996                         lastConverterIndex, pLMBCS, &uniChar, &lastConverterIndex, groups_tried);
    997             }
    998             if (!bytes_written)
    999             {
   1000                /* just check every possible matching converter (Strategy 3D) */
   1001                ulmbcs_byte_t grp_start;
   1002                ulmbcs_byte_t grp_end;
   1003                ulmbcs_byte_t grp_ix;
   1004                grp_start = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS)
   1005                         ? ULMBCS_DOUBLEOPTGROUP_START
   1006                         :  ULMBCS_GRP_L1);
   1007                grp_end = (ulmbcs_byte_t)((group == ULMBCS_AMBIGUOUS_MBCS)
   1008                         ? ULMBCS_GRP_LAST
   1009                         :  ULMBCS_GRP_TH);
   1010                if(group == ULMBCS_AMBIGUOUS_ALL)
   1011                {
   1012                    grp_start = ULMBCS_GRP_L1;
   1013                    grp_end = ULMBCS_GRP_LAST;
   1014                }
   1015                for (grp_ix = grp_start;
   1016                    grp_ix <= grp_end && !bytes_written;
   1017                     grp_ix++)
   1018                {
   1019                   if (extraInfo->OptGrpConverter [grp_ix] && !groups_tried [grp_ix])
   1020                   {
   1021                      bytes_written = (int32_t)LMBCSConversionWorker (extraInfo,
   1022                        grp_ix, pLMBCS, &uniChar,
   1023                        &lastConverterIndex, groups_tried);
   1024                   }
   1025                }
   1026                 /* a final conversion fallback to the exceptions group if its likely
   1027                      to be single byte  (Strategy 3E) */
   1028                if (!bytes_written && grp_start == ULMBCS_GRP_L1)
   1029                {
   1030                   bytes_written = (int32_t)LMBCSConversionWorker (extraInfo,
   1031                      ULMBCS_GRP_EXCEPT, pLMBCS, &uniChar,
   1032                      &lastConverterIndex, groups_tried);
   1033                }
   1034             }
   1035             /* all of our other strategies failed. Fallback to Unicode. (Strategy 4)*/
   1036             if (!bytes_written)
   1037             {
   1038 
   1039                pLMBCS += LMBCSConvertUni(pLMBCS, uniChar);
   1040                bytes_written = (int32_t)(pLMBCS - LMBCS);
   1041             }
   1042          }
   1043       }
   1044 
   1045       /* we have a translation. increment source and write as much as posible to target */
   1046       args->source++;
   1047       pLMBCS = LMBCS;
   1048       while (args->target < args->targetLimit && bytes_written--)
   1049       {
   1050          *(args->target)++ = *pLMBCS++;
   1051          if (args->offsets)
   1052          {
   1053             *(args->offsets)++ = sourceIndex;
   1054          }
   1055       }
   1056       sourceIndex++;
   1057       if (bytes_written > 0)
   1058       {
   1059          /* write any bytes that didn't fit in target to the error buffer,
   1060             common code will move this to target if we get called back with
   1061             enough target room
   1062          */
   1063          uint8_t * pErrorBuffer = args->converter->charErrorBuffer;
   1064          *err = U_BUFFER_OVERFLOW_ERROR;
   1065          args->converter->charErrorBufferLength = (int8_t)bytes_written;
   1066          while (bytes_written--)
   1067          {
   1068             *pErrorBuffer++ = *pLMBCS++;
   1069          }
   1070       }
   1071       /*Fix for SPR#DJOE66JFN3 (Lotus)*/
   1072       extraInfo->localeConverterIndex = OldConverterIndex;
   1073    }
   1074 }
   1075 
   1076 
   1077 /* Now, the Unicode from LMBCS section */
   1078 
   1079 
   1080 /* A function to call when we are looking at the Unicode group byte in LMBCS */
   1081 static UChar
   1082 GetUniFromLMBCSUni(char const ** ppLMBCSin)  /* Called with LMBCS-style Unicode byte stream */
   1083 {
   1084    uint8_t  HighCh = *(*ppLMBCSin)++;  /* Big-endian Unicode in LMBCS compatibility group*/
   1085    uint8_t  LowCh  = *(*ppLMBCSin)++;
   1086 
   1087    if (HighCh == ULMBCS_UNICOMPATZERO )
   1088    {
   1089       HighCh = LowCh;
   1090       LowCh = 0; /* zero-byte in LSB special character */
   1091    }
   1092    return (UChar)((HighCh << 8) | LowCh);
   1093 }
   1094 
   1095 
   1096 
   1097 /* CHECK_SOURCE_LIMIT: Helper macro to verify that there are at least'index'
   1098    bytes left in source up to  sourceLimit.Errors appropriately if not.
   1099    If we reach the limit, then update the source pointer to there to consume
   1100    all input as required by ICU converter semantics.
   1101 */
   1102 
   1103 #define CHECK_SOURCE_LIMIT(index) \
   1104      if (args->source+index > args->sourceLimit){\
   1105          *err = U_TRUNCATED_CHAR_FOUND;\
   1106          args->source = args->sourceLimit;\
   1107          return 0xffff;}
   1108 
   1109 /* Return the Unicode representation for the current LMBCS character */
   1110 
   1111 static UChar32
   1112 _LMBCSGetNextUCharWorker(UConverterToUnicodeArgs*   args,
   1113                          UErrorCode*   err)
   1114 {
   1115     UChar32 uniChar = 0;    /* an output UNICODE char */
   1116     ulmbcs_byte_t   CurByte; /* A byte from the input stream */
   1117 
   1118     /* error check */
   1119     if (args->source >= args->sourceLimit)
   1120     {
   1121         *err = U_ILLEGAL_ARGUMENT_ERROR;
   1122         return 0xffff;
   1123     }
   1124     /* Grab first byte & save address for error recovery */
   1125     CurByte = *((ulmbcs_byte_t  *) (args->source++));
   1126 
   1127     /*
   1128     * at entry of each if clause:
   1129     * 1. 'CurByte' points at the first byte of a LMBCS character
   1130     * 2. '*source'points to the next byte of the source stream after 'CurByte'
   1131     *
   1132     * the job of each if clause is:
   1133     * 1. set '*source' to point at the beginning of next char (nop if LMBCS char is only 1 byte)
   1134     * 2. set 'uniChar' up with the right Unicode value, or set 'err' appropriately
   1135     */
   1136 
   1137     /* First lets check the simple fixed values. */
   1138 
   1139     if(((CurByte > ULMBCS_C0END) && (CurByte < ULMBCS_C1START)) /* ascii range */
   1140     ||  (CurByte == 0)
   1141     ||  CurByte == ULMBCS_HT || CurByte == ULMBCS_CR
   1142     ||  CurByte == ULMBCS_LF || CurByte == ULMBCS_123SYSTEMRANGE)
   1143     {
   1144         uniChar = CurByte;
   1145     }
   1146     else
   1147     {
   1148         UConverterDataLMBCS * extraInfo;
   1149         ulmbcs_byte_t group;
   1150         UConverterSharedData *cnv;
   1151 
   1152         if (CurByte == ULMBCS_GRP_CTRL)  /* Control character group - no opt group update */
   1153         {
   1154             ulmbcs_byte_t  C0C1byte;
   1155             CHECK_SOURCE_LIMIT(1);
   1156             C0C1byte = *(args->source)++;
   1157             uniChar = (C0C1byte < ULMBCS_C1START) ? C0C1byte - ULMBCS_CTRLOFFSET : C0C1byte;
   1158         }
   1159         else
   1160         if (CurByte == ULMBCS_GRP_UNICODE) /* Unicode compatibility group: BigEndian UTF16 */
   1161         {
   1162             CHECK_SOURCE_LIMIT(2);
   1163 
   1164             /* don't check for error indicators fffe/ffff below */
   1165             return GetUniFromLMBCSUni(&(args->source));
   1166         }
   1167         else if (CurByte <= ULMBCS_CTRLOFFSET)
   1168         {
   1169             group = CurByte;                   /* group byte is in the source */
   1170             extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo;
   1171             if (group > ULMBCS_GRP_LAST || (cnv = extraInfo->OptGrpConverter[group]) == NULL)
   1172             {
   1173                 /* this is not a valid group byte - no converter*/
   1174                 *err = U_INVALID_CHAR_FOUND;
   1175             }
   1176             else if (group >= ULMBCS_DOUBLEOPTGROUP_START)    /* double byte conversion */
   1177             {
   1178 
   1179                 CHECK_SOURCE_LIMIT(2);
   1180 
   1181                 /* check for LMBCS doubled-group-byte case */
   1182                 if (*args->source == group) {
   1183                     /* single byte */
   1184                     ++args->source;
   1185                     uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 1, FALSE);
   1186                     ++args->source;
   1187                 } else {
   1188                     /* double byte */
   1189                     uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source, 2, FALSE);
   1190                     args->source += 2;
   1191                 }
   1192             }
   1193             else {                                  /* single byte conversion */
   1194                 CHECK_SOURCE_LIMIT(1);
   1195                 CurByte = *(args->source)++;
   1196 
   1197                 if (CurByte >= ULMBCS_C1START)
   1198                 {
   1199                     uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte);
   1200                 }
   1201                 else
   1202                 {
   1203                     /* The non-optimizable oddballs where there is an explicit byte
   1204                     * AND the second byte is not in the upper ascii range
   1205                     */
   1206                     char bytes[2];
   1207 
   1208                     extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo;
   1209                     cnv = extraInfo->OptGrpConverter [ULMBCS_GRP_EXCEPT];
   1210 
   1211                     /* Lookup value must include opt group */
   1212                     bytes[0] = group;
   1213                     bytes[1] = CurByte;
   1214                     uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, bytes, 2, FALSE);
   1215                 }
   1216             }
   1217         }
   1218         else if (CurByte >= ULMBCS_C1START) /* group byte is implicit */
   1219         {
   1220             extraInfo = (UConverterDataLMBCS *) args->converter->extraInfo;
   1221             group = extraInfo->OptGroup;
   1222             cnv = extraInfo->OptGrpConverter[group];
   1223             if (group >= ULMBCS_DOUBLEOPTGROUP_START)    /* double byte conversion */
   1224             {
   1225                 if (!ucnv_MBCSIsLeadByte(cnv, CurByte))
   1226                 {
   1227                     CHECK_SOURCE_LIMIT(0);
   1228 
   1229                     /* let the MBCS conversion consume CurByte again */
   1230                     uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 1, FALSE);
   1231                 }
   1232                 else
   1233                 {
   1234                     CHECK_SOURCE_LIMIT(1);
   1235                     /* let the MBCS conversion consume CurByte again */
   1236                     uniChar = ucnv_MBCSSimpleGetNextUChar(cnv, args->source - 1, 2, FALSE);
   1237                     ++args->source;
   1238                 }
   1239             }
   1240             else                                   /* single byte conversion */
   1241             {
   1242                 uniChar = _MBCS_SINGLE_SIMPLE_GET_NEXT_BMP(cnv, CurByte);
   1243             }
   1244         }
   1245     }
   1246     return uniChar;
   1247 }
   1248 
   1249 
   1250 /* The exported function that converts lmbcs to one or more
   1251    UChars - currently UTF-16
   1252 */
   1253 static void
   1254 _LMBCSToUnicodeWithOffsets(UConverterToUnicodeArgs*    args,
   1255                      UErrorCode*    err)
   1256 {
   1257    char LMBCS [ULMBCS_CHARSIZE_MAX];
   1258    UChar uniChar;    /* one output UNICODE char */
   1259    const char * saveSource; /* beginning of current code point */
   1260    const char * pStartLMBCS = args->source;  /* beginning of whole string */
   1261    const char * errSource = NULL; /* pointer to actual input in case an error occurs */
   1262    int8_t savebytes = 0;
   1263 
   1264    /* Process from source to limit, or until error */
   1265    while (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit > args->target)
   1266    {
   1267       saveSource = args->source; /* beginning of current code point */
   1268 
   1269       if (args->converter->toULength) /* reassemble char from previous call */
   1270       {
   1271         const char *saveSourceLimit;
   1272         size_t size_old = args->converter->toULength;
   1273 
   1274          /* limit from source is either remainder of temp buffer, or user limit on source */
   1275         size_t size_new_maybe_1 = sizeof(LMBCS) - size_old;
   1276         size_t size_new_maybe_2 = args->sourceLimit - args->source;
   1277         size_t size_new = (size_new_maybe_1 < size_new_maybe_2) ? size_new_maybe_1 : size_new_maybe_2;
   1278 
   1279 
   1280         uprv_memcpy(LMBCS, args->converter->toUBytes, size_old);
   1281         uprv_memcpy(LMBCS + size_old, args->source, size_new);
   1282         saveSourceLimit = args->sourceLimit;
   1283         args->source = errSource = LMBCS;
   1284         args->sourceLimit = LMBCS+size_old+size_new;
   1285         savebytes = (int8_t)(size_old+size_new);
   1286         uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err);
   1287         args->source = saveSource + ((args->source - LMBCS) - size_old);
   1288         args->sourceLimit = saveSourceLimit;
   1289 
   1290         if (*err == U_TRUNCATED_CHAR_FOUND)
   1291         {
   1292             /* evil special case: source buffers so small a char spans more than 2 buffers */
   1293             args->converter->toULength = savebytes;
   1294             uprv_memcpy(args->converter->toUBytes, LMBCS, savebytes);
   1295             args->source = args->sourceLimit;
   1296             *err = U_ZERO_ERROR;
   1297             return;
   1298          }
   1299          else
   1300          {
   1301             /* clear the partial-char marker */
   1302             args->converter->toULength = 0;
   1303          }
   1304       }
   1305       else
   1306       {
   1307          errSource = saveSource;
   1308          uniChar = (UChar) _LMBCSGetNextUCharWorker(args, err);
   1309          savebytes = (int8_t)(args->source - saveSource);
   1310       }
   1311       if (U_SUCCESS(*err))
   1312       {
   1313          if (uniChar < 0xfffe)
   1314          {
   1315             *(args->target)++ = uniChar;
   1316             if(args->offsets)
   1317             {
   1318                *(args->offsets)++ = (int32_t)(saveSource - pStartLMBCS);
   1319             }
   1320          }
   1321          else if (uniChar == 0xfffe)
   1322          {
   1323             *err = U_INVALID_CHAR_FOUND;
   1324          }
   1325          else /* if (uniChar == 0xffff) */
   1326          {
   1327             *err = U_ILLEGAL_CHAR_FOUND;
   1328          }
   1329       }
   1330    }
   1331    /* if target ran out before source, return U_BUFFER_OVERFLOW_ERROR */
   1332    if (U_SUCCESS(*err) && args->sourceLimit > args->source && args->targetLimit <= args->target)
   1333    {
   1334       *err = U_BUFFER_OVERFLOW_ERROR;
   1335    }
   1336    else if (U_FAILURE(*err))
   1337    {
   1338       /* If character incomplete or unmappable/illegal, store it in toUBytes[] */
   1339       args->converter->toULength = savebytes;
   1340       if (savebytes > 0) {
   1341          uprv_memcpy(args->converter->toUBytes, errSource, savebytes);
   1342       }
   1343       if (*err == U_TRUNCATED_CHAR_FOUND) {
   1344          *err = U_ZERO_ERROR;
   1345       }
   1346    }
   1347 }
   1348 
   1349 /* And now, the macroized declarations of data & functions: */
   1350 DEFINE_LMBCS_OPEN(1)
   1351 DEFINE_LMBCS_OPEN(2)
   1352 DEFINE_LMBCS_OPEN(3)
   1353 DEFINE_LMBCS_OPEN(4)
   1354 DEFINE_LMBCS_OPEN(5)
   1355 DEFINE_LMBCS_OPEN(6)
   1356 DEFINE_LMBCS_OPEN(8)
   1357 DEFINE_LMBCS_OPEN(11)
   1358 DEFINE_LMBCS_OPEN(16)
   1359 DEFINE_LMBCS_OPEN(17)
   1360 DEFINE_LMBCS_OPEN(18)
   1361 DEFINE_LMBCS_OPEN(19)
   1362 
   1363 
   1364 DECLARE_LMBCS_DATA(1)
   1365 DECLARE_LMBCS_DATA(2)
   1366 DECLARE_LMBCS_DATA(3)
   1367 DECLARE_LMBCS_DATA(4)
   1368 DECLARE_LMBCS_DATA(5)
   1369 DECLARE_LMBCS_DATA(6)
   1370 DECLARE_LMBCS_DATA(8)
   1371 DECLARE_LMBCS_DATA(11)
   1372 DECLARE_LMBCS_DATA(16)
   1373 DECLARE_LMBCS_DATA(17)
   1374 DECLARE_LMBCS_DATA(18)
   1375 DECLARE_LMBCS_DATA(19)
   1376 
   1377 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */
   1378