1 //******************************************************************************* 2 // 3 // Copyright (C) 2003-2010, International Business Machines 4 // Corporation and others. All Rights Reserved. 5 // 6 // file name: conversion.txt 7 // encoding: US-ASCII 8 // tab size: 8 (not used) 9 // indentation:4 10 // 11 // created on: 2003jul15 12 // created by: Markus W. Scherer 13 // 14 // ICU resource bundle source file with test data for data-driven conversion tests. 15 // 16 //******************************************************************************* 17 18 conversion:table(nofallback) { 19 Info { 20 Description { "Test data for conversion" } 21 LongDescription { 22 "Test data for data-driven conversion tests in icu/source/test/intltest/convtest.cpp\n" 23 "Run intltest conversion\n" 24 25 "Charset names starting with '*' are for testdata names.\n" 26 27 "ICU callbacks are specified as strings with pairs of characters, each optional.\n" 28 "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n" 29 "Callback option - a letter is passed in directly as const char * see ucnv_err.h\n" 30 "Empty string: Sub callback with NULL option\n" 31 32 "In order to specify a charset substitution character (for ucnv_setSubstChars()),\n" 33 "add a NUL (U+0000) to the callback string followed by the subchar bytes as Latin-1\n" 34 "characters. For example, for a Sub callback with no option and a subchar of FC FC,\n" 35 "use the string \"?\x00\xFC\xFC\"\n" 36 37 "In order to specify a substitution string (for ucnv_setSubstString()),\n" 38 "add an '=' to the callback string followed by the substitution string.\n" 39 "For example, for a Sub callback with no option and a substitution string\n" 40 "of \"ab\", use the string \"?=ab\"\n" 41 42 "fallbacks: per-direction boolean, currently only for fromUnicode; see Jitterbug 2401\n" 43 44 "errorCode: (empty)==zero | invalid | illegal | truncated | illesc | unsuppesc\n" 45 } 46 } 47 TestData { 48 toUnicode { 49 Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } 50 Cases { 51 // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters 52 // For details about these encodings see convrtrs.txt. 53 // Standard UTF-16 54 { "UTF-16", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 55 { "UTF-16", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 56 { "UTF-16", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 57 // Java "Unicode" requires a BOM 58 { "UTF-16,version=1", :bin{ 00610062 }, "\\x00\\x61b", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 59 { "UTF-16,version=1", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 60 { "UTF-16,version=1", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 61 // Standard UTF-16BE 62 { "UTF-16BE", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 63 { "UTF-16BE", :bin{ feff0061 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 64 { "UTF-16BE", :bin{ fffe0061 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 65 // Java "UnicodeBig" requires a BE BOM or no BOM; it consumes the BE BOM 66 { "UTF-16BE,version=1", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 67 { "UnicodeBig", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 68 { "UnicodeBig", :bin{ fffe0061 }, "\\xFF\\xFEa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 69 // Standard UTF-16LE 70 { "UTF-16LE", :bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 71 { "UTF-16LE", :bin{ fffe6100 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 72 { "UTF-16LE", :bin{ feff6100 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 73 // Java "UnicodeLittle" requires an LE BOM or no BOM; it consumes the LE BOM 74 { "UTF-16LE,version=1", :bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 75 { "UnicodeLittle", :bin{ fffe6100 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 76 { "x-UTF-16LE-BOM", :bin{ feff6100 }, "\\xFE\\xFFa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 77 78 // Test ticket 5691: consistent illegal sequences 79 // The following test cases are for illegal character byte sequences. 80 // 81 // Unfortunately, we cannot use the Shift-JIS examples from the ticket 82 // comments because our Shift-JIS table is Windows-compatible and 83 // therefore has no illegal single bytes. Same for GBK. 84 // Instead, we use the stricter GB 18030 also for 2-byte examples. 85 // The byte sequences are generally slightly different from the ticket 86 // comment, simply using assigned characters rather than just 87 // theoretically valid sequences. 88 { 89 "gb18030", 90 :bin{ 618140813c81ff7a }, 91 "a\u4e02\\x81<\\x81\\xFFz", 92 :intvector{ 0,1,3,3,3,3,4,5,5,5,5,5,5,5,5,7 }, 93 :int{1}, :int{0}, "", "&C", :bin{""} 94 } 95 { 96 "EUC-JP", 97 :bin{ 618fb0a98fb03c8f3cb0a97a }, 98 "a\u4e28\\x8F\\xB0<\\x8F<\u9022z", 99 :intvector{ 0,1,4,4,4,4,5,5,5,5,6,7,7,7,7,8,9,11 }, 100 :int{1}, :int{0}, "", "&C", :bin{""} 101 } 102 { 103 "gb18030", 104 :bin{ 618130fc318130fc8181303c3e813cfc817a }, 105 "a\u05ed\\x810\u9f07\\x810<>\\x81<\u9f07z", 106 :intvector{ 0,1,5,5,5,5,6,7,9,9,9,9,10,11,12,13,13,13,13,14,15,17 }, 107 :int{1}, :int{0}, "", "&C", :bin{""} 108 } 109 { 110 "UTF-8", 111 :bin{ 61f1808182f180813cf18081fff180ff3cf1ff3c3e7a }, 112 "a\U00040042\\xF1\\x80\\x81<\\xF1\\x80\\x81\\xFF\\xF1\\x80\\xFF<\\xF1\\xFF<>z", 113 :intvector{ 0,1,1,5,5,5,5,5,5,5,5,5,5,5,5,8,9,9,9,9,9,9,9,9,9,9,9,9,12,12,12,12,13,13,13,13,13,13,13,13,15,15,15,15,16,17,17,17,17,18,18,18,18,19,20,21 }, 114 :int{1}, :int{0}, "", "&C", :bin{""} 115 } 116 { 117 "ISO-2022-JP", 118 :bin{ 1b24424141af4142affe41431b2842 }, 119 "\u758f\\xAF\u758e\\xAF\\xFE\u790e", 120 :intvector{ 3,5,5,5,5,6,8,8,8,8,8,8,8,8,10 }, 121 :int{1}, :int{0}, "", "&C", :bin{""} 122 } 123 { 124 "ibm-25546", 125 :bin{ 411b242943420e4141af4142affe41430f5a }, 126 "AB\uc88b\\xAF\uc88c\\xAF\\xFE\uc88dZ", 127 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 128 :int{1}, :int{0}, "", "&C", :bin{""} 129 } 130 { 131 "ISO-2022-KR", 132 :bin{ 411b242943420e4141af4142affe41430f5a }, 133 "AB\uc88b\\xAF\uc88c\\xAF\\xFE\uc88dZ", 134 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 135 :int{1}, :int{0}, "", "&C", :bin{""} 136 } 137 { 138 "ISO-2022-CN", 139 :bin{ 411b242941420e4141af4142affe41430f5a }, 140 "AB\u4eae\\xAF\u8c05\\xAF\\xFE\u64a9Z", 141 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 142 :int{1}, :int{0}, "", "&C", :bin{""} 143 } 144 { 145 "HZ", 146 :bin{ 417e7b4141af4142affe41437e7d5a }, 147 "A\u4eae\\xAF\u8c05\\xAF\\xFE\u64a9Z", 148 :intvector{ 0,3,5,5,5,5,6,8,8,8,8,8,8,8,8,10,14 }, 149 :int{1}, :int{0}, "", "&C", :bin{""} 150 } 151 // Test ticket 5691: consistent illegal sequences 152 // The following test cases are for illegal escape/designator/shift sequences. 153 // 154 // ISO-2022-JP and -CN with illegal escape sequences. 155 { 156 "ISO-2022-JP", 157 :bin{ 611b24201b244241411b283f1b28427a }, 158 "a\\x1B$ \u758f\\x1B\u2538z", 159 :intvector{ 0,1,1,1,1,2,3,7,9,9,9,9,10,15 }, 160 :int{1}, :int{0}, "", "&C", :bin{""} 161 } 162 { 163 "ISO-2022-CN", 164 :bin{ 611b2429201b2429410e41410f7a }, 165 "a\\x1B$) \u4eaez", 166 :intvector{ 0,1,1,1,1,2,3,4,10,13 }, 167 :int{1}, :int{0}, "", "&C", :bin{""} 168 } 169 // Test ticket 5691: ISO-2022-JP-2 with illegal single-shift SS2 and SS3 sequences. 170 // The first ESC N comes before its designator sequence, the last sequence is ESC+space. 171 { 172 "ISO-2022-JP-2", 173 :bin{ 4e1b4e4e1b2e414e1b4e4e4e1b204e }, 174 "N\\x1BNNN\xceN\\x1B N", 175 :intvector{ 0,1,1,1,1,2,3,7,10,11,12,12,12,12,13,14 }, 176 :int{1}, :int{0}, "", "&C", :bin{""} 177 } 178 { 179 "ISO-2022-CN-EXT", 180 :bin{ 4e1b4e4e1b242a484e1b4e4e4e4e1b204e }, 181 "N\\x1BNNN\u8f0eN\\x1B N", 182 :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, 183 :int{1}, :int{0}, "", "&C", :bin{""} 184 } 185 /* 186 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 187 { 188 "ISO-2022-CN-EXT", 189 :bin{ 4f1b4f4f1b242b494f1b4f4f4f4f1b204f }, 190 "O\\x1BOOO\u492bO\\x1B O", 191 :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, 192 :int{1}, :int{0}, "", "&C", :bin{""} 193 } 194 */ 195 // Test ticket 5691: HZ with illegal tilde sequences. 196 { 197 "HZ", 198 :bin{ 417e20427e21437e80447e7b41417e207e41427e7f41437e7d5a }, 199 "A\\x7E B\\x7E!C\\x7E\\x80D\u4eae\\x7E\\x20\\x7E\u8c05\\x7E\\x7F\u64a9Z", 200 :intvector{ 0,1,1,1,1,2,3,4,4,4,4,5,6,7,7,7,7,7,7,7,7,9, // SBCS 201 12,14,14,14,14,14,14,14,14,16,16,16,16,17,19,19,19,19,19,19,19,19,21, // DBCS 202 25 }, // SBCS 203 :int{1}, :int{0}, "", "&C", :bin{""} 204 } 205 // Test ticket 5691: Example from Peter Edberg. 206 { 207 "ISO-2022-JP", 208 :bin{ 1b244230212f7e742630801b284a621b2458631b2842648061 }, 209 "\u4e9c\ufffd\u7199\ufffdb\ufffd$Xcd\ufffda", 210 :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 }, 211 :int{1}, :int{0}, "", "?", :bin{""} 212 } 213 // Test bug 6071 (2:1 Unicode:charset SBCS mapping). 214 { 215 "*test1bmp", 216 :bin{ 050008 }, 217 "e@uv", 218 :intvector{ 0,1,2,2 }, 219 :int{1}, :int{1}, "", "?", :bin{""} 220 } 221 // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e 222 { 223 "HZ", 224 :bin{ 7e7b21212120217e217f772100007e217e7e7d207e7e807e0a2b }, 225 "\u3000\ufffd\u3013\ufffd\u9ccc\ufffd\ufffd\u3013 ~\ufffd+", 226 :intvector{ 2,4,6,8,10,12,14,15,19,20,22,25 }, 227 :int{1}, :int{1}, "", "?", :bin{""} 228 } 229 // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and 230 // using the Shift-JIS table for JIS X 0208 (ticket #5797) 231 { 232 "ISO-2022-JP", 233 :bin{ 1b284a7d7e801b2442306c20217f7e21202160217f22202225227f5f211b2842 }, 234 "}\u203e\ufffd\u4e00\ufffd\ufffd\ufffd\ufffd\xf7\ufffd\ufffd\u25b2\ufffd\u6f3e", 235 :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 }, 236 :int{1}, :int{1}, "", "?", :bin{""} 237 } 238 // improve coverage of ISO-2022-JP converter by simulating erroneous input 239 { 240 "ISO-2022-JP-2", 241 :bin{ 0f0ed11b2e41461b244141411b4e411b2e4147451b4ed31b2e4641411b4ed2 }, 242 "\u0046\u4eae\u00c1\u6865\u4eae", 243 :intvector{ 6, 10, 14, 18, 26 }, 244 :int{1}, :int{0}, "", "0", :bin{""} 245 } 246 // improve coverage of JIS7 converter by simulating incomplete shifted input 247 { 248 "JIS7", 249 :bin{ 0e11 }, 250 "", 251 :intvector{}, 252 :int{1}, :int{0}, "", "0", :bin{""} 253 } 254 // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets() 255 { 256 "ISO-8859-3", 257 :bin{ 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627 }, 258 "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\x22#$%&'", 259 :intvector{ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 }, 260 :int{1}, :int{0}, "", "?", :bin{""} 261 } 262 // test that ISO-2022-JP encodes ASCII as itself 263 { 264 "ISO-2022-JP", 265 :bin{ 3f4041424344454647 }, 266 "?@ABCDEFG", 267 :intvector{ 0,1,2,3,4,5,6,7,8 }, 268 :int{1}, :int{1}, "", "?", :bin{""} 269 } 270 // test that ISO-2022-CN encodes ASCII as itself 271 { 272 "ISO-2022-CN", 273 :bin{ 3f4041424344454647 }, 274 "?@ABCDEFG", 275 :intvector{ 0,1,2,3,4,5,6,7,8 }, 276 :int{1}, :int{1}, "", "?", :bin{""} 277 } 278 279 // ISO-2022-KR 280 281 // truncated, partial escape sequence 282 { 283 "ibm-25546", 284 :bin{ 1b }, "", :intvector{}, 285 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 286 } 287 { 288 "ibm-25546", 289 :bin{ 1b24 }, "", :intvector{}, 290 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 291 } 292 { 293 "ibm-25546", 294 :bin{ 1b2429 }, "", :intvector{}, 295 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 296 } 297 // complete escape sequence but nothing else 298 { 299 "ibm-25546", 300 :bin{ 1b242943 }, "", :intvector{}, 301 :int{1}, :int{1}, "", ".", :bin{""} 302 } 303 { 304 "ibm-25546", 305 :bin{ 1b2429430e }, "", :intvector{}, 306 :int{1}, :int{1}, "", ".", :bin{""} 307 } 308 // escape plus ASCII character 309 { 310 "ibm-25546", 311 :bin{ 1b24294341 }, "A", :intvector{ 4 }, 312 :int{1}, :int{1}, "", ".", :bin{""} 313 } 314 // escape plus incomplete DBCS character 315 { 316 "ibm-25546", 317 :bin{ 1b2429430e41 }, "", :intvector{}, 318 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 319 } 320 // all complete with DBCS character 321 { 322 "ibm-25546", 323 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 }, 324 :int{1}, :int{1}, "", ".", :bin{""} 325 } 326 // more complicated example 327 { 328 "ibm-25546", 329 :bin{ 411b242943420e4141affe0f43 }, 330 "AB\uc88b%XAF%XFEC", 331 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 }, 332 :int{1}, :int{1}, "", "&", :bin{""} 333 } 334 335 // truncated, partial escape sequence 336 { 337 "ISO-2022-KR", 338 :bin{ 1b }, "", :intvector{}, 339 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 340 } 341 { 342 "ISO-2022-KR", 343 :bin{ 1b24 }, "", :intvector{}, 344 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 345 } 346 { 347 "ISO-2022-KR", 348 :bin{ 1b2429 }, "", :intvector{}, 349 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 350 } 351 // complete escape sequence but nothing else 352 { 353 "ISO-2022-KR", 354 :bin{ 1b242943 }, "", :intvector{}, 355 :int{1}, :int{1}, "", ".", :bin{""} 356 } 357 { 358 "ISO-2022-KR", 359 :bin{ 1b2429430e }, "", :intvector{}, 360 :int{1}, :int{1}, "", ".", :bin{""} 361 } 362 // escape plus ASCII character 363 { 364 "ISO-2022-KR", 365 :bin{ 1b24294341 }, "A", :intvector{ 4 }, 366 :int{1}, :int{1}, "", ".", :bin{""} 367 } 368 // escape plus incomplete DBCS character 369 { 370 "ISO-2022-KR", 371 :bin{ 1b2429430e41 }, "", :intvector{}, 372 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 373 } 374 // all complete with DBCS character 375 { 376 "ISO-2022-KR", 377 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 }, 378 :int{1}, :int{1}, "", ".", :bin{""} 379 } 380 // more complicated example 381 { 382 "ISO-2022-KR", 383 :bin{ 411b242943420e4141affe0f43 }, 384 "AB\uc88b%XAF%XFEC", 385 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 }, 386 :int{1}, :int{1}, "", "&", :bin{""} 387 } 388 // empty segment (using substitution and stop) 389 { 390 "ISO-2022-KR", 391 :bin{ 1b242943610e0f620d0a }, 392 "a\uFFFDb\u000D\u000A", 393 :intvector{ 4, 6, 7, 8, 9 }, 394 :int{1}, :int{1}, "", "?", :bin{""} 395 } 396 { 397 "ISO-2022-KR", 398 :bin{ 1b242943610e0f620d0a }, 399 "a", 400 :intvector{ 4 }, 401 :int{1}, :int{1}, "illesc", ".", :bin{"0f"} 402 } 403 404 // ISO-2022-JP 405 406 // truncated, partial escape sequence 407 { 408 "ISO-2022-JP", 409 :bin{ 1b }, "", :intvector{}, 410 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 411 } 412 { 413 "ISO-2022-JP-2", 414 :bin{ 1b24 }, "", :intvector{}, 415 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 416 } 417 // complete escape sequence but nothing else 418 { 419 "ISO-2022-JP-2", 420 :bin{ 1b2442 }, "", :intvector{}, 421 :int{1}, :int{1}, "", ".", :bin{""} 422 } 423 // escape plus incomplete DBCS character 424 { 425 "ISO-2022-JP-2", 426 :bin{ 1b244241 }, "", :intvector{}, 427 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 428 } 429 // all complete with DBCS character 430 { 431 "ISO-2022-JP-2", 432 :bin{ 1b24424141 }, "\u758f", :intvector{ 3 }, 433 :int{1}, :int{1}, "", ".", :bin{""} 434 } 435 // test the G2 designator & SS2 shift 436 { 437 "ISO-2022-JP-2", 438 :bin{ 431b2e46461b244241411b4e4e353f }, "CF\u758f\u039e\u7591", :intvector{ 0, 4, 8, 12, 13 }, 439 :int{1}, :int{1}, "", ".", :bin{""} 440 } 441 // JIS7 with Katakana 442 { 443 "JIS7", 444 :bin{ 41420e41420f4142 }, "AB\uff81\uff82AB", :intvector{ 0, 1, 3, 4, 6, 7 }, 445 :int{1}, :int{1}, "", ".", :bin{""} 446 } 447 // JIS8 with Katakana 448 { 449 "JIS8", 450 :bin{ 41c15c1b284a5cc242 }, "A\uff81\\\xa5\uff82B", :intvector{ 0, 1, 2, 6, 7, 8 }, 451 :int{1}, :int{1}, "", ".", :bin{""} 452 } 453 // empty segment (using substitution and stop) 454 { 455 "ISO-2022-JP", 456 :bin{ 61621b24421b284263640d0a }, 457 "ab\uFFFDcd\u000D\u000A", 458 :intvector{ 0, 1, 5, 8, 9, 10, 11 }, 459 :int{1}, :int{1}, "", "?", :bin{""} 460 } 461 { 462 "ISO-2022-JP", 463 :bin{ 61621b24421b284263640d0a }, 464 "ab", 465 :intvector{ 0, 1 }, 466 :int{1}, :int{1}, "illesc", ".", :bin{"1b2842"} 467 } 468 469 // ISO-2022-CN 470 471 // truncated, partial escape sequence 472 { 473 "ISO_2022,locale=zh,version=1", 474 :bin{ 1b }, "", :intvector{}, 475 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 476 } 477 { 478 "ISO_2022,locale=zh,version=1", 479 :bin{ 1b24 }, "", :intvector{}, 480 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 481 } 482 { 483 "ISO_2022,locale=zh,version=1", 484 :bin{ 1b2429 }, "", :intvector{}, 485 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 486 } 487 // complete escape sequence but nothing else 488 { 489 "ISO_2022,locale=zh,version=1", 490 :bin{ 1b242941 }, "", :intvector{}, 491 :int{1}, :int{1}, "", ".", :bin{""} 492 } 493 { 494 "ISO_2022,locale=zh,version=1", 495 :bin{ 1b2429410e }, "", :intvector{}, 496 :int{1}, :int{1}, "", ".", :bin{""} 497 } 498 // escape plus ASCII character 499 { 500 "ISO_2022,locale=zh,version=1", 501 :bin{ 1b24294141 }, "\x41", :intvector{ 4 }, 502 :int{1}, :int{1}, "", ".", :bin{""} 503 } 504 // escape plus incomplete DBCS character 505 { 506 "ISO_2022,locale=zh,version=1", 507 :bin{ 1b2429410e41 }, "", :intvector{}, 508 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 509 } 510 // all complete with DBCS character 511 { 512 "ISO_2022,locale=zh,version=1", 513 :bin{ 1b2429410e4141 }, "\u4eae", :intvector{ 5 }, 514 :int{1}, :int{1}, "", ".", :bin{""} 515 } 516 /* 517 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 518 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points 519 { 520 "ISO-2022-CN-EXT", 521 :bin{ 1b2429411b242a480e41411b2429457e7c1b4e70341b242b4d1b2429477c341b4f664c2421 }, 522 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30", 523 :intvector{ 9, 15, 19, 29, 33, 33, 35 }, 524 :int{1}, :int{1}, "", ".", :bin{""} 525 } 526 */ 527 528 // illegal and unsupported escape sequences 529 // SS2 without designator: illegal 530 { 531 "ISO-2022-CN-EXT", 532 :bin{ 411b4e2121 }, "\x41", :intvector{ 0 }, 533 :int{1}, :int{1}, "illesc", ".", :bin{ 1b } 534 } 535 // G3 designator: recognized, but not supported for -CN (only for -CN-EXT) 536 { 537 "ISO-2022-CN", 538 :bin{ 411b242b491b4f2121 }, "\x41", :intvector{ 0 }, 539 :int{1}, :int{1}, "unsuppesc", ".", :bin{ 1b242b49 } 540 } 541 // empty segment 1 (using substitution and stop) 542 { 543 "ISO-2022-CN", 544 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a }, 545 "ab\uFFFD\u994Cc\u000D\u000A", 546 :intvector{ 0, 5, 7, 14, 16, 17, 18 }, 547 :int{1}, :int{1}, "", "?", :bin{""} 548 } 549 { 550 "ISO-2022-CN", 551 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a }, 552 "ab", 553 :intvector{ 0, 5 }, 554 :int{1}, :int{1}, "illesc", ".", :bin{"0f"} 555 } 556 // empty segment 2 (using substitution and stop) 557 { 558 "ISO-2022-CN", 559 :bin{ 611b242941620e1b24294768640f630d0a }, 560 "ab\uFFFD\u5F70c\u000D\u000A", 561 :intvector{ 0, 5, 7, 11, 14, 15, 16 }, 562 :int{1}, :int{1}, "", "?", :bin{""} 563 } 564 { 565 "ISO-2022-CN", 566 :bin{ 611b242941620e1b24294768640f630d0a }, 567 "ab", 568 :intvector{ 0, 5 }, 569 :int{1}, :int{1}, "illesc", ".", :bin{"1b242947"} 570 } 571 572 // ISO-2022 SBCS 573 // [U_ENABLE_GENERIC_ISO_2022] 574 // The _generic_ ISO-2022 converter is disabled starting 2003-dec-03 (ICU 2.8). 575 // For details see the icu mailing list from 2003-dec-01 and the ucnv2022.c file. 576 // Language-specific variants of ISO-2022 continue to be available as listed below. 577 //{ 578 // "ISO_2022", 579 // :bin{ 0008090a0d1a1c1f203f415c7d7e7f }, 580 // "\x00\x08\t\n\r\x1a\x1c\x1f ?A\\}~\x7f", 581 // :intvector{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, 582 // :int{1}, :int{1}, "", ".", :bin{""} 583 //} 584 585 // HZ-GB-2312 586 587 // empty segment 1 (using substitution and stop) 588 { 589 "HZ-GB-2312", 590 :bin{ 61627e7b7e7d6364 }, 591 "ab\uFFFDcd", 592 :intvector{ 0, 1, 4, 6, 7 }, 593 :int{1}, :int{1}, "", "?", :bin{""} 594 } 595 { 596 "HZ-GB-2312", 597 :bin{ 61627e7b7e7d63640d0a }, 598 "ab", 599 :intvector{ 0, 1 }, 600 :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"} 601 } 602 // empty segment 2 & legal redundant switches (using substitution and stop) 603 { 604 "HZ-GB-2312", 605 :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, 606 "ab\u4E0D\u7A7A\uFFFD\u4E00cdef\uFFFD", 607 :intvector{ 0, 1, 4, 6, 10, 12, 16, 17, 20, 21, 24 }, 608 :int{1}, :int{1}, "", "?", :bin{""} 609 } 610 { 611 "HZ-GB-2312", 612 :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, 613 "ab\u4E0D\u7A7A", 614 :intvector{ 0, 1, 4, 6 }, 615 :int{1}, :int{1}, "illesc", ".", :bin{"7e7b"} 616 } 617 618 // DBCS-only extensions 619 { 620 "ibm-970", 621 :bin{ 617eece9b2eb }, 622 "\x61\x7e\u4e00\ub000", 623 :intvector{ 0, 1, 2, 4 }, 624 :int{1}, :int{1}, "", "?", :bin{""} 625 } 626 627 { 628 "ibm-971", 629 :bin{ 617eece9b2eb }, 630 "\ufffd\u4e00\ub000", 631 :intvector{ 0, 2, 4 }, 632 :int{1}, :int{1}, "", "?", :bin{""} 633 } 634 635 { 636 "ibm-16684", 637 :bin{ 430e4395ecc1404042e1 }, 638 "\ufffd\u30C8\u30C8\u309A\u3000\u20ac", 639 :intvector{ 0, 2, 4, 4, 6, 8 }, 640 :int{1}, :int{0}, "", "?", :bin{""} 641 } 642 643 { 644 "ibm-1399", 645 :bin{ 430e4395ecc140400fe1 }, 646 "\uff62\u30C8\u30C8\u309A\u3000\u20ac", 647 :intvector{ 0, 2, 4, 4, 6, 9 }, 648 :int{1}, :int{0}, "", "?", :bin{""} 649 } 650 651 // extensions 652 { 653 "ibm-1390", 654 :bin{ 430e4395ecc1 }, 655 "\uff63\u30C8\u30C8\u309A", 656 :intvector{ 0, 2, 4, 4 }, 657 :int{1}, :int{0}, "", "?", :bin{""} 658 } 659 660 { 661 "ibm-16684", 662 :bin{ ececec8bec8cec8d4386ecb5ecb6ecb7 }, 663 "\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a", 664 :intvector{ 0, 2, 4, 6, 8, 10, 10, 12, 12, 14, 14 }, 665 :int{1}, :int{0}, "", "?", :bin{""} 666 } 667 668 { 669 "ibm-1390", 670 :bin{ 43860eececec8bec8cec8d4386ecb5ecb6ecb7ecc10fec }, 671 "\uff63\uff76\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a\u30C8\u309A\x1a", 672 :intvector{ 0, 1, 3, 5, 7, 9, 11, 13, 13, 15, 15, 17, 17, 19, 19, 22 }, 673 :int{1}, :int{0}, "", "?", :bin{""} 674 } 675 676 { 677 "*test3", 678 :bin{ 00050601020b0701020a01020c }, 679 "\u20ac\x05\x06\x0b\U00101234\U00023456\ufffd", 680 :intvector{ 0, 1, 2, 3, 6, 6, 7, 7, 10 }, 681 :int{1}, :int{0}, "", "?", :bin{""} 682 } 683 684 // normal conversions 685 { 686 "UTF-16LE", 687 :bin{ 310000d801dc00d902dc320000d8330001dc3400 }, 688 "1\U00010001\U000500022\ufffd3\ufffd4", 689 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, 690 :int{1}, :int{0}, "", "?", :bin{""} 691 } 692 { "UTF-16LE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } 693 { "UTF-16LE", :bin{ 00d800 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00d800 } } 694 695 { 696 "UTF-16BE", 697 :bin{ 0031d800dc01d900dc020032d8000033dc010034 }, 698 "1\U00010001\U000500022\ufffd3\ufffd4", 699 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, 700 :int{1}, :int{0}, "", "?", :bin{""} 701 } 702 { "UTF-16BE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } 703 { "UTF-16BE", :bin{ d800dc }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ d800dc } } 704 705 // e4b8 is a partial sequence 706 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c", :intvector{ 0, 1 }, :int{1}, :int{0}, "truncated", ".", :bin{ e4b8 } } 707 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c\ufffd", :intvector{ 0, 1, 4 }, :int{1}, :int{0}, "", "?", :bin{""} } 708 709 // LMBCS with escape callback (1292a0 is unassigned) 710 { 711 "LMBCS", 712 :bin{ 12c9501292a01292a1 }, 713 "\u4e2e%X12%X92%XA0\ue5c4", 714 :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 }, 715 :int{1}, :int{0}, "", "&", :bin{""} 716 } 717 { 718 "LMBCS", 719 :bin{ 61012981a00f270f91140a7414f60214d84d14dc561088a0 }, 720 "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", 721 :intvector{ 0, 1, 3, 4, 5, 7, 9, 12, 15, 18, 21 }, 722 :int{1}, :int{0}, "", ".", :bin{""} 723 } 724 725 // IMAP-mailbox-name with SUB 726 // a<DEL> a&AB~ a&AB\x0c a&AB- a&AB. a&. 727 { 728 "IMAP-mailbox-name", 729 :bin{ 617f612641427e612641420c612641422d612641422e61262e }, 730 "a\ufffda\ufffda\ufffda\ufffda\ufffda\ufffd", 731 :intvector{ 0, 1, 2, 4, 7, 9, 12, 14, 17, 19, 22, 23 }, 732 :int{1}, :int{0}, "", "?", :bin{""} 733 } 734 735 // using testdata_test1.cnv 736 { "*test1", :bin{ 000506070809 }, "\u20ac\x05\x06\U00101234\ufffd\ufffd", :intvector{ 0, 1, 2, 3, 3, 4, 5 }, :int{1}, :int{0}, "", "", :bin{""} } 737 738 // surrogates in CESU-8 739 { "CESU-8", :bin{ eda080eda081edb081 }, "\ud800\U00010401", :intvector{ 0, 3, 6 }, :int{1}, :int{0}, "", "", :bin{""} } 740 // e080 is a partial sequence 741 { "UTF-8", :bin{ 31ffe4ba8ce08061 }, "1\ufffd\u4e8c\ufffda", :intvector{ 0, 1, 2, 5, 7 }, :int{0}, :int{0}, "", "", :bin{ e080 } } 742 // fbbfbfbfbf exceedes U+10ffff 743 { "UTF-8", :bin{ 31fbbfbfbfbf61 }, "1\ufffda", :intvector{ 0, 1, 6 }, :int{0}, :int{0}, "", "", :bin{ fbbfbfbfbf } } 744 745 // lead byte a2 without trail byte 746 { "ibm-1363", :bin{ a2aea2 }, "\u00a1", :intvector{ 0 }, :int{1}, :int{0}, "truncated", ".", :bin{ a2 } } 747 { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} } 748 749 // simple sample, no error handling 750 { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} } 751 // Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } 752 { 753 "iscii-dev", 754 :bin{ EF4BC0E9BFE9E8D80AEF4AC0D4BFD4E8D80AEF4838B30AEF4939B30AEF4A3AB30AEF4B3BB30A}, 755 "\u0A5C\u0A4D\u0A39\u0A5C\u0A4D\u0A39\u000A" /* Gurmukhi test */ 756 "\u0AA2\u0AB5\u0AA1\u0AB5\u0ACD\u0AB9\u000A" /* Gujarati test */ 757 "\u0038\u0C95\u000A" /* Kannada test */ 758 "\u0039\u0D15\u000A" /* Malayalam test */ 759 "\u003A\u0A95\u000A" /* Gujarati test */ 760 "\u003B\u0A15\u000A" /* Punjabi test */, 761 :intvector { 2, 2, 2, 4, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 20, 21, 22, 25, 26, 27, 30, 31, 32, 35, 36, 37 }, 762 :int{1}, :int{0}, "", "", :bin{ "" } 763 } 764 { 765 "iscii-gur", 766 :bin{3BB30AC0E9BFE9E8D80AEF43C0E9BFE9E8D80A3BB30AEF403BB30A}, 767 "\u003b\u0a15\u000a" /* Easy characters */ 768 "\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39\u000a" /* Gurmukhi test */ 769 "\u09dd\u09dc\u09cd\u09b9\u000a" /* Switch script: to Bengali*/ 770 "\u003b\u0a15\u000a" /* Easy characters - new line" so should default!*/ 771 "\u003b\u0a15\u000a", /* Back to Gurmukhi*/ 772 :intvector { 0, 1, 2, 3, 3, 3, 5, 7, 8, 9, 13, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26 }, 773 :int{1}, :int{0}, "", "", :bin{ "" } 774 } 775 // Test Gurmukhi (Bindi Tippi and Consonant Cluster) 776 { 777 "iscii-gur", 778 :bin{EF4BB3A2A2B3E8B3C0E9BFE9E8D8}, 779 "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39", 780 :intvector { 2, 3, 4, 5, 5, 8, 8, 8, 10, 12, 13 }, 781 :int{1}, :int{0}, "", ".", :bin{ "" } 782 } 783 { // Verify Supplementary support 784 "Big5-HKSCS", 785 :bin{ fcfcfcfd }, 786 "\U000233E6\U00026DA0", 787 :intvector{ 0, 0, 2, 2 }, 788 :int{1}, :int{0}, "", "?", :bin{""} 789 } 790 { // Verify non-supplementary support 791 "big5-hkscs:unicode3.0", 792 :bin{ fcfcfcfd }, 793 "\uE1D4\uE1D5", 794 :intvector{ 0, 2 }, 795 :int{1}, :int{0}, "", "?", :bin{""} 796 } 797 { // Verify gb18030 enumeration 798 "gb18030", 799 :bin{ 8130D2398130D3308136A5318136A532 }, 800 "\u0450\u0452\u200F\u2011", 801 :intvector{ 0, 4, 8, 12 }, 802 :int{1}, :int{0}, "", "?", :bin{""} 803 } 804 // Improve coverage of ISCII 805 { 806 "iscii-bng", 807 :bin{ f0a0f0b0b8 }, 808 "", 809 :intvector{}, 810 :int{1}, :int{0}, "", "0", :bin{""} 811 } 812 // Test iso-2022-jp-2 miscellaneous symbols 813 { 814 "iso-2022-jp-2", 815 :bin{ 1b242843224f224e1b2842 }, 816 "\u260E\u260F", 817 :intvector{ 4, 6 }, 818 :int{1}, :int{0}, "", ".", :bin{""} 819 } 820 821 // Improve Code Coverage for BOCU-1 822 { 823 "BOCU-1", 824 :bin{ 91fbc555fd6349 }, 825 "\u0041\ud841\ud888\udc81", 826 :intvector{ 0, 1, 4, 4 }, 827 :int{1}, :int{0}, "", ".", :bin{""} 828 } 829 { 830 "BOCU-1", 831 :bin{ fbeda44ff0fe189bb821f05926 }, 832 "\ufe88\ufe70\udbff\udfff\u0061", 833 :intvector{}, 834 :int{1}, :int{0}, "", ".", :bin{""} 835 } 836 { 837 "BOCU-1", 838 :bin{ 5b4bccf9 }, 839 "\u000b", 840 :intvector{}, 841 :int{1}, :int{0}, "", "0", :bin{""} 842 } 843 { 844 "BOCU-1", 845 :bin{ fe0053c6 }, 846 "\u0003\u0076", 847 :intvector{}, 848 :int{1}, :int{0}, "", "0", :bin{""} 849 } 850 851 //Improve code coverage for SCSU 852 { 853 "SCSU", 854 :bin{ 0fd899dc7fd888dc99e041424361 }, 855 "\ud899\udc7f\ud888\udc99\u0041\u0042\u0043\u0061", 856 :intvector{}, 857 :int{1}, :int{0}, "", "0", :bin{""} 858 } 859 { 860 "SCSU", 861 :bin{ 41df1281035f10df1b03df1c88800bbfffff }, 862 "\u0041\u00df\u0401\u015f\u00df\u01df\uf000\udbff\udfff", 863 :intvector{}, 864 :int{1}, :int{0}, "", ".", :bin{""} 865 } 866 { 867 "SCSU", 868 :bin{ 1b9a1b541bb2411bfd1b0041 }, 869 "", 870 :intvector{}, 871 :int{1}, :int{0}, "", ".", :bin{""} 872 } 873 { 874 "SCSU", 875 :bin{ 0f6441b413a733f2 }, 876 "\u6441\ub413\ua733", 877 :intvector{}, 878 :int{1}, :int{0}, "illegal", ".", :bin{ f2 } 879 } 880 881 //Improve code coverage for MBCS 882 { 883 "*test5", 884 :bin{ 0506 }, 885 "\ufffd\x06", 886 :intvector{}, 887 :int{1}, :int{0}, "", "?", :bin{""} 888 } 889 { 890 "ibm-1390,swaplfnl", 891 :bin{ 430e4395ecc140400fc1e115 }, 892 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 893 :intvector{ 0, 2, 4, 4, 6, 9, 10, 11 }, 894 :int{1}, :int{0}, "", "?", :bin{""} 895 } 896 /* 897 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 898 { 899 "ISO-2022-CN-EXT", 900 :bin{ 1b242b4d1b4f66791b242b4d1b4f216a }, 901 "\u3667", 902 :intvector{ 14 }, 903 :int{1}, :int{0}, "", "0", :bin{""} 904 } 905 */ 906 { 907 "*test1bmp", 908 :bin{ 060a05 }, 909 "\u0066\ufffd\u0065", 910 :intvector{ 0, 2, 2 }, 911 :int{0}, :int{1}, "", "?", :bin{""} 912 } 913 { 914 "*test1bmp", 915 :bin{ 060708 }, 916 "\u0066", 917 :intvector{}, 918 :int{1}, :int{0}, "invalid", ".", :bin{ 07 } 919 } 920 { 921 "*test5", 922 :bin{ 010304 }, 923 "\u0034\ufffd", 924 :intvector{}, 925 :int{1}, :int{0}, "", "?", :bin{""} 926 } 927 { 928 "*test1", 929 :bin{ 0a0b }, 930 "", 931 :intvector{}, 932 :int{1}, :int{0}, "", "0", :bin{""} 933 } 934 { 935 "*test1bmp", 936 :bin{ 0c06 }, 937 "\u0066", 938 :intvector{}, 939 :int{0}, :int{0}, "", ".", :bin{""} 940 } 941 { 942 "*test5", 943 :bin{ 0906 }, 944 "\udbc8\udf45\u0006", 945 :intvector{}, 946 :int{0}, :int{0}, "", ".", :bin{""} 947 } 948 { 949 "ibm-16684", 950 :bin{ 0e }, 951 "", 952 :intvector{}, 953 :int{0}, :int{0}, "illegal", ".", :bin{ 0e } 954 } 955 } 956 } 957 958 // --------------------------------------------------------------------- *** 959 960 fromUnicode { 961 Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" } 962 Cases { 963 // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters 964 // For details about these encodings see convrtrs.txt. 965 // Standard UTF-16BE 966 { "UTF-16BE", "a", :bin{ 0061 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } 967 // Java "UnicodeBig" writes a BOM 968 { "UnicodeBig", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 969 // Standard UTF-16LE 970 { "UTF-16LE", "a", :bin{ 6100 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } 971 // Java "UnicodeLittle" writes a BOM 972 { "UnicodeLittle", "a", :bin{ fffe6100 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 973 974 // Test bug 6071 (1:2 Unicode:charset SBCS mapping). 975 { 976 "*test1bmp", 977 "e@t", 978 :bin{ 05000709 }, 979 :intvector{ 0,1,2,2 }, 980 :int{1}, :int{0}, "", "?", "" 981 } 982 // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and 983 // using the Shift-JIS table for JIS X 0208 (ticket #5797) 984 { 985 "ISO-2022-JP", 986 "\u203e\xa5\u4e00\ufa10\u6f3e\u0391", 987 :bin{ 1b284a7e5c1b2442306c222e5f2126211b2842 }, 988 :intvector{ 0,0,0,0,1,2,2,2,2,2,3,3,4,4,5,5,5,5,5 }, 989 :int{1}, :int{0}, "", "?=\u3013", "" // U+3013 Geta Mark converts to 222e 990 } 991 // Verify that mappings that would result in byte values outside 20..7F (for SBCS) 992 // or 21..7E (for DBCS) are not used. 993 // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46): 994 // <U009F> \x9F |0 (also in ISO 8859-1) 995 // <U0387> \xB7 |1 996 // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843): 997 // <UC829> \xA0\xA1 |0 998 // <UD4FE> \xC0\x41 |0 999 // <UD79D> \xC8\xFE |0 1000 { 1001 "JIS8", // =ISO_2022,locale=ja,version=4 1002 "\u009f\u0387\uc829\ud4fe\ud79d", 1003 :bin{ 1a1b2e461b4e371a1a1b242843487e1b2842 }, 1004 :intvector{ 0,1,1,1,1,1,1,2,3,4,4,4,4,4,4,4,4,4 }, 1005 :int{1}, :int{1}, "", "?", "" 1006 } 1007 // Ticket 5483: ISO 2022 converter incorrectly using fallback mapping 1008 // Verify that a roundtrip mapping is used even when a fallback mapping is 1009 // available in the current state. 1010 // U+FF61 is handled in code 1011 // jisx-208.ucm (<ESC>$B=1b2442): 1012 // <U30FE> \x21\x34 |0 1013 // <UFF5D> \x21\x51 |0 and 1014 // ibm-897_P100-1995.ucm (JIS X 0201, <ESC>(J=1b284a): 1015 // <UFF5D> \x7D |1 1016 // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46): 1017 // <U03D5> \xF6 |1 1018 // <U2015> \xAF |0 1019 // <UFF5D> \x7D |1 (not legal for ISO 2022) 1020 // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843): 1021 // <UAC00> \xB0\xA1 |0 1022 // <UFF5D> \xA3\xFD |0 1023 // <U223C> \xA1\xAD |0 (in extension table) 1024 { 1025 "JIS8", // =ISO_2022,locale=ja,version=4 1026 "a\uff61\u03d5\uff5d\uac00\u223c\uff5d\u30fe\uff5d", // Make it switch to ISO-8859-7, KSC 5601 and JIS X 0208. 1027 :bin{ 61a11b2e461b4e761b244221511b2428433021212d237d1b2442213421511b2842 }, 1028 :intvector{ 0,1,2,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,4,5,5,6,6,7,7,7,7,7,8,8,8,8,8 }, 1029 :int{1}, :int{1}, "", "?", "" 1030 } 1031 // Code coverage for UTF-8->SBCS conversion (ucnv_convertEx()). 1032 // Test code path for non-roundtripping ASCII characters 1033 // (try EBCDIC SBCS, and IBM PC SBCS with control code rotation). 1034 { 1035 "ibm-37", 1036 "a\x85c", 1037 :bin{ 811583 }, 1038 :intvector{ 0,1,2 }, 1039 :int{1}, :int{0}, "", "?", "" 1040 } 1041 { 1042 "ibm-850", 1043 "a\x1ac", 1044 :bin{ 617f63 }, 1045 :intvector{ 0,1,2 }, 1046 :int{1}, :int{0}, "", "?", "" 1047 } 1048 // Code coverage for UTF-8->DBCS conversion (ucnv_convertEx()). 1049 // Test code path for non-roundtripping ASCII characters 1050 // (try IBM PC DBCS with control code rotation). 1051 { 1052 "ibm-943", 1053 "a\x1ac\u30a1\x7ff", 1054 :bin{ 617f6383401c66 }, 1055 :intvector{ 0,1,2,3,3,4,5 }, 1056 :int{1}, :int{0}, "", "?", "" 1057 } 1058 // SCSU regression test. 1059 { 1060 "SCSU", 1061 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", 1062 :bin{ 310be0008102050fd900dc02e7320efffd330efffd34186881 }, 1063 :intvector{ 0,1,1,1,1,3,3,4,4,4,4,4,6,6,7,7,7,8,9,9,9,10,11,11,11 }, 1064 :int{1}, :int{0}, "", "?", "" 1065 } 1066 // LMBCS 1067 { 1068 "lmbcs", 1069 "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", 1070 :bin { 61039681a00f270f91140a7414f60214d84d14dc561088a0 }, 1071 :intvector{ 0, 1, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10 }, 1072 :int{1}, :int{0}, "", ".", "" 1073 } 1074 // Test substitution strings. 1075 { 1076 "windows-1252", // stateless MBCS with WriteSub(), should internally set char * 1077 "a\ufdd0c", 1078 :bin{ 61402421402463 }, 1079 :intvector{ 0,1,1,1,1,1,2 }, 1080 :int{1}, :int{0}, "", "?=@$!@$", "" 1081 } 1082 { 1083 "windows-1252", 1084 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", 1085 :bin{ 311a1a1a321a331a341a }, 1086 :intvector{ 0,1,3,4,6,7,8,9,10,11 }, 1087 :int{1}, :int{0}, "", "?", "" 1088 } 1089 { 1090 "windows-1252", 1091 "\uD87E", // lone surrogate can cause an offset overflow 1092 :bin{ 1a }, 1093 :intvector{ 0 }, 1094 :int{1}, :int{0}, "", "?", "" 1095 } 1096 { 1097 "windows-1252", 1098 "\uD87E", // lone surrogate can cause an offset overflow 1099 :bin{ 6875683f }, 1100 :intvector{ 0,0,0,0 }, 1101 :int{1}, :int{0}, "", "?=huh?", "" // Use a long substitution character 1102 } 1103 { 1104 "*test4", 1105 "\u30ab", // An incomplete multi-codepoint character 1106 :bin{ ff }, 1107 :intvector{ 0 }, 1108 :int{1}, :int{0}, "", "?", "" 1109 } 1110 { 1111 "ibm-930", // stateful MBCS 1112 "a\ufdd0\u4e00\ufdd0e", 1113 :bin{ 620e4bce0f400e45414bce0f4066 }, 1114 :intvector{ 0,1,1,1,1,1,2,2,2,3,3,3,3,4 }, 1115 :int{1}, :int{0}, "", "?=\u4e01 ", "" 1116 } 1117 { 1118 "iso-2022-jp", 1119 "a\x1bc", // Unicode ESC must not occur as a character 1120 :bin{ 6163 }, 1121 :intvector{ 0,2 }, 1122 :int{1}, :int{0}, "", "?=", "" // empty substitution string 1123 } 1124 { 1125 "iso-2022-cn", 1126 "a\x1bc", // Unicode ESC must not occur as a character 1127 :bin{ 61202063 }, 1128 :intvector{ 0,1,1,2 }, 1129 :int{1}, :int{0}, "", "?= ", "" 1130 } 1131 { 1132 "iso-2022-cn", 1133 "a\x1bc", // Unicode ESC must not occur as a character 1134 :bin{ 611b2429410e523b0f2063 }, 1135 :intvector{ 0,1,1,1,1,1,1,1,1,1,2 }, 1136 :int{1}, :int{0}, "", "?=\u4e00 ", "" 1137 } 1138 { 1139 "us-ascii", 1140 "a\x85c", 1141 :bin{ 61402421402463 }, 1142 :intvector{ 0,1,1,1,1,1,2 }, 1143 :int{1}, :int{0}, "", "?=@$!@$", "" 1144 } 1145 // ISO 2022-CN: test a single-byte subchar, j5171 1146 { 1147 "iso-2022-cn", 1148 "a\x1bc", // Unicode ESC must not occur as a character 1149 :bin{ 612163 }, 1150 :intvector{ 0,1,2 }, 1151 :int{1}, :int{0}, "", "?\x00\x21", "" 1152 } 1153 // UTF-16/32: do not output a BOM if there is no data at all 1154 { 1155 "UTF-16", 1156 "", 1157 :bin{ "" }, 1158 :intvector{ }, 1159 :int{1}, :int{1}, "", "?", "" 1160 } 1161 { 1162 "UTF-32", 1163 "", 1164 :bin{ "" }, 1165 :intvector{ }, 1166 :int{1}, :int{1}, "", "?", "" 1167 } 1168 1169 // do not convert SO/SI/ESC 1170 { 1171 "iso-2022-jp", 1172 "A\x0eB\x0f\x09\x1bC", 1173 :bin{ 411a421a091a43 }, 1174 :intvector{ 0,1,2,3,4,5,6 }, 1175 :int{1}, :int{1}, "", "?", "" 1176 } 1177 { 1178 "iso-2022-cn", 1179 "A\x0eB\x0f\x09\x1bC", 1180 :bin{ 411a421a091a43 }, 1181 :intvector{ 0,1,2,3,4,5,6 }, 1182 :int{1}, :int{1}, "", "?", "" 1183 } 1184 { 1185 "iso-2022-kr", 1186 "A\x0eB\x0f\x09\x1bC", 1187 :bin{ 1b242943411a421a091a43 }, 1188 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 }, 1189 :int{1}, :int{1}, "", "?", "" 1190 } 1191 { 1192 "ibm-25546", 1193 "A\x0eB\x0f\x09\x1bC", 1194 :bin{ 1b242943411a421a091a43 }, 1195 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 }, 1196 :int{1}, :int{1}, "", "?", "" 1197 } 1198 1199 // test ISO 8859-1/7 vs. JIS X 0201 1200 { 1201 "ISO-2022-JP-2", 1202 "?@A\u00e4\u03b1\u203EB", 1203 :bin{ 3f40411B2E411B4E641B2E461B4E611B284A7E421B2842 }, 1204 :intvector{ 0,1,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,6,6,6,6 }, 1205 :int{1}, :int{1}, "", "?", "" 1206 } 1207 1208 // Improve ucnv_ext.c code coverage: 1209 // There will be a partial match up to the lead surrogate of U+603ff 1210 // which then results in one more unit in the prefetch buffer 1211 // than the match length when converting one code unit at a time. 1212 // See ucnv_extContinueMatchFromU() comment 1213 // "the match did not use all of preFromU[] - keep the rest for replay" 1214 { 1215 "*test3", 1216 "\U00101234\U00101234\U00050005\U000603ff", 1217 :bin{ 07070001020e05ff }, 1218 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6 }, 1219 :int{1}, :int{0}, "", "?", "" 1220 } 1221 1222 // test that ISO-2022-JP encodes ASCII as itself 1223 { 1224 "ISO-2022-JP", 1225 "?@ABCDEFG", 1226 :bin{ 3f4041424344454647 }, 1227 :intvector{ 0,1,2,3,4,5,6,7,8 }, 1228 :int{1}, :int{1}, "", "?", "" 1229 } 1230 // test that ISO-2022-CN encodes ASCII as itself 1231 { 1232 "ISO-2022-CN", 1233 "?@ABCDEFG", 1234 :bin{ 3f4041424344454647 }, 1235 :intvector{ 0,1,2,3,4,5,6,7,8 }, 1236 :int{1}, :int{1}, "", "?", "" 1237 } 1238 1239 // moved from cintltst /tsconv/nccbtst/TestSkipCallBack 1240 { 1241 "iso-2022-jp", 1242 "\u3000\xe9\u3001", 1243 :bin{ 1b2442212121221b2842 }, 1244 :intvector{ 0,0,0,0,0,2,2,2,2,2 }, 1245 :int{1}, :int{1}, "", "0", "" 1246 } 1247 // moved from cintltst /tsconv/nccbtst/TestSubCallBack 1248 { 1249 "iso-2022-jp", 1250 "A\xe9B\xe9\u3000", 1251 :bin{ 411a421a1b244221211b2842 }, 1252 :intvector{ 0,1,2,3,4,4,4,4,4,4,4,4 }, 1253 :int{1}, :int{1}, "", "?", "" 1254 } 1255 // moved from cintltst /tsconv/nccbtst/TestSubWithValueCallBack 1256 { 1257 "iso-2022-jp", 1258 "A\xe9B\xe9\u3000", 1259 :bin{ 41255530304539422555303045391b244221211b2842 }, 1260 :intvector{ 0,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, 1261 :int{1}, :int{1}, "", "&", "" 1262 } 1263 { 1264 "iso-2022-cn", 1265 "\u4e00\u3712\u4e01", 1266 :bin{ 1b2429410e523b0f2555333731320e36210f }, 1267 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 }, 1268 :int{1}, :int{1}, "", "&", "" 1269 } 1270 { 1271 "iso-2022-cn", 1272 "A\u3712\u4e00", 1273 :bin{ 412555333731321b2429410e523b0f }, 1274 :intvector{ 0,1,1,1,1,1,1,2,2,2,2,2,2,2,2 }, 1275 :int{1}, :int{1}, "", "&", "" 1276 } 1277 { 1278 "iso-2022-cn", 1279 "\u3000\u3712\u3001", 1280 :bin{ 1b2429410e21210f2555333731320e21220f }, 1281 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 }, 1282 :int{1}, :int{1}, "", "&", "" 1283 } 1284 1285 // moved from cintltst /tsconv/nucnvtst/TestJIS 1286 { 1287 "JIS", 1288 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1289 :bin{ 1b244225412544256c256d256e256F25622564256625682569256a1b2842 }, 1290 :intvector{ 0,0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,11,11,11 }, 1291 :int{1}, :int{1}, "", "?", "" 1292 } 1293 { 1294 "JIS7", 1295 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1296 :bin{ 0e41420f1b2442256c256d256e256F0e5354555657580f1b2842 }, 1297 :intvector{ 0,0,1,2,2,2,2,2,2,3,3,4,4,5,5,6,6,7,8,9,10,11,11,11,11,11 }, 1298 :int{1}, :int{1}, "", "?", "" 1299 } 1300 { 1301 "JIS8", 1302 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1303 :bin{ C1C21b2442256c256d256e256F1b284AD3D4D5D6D7D81b2842 }, 1304 :intvector{ 0,1,2,2,2,2,2,3,3,4,4,5,5,6,6,6,6,7,8,9,10,11,11,11,11 }, 1305 :int{1}, :int{1}, "", "?", "" 1306 } 1307 1308 // moved from cintltst /tsconv/ncnvtst/TestErrorBehaviour 1309 { 1310 "iso-2022-jp", 1311 "\u3000\x50\udc01\u3001", 1312 :bin{ 1B244221211B2842501A1B24422122 }, 1313 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3 }, 1314 :int{0}, :int{1}, "", "?", "\udc01" 1315 } 1316 { 1317 "iso-2022-jp", 1318 "\u3000\x50\udc01\u3001", 1319 :bin{ 1B244221211B2842501A1B244221221b2842 }, 1320 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3,3,3,3 }, 1321 :int{1}, :int{1}, "", "?", "" 1322 } 1323 { 1324 "iso-2022-kr", 1325 "\x61\u4e00\udc01\u4e00", 1326 :bin{ 1b242943610e6c690f1a0e6c69 }, 1327 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3 }, 1328 :int{0}, :int{1}, "", "?", "\udc01" 1329 } 1330 { 1331 "iso-2022-kr", 1332 "\x61\u4e00\udc01\u4e00", 1333 :bin{ 1b242943610e6c690f1a0e6c690f }, 1334 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3,3 }, 1335 :int{1}, :int{1}, "", "?", "" 1336 } 1337 1338 // ISO-2022-KR 1339 { 1340 "ibm-25546", 1341 "AB\uc88b\U00050005\uacccC", 1342 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 }, 1343 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1344 :int{1}, :int{1}, "", "&U", "" 1345 } 1346 { 1347 "ibm-25546", 1348 "AB\uc88b\U00050005\uacccC", 1349 :bin{ 1b24294341420e41410f1a0e306a0f43 }, 1350 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 }, 1351 :int{1}, :int{1}, "", "?\x00\x1a", "" 1352 } 1353 { 1354 "ibm-25546", 1355 "AB\uc88b\U00050005\uacccC", 1356 :bin{ 1b24294341420e41412f7e306a0f43 }, 1357 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 }, 1358 :int{1}, :int{1}, "", "?", "" 1359 } 1360 { 1361 "ibm-25546", 1362 "AB\uc88b\U00050005\uaccc", 1363 :bin{ 1b24294341420e41412f7e306a0f }, 1364 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, 1365 :int{1}, :int{1}, "", "?", "" 1366 } 1367 { 1368 "ISO-2022-KR", 1369 "AB\uc88b\U00050005\uacccC", 1370 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 }, 1371 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1372 :int{1}, :int{1}, "", "&U", "" 1373 } 1374 { 1375 "ISO-2022-KR", 1376 "AB\uc88b\U00050005\uacccC", 1377 :bin{ 1b24294341420e41410f1a0e306a0f43 }, 1378 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 }, 1379 :int{1}, :int{1}, "", "?", "" 1380 } 1381 { 1382 "ISO-2022-KR", 1383 "AB\uc88b\U00050005\uacccC", 1384 :bin{ 1b24294341420e41412f7e306a0f43 }, 1385 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 }, 1386 :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" 1387 } 1388 { 1389 "ISO-2022-KR", 1390 "AB\uc88b\U00050005\uaccc", 1391 :bin{ 1b24294341420e41412f7e306a0f }, 1392 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, 1393 :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" 1394 } 1395 // ISO-2022-KR 1396 { 1397 "ibm-25546", 1398 "AB\uc88b\U00050005\uacccC", 1399 :bin{ 1b24294341420e41410f5c3530303035200e306a0f43 }, 1400 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1401 :int{1}, :int{1}, "", "&S", "" 1402 } 1403 1404 // ISO-2022-JP-2 with G2 designator & SS2 shift 1405 { 1406 "ISO-2022-JP-2", 1407 "CF\u758f\u038f\u7591", 1408 :bin{ 43461b244241411b2e461b4e3f353f1b2842 }, 1409 :intvector{ 0,1,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4 }, 1410 :int{1}, :int{1}, "", ".", "" 1411 } 1412 // JIS7 with Katakana 1413 { 1414 "JIS7", 1415 "AB\uff81\uff82AB", 1416 :bin{ 41420e41420f4142 }, 1417 :intvector{ 0,1,2,2,3,4,4,5 }, 1418 :int{1}, :int{1}, "", ".", "" 1419 } 1420 // JIS7 with shift to ASCII at the very end 1421 { 1422 "JIS7", 1423 "AB\uff81\uff82", 1424 :bin{ 41420e41420f }, 1425 :intvector{ 0,1,2,2,3,3 }, 1426 :int{1}, :int{1}, "", ".", "" 1427 } 1428 // JIS8 with Katakana 1429 { 1430 "JIS8", 1431 "A\uff81\\\xa5\uff82B", 1432 :bin{ 41c15c1b284a5cc2421b2842 }, 1433 :intvector{ 0,1,2,3,3,3,3,4,5,5,5,5 }, 1434 :int{1}, :int{1}, "", ".", "" 1435 } 1436 1437 /* 1438 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 1439 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points 1440 { 1441 "ISO-2022-CN-EXT", 1442 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30", 1443 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f30 }, 1444 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,6,6 }, 1445 :int{1}, :int{1}, "", ".", "" 1446 } 1447 // ISO-2022-CN-EXT with shift to ASCII at the very end 1448 { 1449 "ISO-2022-CN-EXT", 1450 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6", 1451 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f }, 1452 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4 }, 1453 :int{1}, :int{1}, "", ".", "" 1454 } 1455 // ISO-2022-CN-EXT without flush so do not shift to ASCII at the very end 1456 { 1457 "ISO-2022-CN-EXT", 1458 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6", 1459 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c }, 1460 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, 1461 :int{0}, :int{1}, "", ".", "" 1462 } 1463 */ 1464 1465 // windows-936 vs. ibm-1386 1466 { 1467 "ibm-1386", 1468 "\x1a\u20ac\u5555\x80\x81\U00055555", 1469 :bin{ 7fa2e3dffb7f7fa1a1 }, 1470 :intvector{ 0, 1, 1, 2, 2, 3, 4, 5, 5 }, 1471 :int{1}, :int{1}, "", "?", "" 1472 } 1473 { 1474 "windows-936", 1475 "\x1a\u20ac\u5555\x80\x81\U00055555", 1476 :bin{ 1a80dffb3f3f3f }, 1477 :intvector{ 0, 1, 2, 2, 3, 4, 5 }, 1478 :int{1}, :int{1}, "", "?", "" 1479 } 1480 1481 // verify that if a conversion table does not have any mapping for U+0000, 1482 // then there will not even be a phantom fallback to 00 1483 { 1484 "ibm-971", 1485 "\x00", 1486 :bin{ affe }, 1487 :intvector{ 0, 0 }, 1488 :int{1}, :int{1}, "", "?", "" 1489 } 1490 1491 { 1492 "*test4", 1493 "\x00", 1494 :bin{ ff }, 1495 :intvector{ 0 }, 1496 :int{1}, :int{1}, "", "?", "" 1497 } 1498 1499 // extension in testdata 1500 { 1501 "*test4x", 1502 "\u20ac\x09", 1503 :bin{ 0009 }, 1504 :intvector{ 0, 1 }, 1505 :int{1}, :int{1}, "", "?", "" 1506 } 1507 1508 // DBCS-only extensions 1509 { 1510 "ibm-970", 1511 "\x61\uffa1\u2015\ub000", 1512 :bin{ 611aa1aab2eb }, 1513 :intvector{ 0, 1, 2, 2, 3, 3 }, 1514 :int{1}, :int{1}, "", "?", "" 1515 } 1516 1517 { 1518 "ibm-971", 1519 "\x61\uffa1\u2015\ub000", 1520 :bin{ affeaffeaffeb2eb }, 1521 :intvector{ 0, 0, 1, 1, 2, 2, 3, 3 }, 1522 :int{1}, :int{1}, "", "?", "" 1523 } 1524 1525 { 1526 "ibm-1390,swaplfnl", 1527 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1528 :bin{ 430e4395ecc140400fc1e115 }, 1529 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 }, 1530 :int{1}, :int{0}, "", "?", "" 1531 } 1532 1533 { 1534 "ibm-16684", 1535 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1536 :bin{ fefe4395ecc14040fefe42e1fefe }, 1537 :intvector{ 0, 0, 1, 1, 2, 2, 4, 4, 5, 5, 6, 6, 7, 7 }, 1538 :int{1}, :int{0}, "", "?", "" 1539 } 1540 1541 { 1542 "ibm-1399", 1543 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1544 :bin{ 440e4395ecc140400fc1e125 }, 1545 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 }, 1546 :int{1}, :int{0}, "", "?", "" 1547 } 1548 1549 // <subchar1> from |2 mappings 1550 { 1551 "ibm-1390", 1552 "\x0e\x0f\u0901\U00050000\uffe8\uffee", 1553 :bin{ 3f3f0efefefefe0f3f3f }, 1554 :intvector{ 0, 1, 2, 2, 2, 3, 3, 5, 5, 6 }, 1555 :int{1}, :int{1}, "", "?", "" 1556 } 1557 1558 // <subchar1> from |2 mappings, and also contains a fallback to 00 1559 { 1560 "*test4", 1561 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30", 1562 :bin{ 0000e10102030affff }, 1563 :intvector{ 0, 1, 2, 4, 4, 4, 4, 6, 8 }, 1564 :int{1}, :int{1}, "", "?", "" 1565 } 1566 1567 // setting a <subchar> resets the <subchar1> 1568 { 1569 "*test4", 1570 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30", 1571 :bin{ 00000102030f0102030a0102030f0102030f }, 1572 :intvector{ 0, 1, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 8, 8, 8, 8 }, 1573 :int{1}, :int{1}, "", "?\x00\x01\x02\x03\x0f", "" 1574 } 1575 1576 // fallback to 00 with old single-byte data structure 1577 { 1578 "*test1", 1579 "\u20ac\u20ad\U00101234\U00050000", 1580 :bin{ 000007ff }, 1581 :intvector{ 0, 1, 2, 4 }, 1582 :int{1}, :int{1}, "", "?", "" 1583 } 1584 1585 // extensions 1586 { 1587 "ibm-1390", 1588 "\u025a\u025a\u0300\u025a\u0301\u025a\u0302\uffe8\U0002a0f9", 1589 :bin{ 0ed896eccaeccbd896ea530f3f0eb7c20f }, 1590 :intvector{ 0, 0, 0, 1, 1, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8 }, 1591 :int{1}, :int{0}, "", "?", "" 1592 } 1593 1594 { 1595 "*test3", 1596 "\xc4\xc4\xc4\U00101234\xc4\xc4\U00101234\x05", 1597 :bin{ ffffff070501020c }, 1598 :intvector{ 0, 1, 2, 3, 5, 5, 5, 5 }, 1599 :int{1}, :int{0}, "", "?", "" 1600 } 1601 1602 { 1603 "*test3", 1604 "\U00101234\U00101234\U00050005\U00101234\U00050005\U00060006", 1605 :bin{ 07070001020e05070001020f09 }, 1606 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6, 6, 6, 6, 6, 6 }, 1607 :int{1}, :int{0}, "", "?", "" 1608 } 1609 1610 // normal conversions 1611 { 1612 "UTF-16LE", 1613 "1\U00010001\U000500022\ud8003\udc014", 1614 :bin{ 310000d801dc00d902dc3200fdff3300fdff3400 }, 1615 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, 1616 :int{1}, :int{0}, "", "?", "" 1617 } 1618 { "UTF-16LE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } 1619 1620 { 1621 "UTF-16BE", 1622 "1\U00010001\U000500022\ud8003\udc014", 1623 :bin{ 0031d800dc01d900dc020032fffd0033fffd0034 }, 1624 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, 1625 :int{1}, :int{0}, "", "?", "" 1626 } 1627 { "UTF-16BE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } 1628 1629 { 1630 "SCSU", 1631 "1\U00010001\u00082\ud8003\udc014\ue001", 1632 :bin{ 310be000810108320efffd330efffd34186881 }, 1633 :intvector{ 0, 1, 1, 1, 1, 3, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9 }, 1634 :int{1}, :int{0}, "", "?", "" 1635 } 1636 // Test Gurmukhi (Bindi Tippi and Consonant clusters) 1637 { 1638 "iscii-gur", 1639 "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u000a\u0043\u0041\u000a", 1640 :bin { ef4bb3a2a2b3e8b3bfe9e8d80aef4b43410a }, 1641 :intvector{ 0, 0, 0, 1, 2, 3, 3, 3, 5, 5, 6, 7, 8, 8, 8, 9, 10, 11 }, 1642 :int{1}, :int{0}, "", ".", "" 1643 } 1644 // escape callback 1645 { 1646 "iscii-dev", 1647 "A\u0901\U00023456\u0902B\U00023456C", 1648 :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 }, 1649 :intvector{ 1650 0, 1651 1,1,1, 1652 2,2,2,2,2,2, 1653 2,2,2,2,2,2, 1654 4, 1655 5, 1656 6,6,6,6,6,6, 1657 6,6,6,6,6,6, 1658 8 1659 }, 1660 :int{1}, :int{0}, "", "&", "" 1661 } 1662 1663 // escape callback (hex) 1664 { 1665 "iso-2022-jp", 1666 "\u3000\U00023456\u3001\U00023456B\u901c", 1667 :bin{ 1b244221211b284226237832333435363b1b244221221b284226237832333435363b42262378393031433b }, 1668 :intvector{ 1669 0,0,0,0,0, 1670 1,1,1,1,1,1,1,1,1,1,1,1, 1671 3,3,3,3,3, 1672 4,4,4,4,4,4,4,4,4,4,4,4, 1673 6, 1674 7,7,7,7,7,7,7,7 1675 }, 1676 :int{1}, :int{0}, "", "&X", "" 1677 } 1678 1679 // sub callback 1680 { 1681 "gb18030", 1682 "$\x7f\x80\u01f9\u20ac\u4e00\u9fa6\uffff\U00010000\U0010ffff", 1683 :bin{ 247f81308130a8bfa2e3d2bb82358f338431a43990308130e3329a35 }, 1684 :intvector{ 0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 10, 10, 10, 10 }, 1685 :int{1}, :int{0}, "", "?", "" 1686 } 1687 1688 // skip callback 1689 { "ibm-930", "\u6D63\u6D64\u6D65\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } 1690 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } 1691 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63 }, :intvector{ 0, 0, 0, 1, 1 }, :int{1}, :int{0}, "illegal", "0i", "\ud89a" } 1692 1693 // sub callback for supplementary code point 1694 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } 1695 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } 1696 1697 // sub callback with AA as subchar 1698 { "ibm-920", "1\U000104012", :bin{ 31AA32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "?\x00\xAA", "" } 1699 1700 // same but not flushing 1701 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } 1702 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } 1703 1704 // simple sample, no error handling 1705 { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1706 1707 // Verify that incomplete surrogates are handled as an error 1708 { "UTF-8", "a\udc00", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1709 { "UTF-8", "a\ud800", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1710 { "UTF-8", "a\udc00b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } 1711 { "UTF-8", "a\ud800b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } 1712 1713 // Code coverage for the EUC variants. 1714 { "EUC-JP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4ae618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 6, 7, 7 }, :int{1}, :int{0}, "", "0", "" } 1715 { "EUC-JP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4aef4fef4fe618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7 }, :int{1}, :int{0}, "", "", "" } 1716 { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce561e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "0", "" } 1717 { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce5fdfefdfe61e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "", "" } 1718 1719 // Code Coverage for BOCU-1 1720 { 1721 "BOCU-1", 1722 "\u0041\ud841\ud888\udc81", 1723 :bin{ 91fbc555fd6349 }, 1724 :intvector{ 0, 1, 1, 1, 2, 2, 2 }, 1725 :int{1}, :int{0}, "", ".", "" 1726 } 1727 { 1728 "BOCU-1", 1729 "\ufe88\ufe70\udbff\udfff\u0061", 1730 :bin{ fbeda44ff0fe189bb821f05926 }, 1731 :intvector{}, 1732 :int{1}, :int{0}, "", ".", "" 1733 } 1734 // Improve code coverage for SCSU 1735 { 1736 "SCSU", 1737 "\ud899\udc7f\ud977\ud888\udc99\ud888\u0041", 1738 :bin{ 0fd899dc7fd888dc99e041 }, 1739 :intvector{}, 1740 :int{1}, :int{0}, "", "0", "" 1741 } 1742 } 1743 } 1744 1745 getUnicodeSet { 1746 // charset - will be opened, and ucnv_getUnicodeSet() called on it 1747 // map - set of code points and strings that must be in the returned set 1748 // mapnot - set of code points and strings that must *not* be in the returned set 1749 // which - numeric UConverterUnicodeSet value 1750 Headers { "charset", "map", "mapnot", "which" } 1751 Cases { 1752 // Unicode charsets that do not map surrogate code points 1753 { 1754 "UTF-8", 1755 "[\x00-\ud7ff\ue000-\U0010ffff]", 1756 "[\ud800-\udfff]", 1757 :int{0} 1758 } 1759 { 1760 "UTF-16", 1761 "[\x00-\ud7ff\ue000-\U0010ffff]", 1762 "[\ud800-\udfff]", 1763 :int{0} 1764 } 1765 { 1766 "UTF-16BE", 1767 "[\x00-\ud7ff\ue000-\U0010ffff]", 1768 "[\ud800-\udfff]", 1769 :int{0} 1770 } 1771 { 1772 "UTF-16LE", 1773 "[\x00-\ud7ff\ue000-\U0010ffff]", 1774 "[\ud800-\udfff]", 1775 :int{0} 1776 } 1777 { 1778 "UTF-32", 1779 "[\x00-\ud7ff\ue000-\U0010ffff]", 1780 "[\ud800-\udfff]", 1781 :int{0} 1782 } 1783 { 1784 "UTF-32BE", 1785 "[\x00-\ud7ff\ue000-\U0010ffff]", 1786 "[\ud800-\udfff]", 1787 :int{0} 1788 } 1789 { 1790 "UTF-32LE", 1791 "[\x00-\ud7ff\ue000-\U0010ffff]", 1792 "[\ud800-\udfff]", 1793 :int{0} 1794 } 1795 1796 // Unicode charsets that do map surrogate code points 1797 { 1798 "UTF-7", 1799 "[\x00-\U0010ffff]", 1800 "[]", 1801 :int{0} 1802 } 1803 { 1804 "SCSU", 1805 "[\x00-\U0010ffff]", 1806 "[]", 1807 :int{0} 1808 } 1809 { 1810 "BOCU-1", 1811 "[\x00-\U0010ffff]", 1812 "[]", 1813 :int{0} 1814 } 1815 { 1816 "CESU-8", 1817 "[\x00-\U0010ffff]", 1818 "[]", 1819 :int{0} 1820 } 1821 1822 // versions of ISO-2022-KR 1823 { 1824 "ISO-2022-KR", 1825 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac02\uffe6]", 1826 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac03\uffe7-\U0010ffff]", 1827 :int{0} 1828 } 1829 { 1830 "ibm-25546", 1831 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac01\uffe6]", 1832 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac02\uffe7-\U0010ffff]", 1833 :int{0} 1834 } 1835 1836 // versions of ISO-2022-JP 1837 { 1838 "ISO-2022-JP", 1839 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2015\u203e\u4e00\u4e01\uffe5]", 1840 "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u2014\u301c\u4e02\u4e27-\u4e29\u4fe0\u663b\u9eb5\ufa0e-\ufa2d\uff61-\uff9f\uffe4\uffe6-\U0010ffff]", 1841 :int{0} 1842 } 1843 { 1844 "ISO-2022-JP-2", 1845 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uffe6]", 1846 "[\x0e\x0f\x1b\uff61-\uff9f\uffe4\uffe7-\U0010ffff]", 1847 :int{0} 1848 } 1849 { 1850 "JIS7", 1851 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uff61-\uff9f\uffe6]", 1852 "[\x0e\x0f\x1b\uffe4\uffe7-\U0010ffff]", 1853 :int{0} 1854 } 1855 // with fallbacks 1856 { 1857 "ISO-2022-JP", 1858 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2014\u2015\u203e\u301c\u4e00\u4e01\u4fe0\u9eb5\uff61-\uff9f\uffe5]", 1859 "[\x0e\x0f\x1b\xa6\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\u663b\ufa0e-\ufa2d\uffe4\uffe6-\U0010ffff]", 1860 :int{1} 1861 } 1862 1863 // versions of ISO-2022-CN 1864 { 1865 "ISO-2022-CN", 1866 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00\u4e01\u9f98\ufe6b]", 1867 "[\x0e\x0f\x1b\u4e29\uffe6-\U0010ffff]", 1868 :int{0} 1869 } 1870 /* 1871 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 1872 { 1873 "ISO-2022-CN-EXT", 1874 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]", 1875 "[\x0e\x0f\x1b\U00020001\U00020002\U0002a6d7-\U0010ffff]", 1876 :int{0} 1877 } 1878 */ 1879 1880 // HZ 1881 { 1882 "HZ", 1883 "[\u0410-\u044f\u4e00\u4e01\u4e03]", 1884 "[\u4e02\u4e04-\u4e06\uac00-\ud7ff]", 1885 :int{0} 1886 } 1887 1888 // LMBCS 1889 { 1890 "LMBCS", 1891 "[\x00-\U0010ffff]", 1892 "[]", 1893 :int{0} 1894 } 1895 1896 // ISCII,version=0 (note: all versions of ISCII generates the same Unicodeset) 1897 { 1898 "iscii-dev", 1899 "[\x00-\xa0\u0909-\u0939\u0993-\u09a8\u0a13-\u0a28\u0a93-\u0aa8\u0ae6-\u0aef\u0b05-\u0b0c\u0b13-\u0b28\u0bae-\u0bb5\u0c12-\u0c28\u0c92-\u0ca8\u0d12-\u0d28]", 1900 "[\u0971-\u0975\u09e4\u0a4e-\u0a58\u0a80\u0b72-\u0b7a\u0bfb-\u0bff\u0c70-\u0c7f\u0c80\u0d00]", 1901 :int{0} 1902 } 1903 1904 { 1905 "iso-8859-1", 1906 "[\x00-\xff]", 1907 "[\u0100-\u01ff]", 1908 :int{0} 1909 } 1910 1911 { 1912 "us-ascii", 1913 "[\x00-\x7f]", 1914 "[\u0100-\u01ff]", 1915 :int{0} 1916 } 1917 // DBCS-only 1918 { 1919 "ibm-971", 1920 "[\xa1\xa4\uac01\ub000]", 1921 "[\x00-\x9f\u2015]", 1922 :int{0} 1923 } 1924 1925 { 1926 "ibm-16684", 1927 "[\xa0\xa1\xa4\xa6-\xab\xad-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2" 1928 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]", 1929 "[\x00-0x9f\xa2\xa3\xa5\xac\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]", 1930 :int{0} 1931 } 1932 1933 // extensions 1934 { 1935 "ibm-1390", 1936 "[\x00-\x0d\x10-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2" 1937 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]", 1938 "[\x0e\x0f\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]", 1939 :int{0} 1940 } 1941 1942 { 1943 "*test3", 1944 "[\x05\x0b\xc0\u20ac\U00023456\U00101234" 1945 "{\U00101234\U00050005\U00060006}{\U00101234\U00050005}{\U00101234\U00060006}{\xc4\xc4\U00101234\x05}]", 1946 "[\x06\x0e\U00034567\U000febcd{\U00101234\U00070007}]", 1947 :int{0} 1948 } 1949 } 1950 } 1951 } 1952 } 1953
