1 /******************************************************************** 2 * COPYRIGHT: 3 * Copyright (c) 1997-2011, International Business Machines Corporation and 4 * others. All Rights Reserved. 5 ********************************************************************/ 6 7 #include "unicode/utypes.h" 8 9 #if !UCONFIG_NO_NORMALIZATION 10 11 #include "unicode/uchar.h" 12 #include "unicode/errorcode.h" 13 #include "unicode/normlzr.h" 14 #include "unicode/uniset.h" 15 #include "unicode/usetiter.h" 16 #include "unicode/schriter.h" 17 #include "unicode/utf16.h" 18 #include "cstring.h" 19 #include "normalizer2impl.h" 20 #include "tstnorm.h" 21 22 #define LENGTHOF(array) ((int32_t)(sizeof(array)/sizeof((array)[0]))) 23 #define ARRAY_LENGTH(array) LENGTHOF(array) 24 25 #define CASE(id,test) case id: \ 26 name = #test; \ 27 if (exec) { \ 28 logln(#test "---"); \ 29 logln((UnicodeString)""); \ 30 test(); \ 31 } \ 32 break 33 34 static UErrorCode status = U_ZERO_ERROR; 35 36 void BasicNormalizerTest::runIndexedTest(int32_t index, UBool exec, 37 const char* &name, char* /*par*/) { 38 switch (index) { 39 CASE(0,TestDecomp); 40 CASE(1,TestCompatDecomp); 41 CASE(2,TestCanonCompose); 42 CASE(3,TestCompatCompose); 43 CASE(4,TestPrevious); 44 CASE(5,TestHangulDecomp); 45 CASE(6,TestHangulCompose); 46 CASE(7,TestTibetan); 47 CASE(8,TestCompositionExclusion); 48 CASE(9,TestZeroIndex); 49 CASE(10,TestVerisign); 50 CASE(11,TestPreviousNext); 51 CASE(12,TestNormalizerAPI); 52 CASE(13,TestConcatenate); 53 CASE(14,FindFoldFCDExceptions); 54 CASE(15,TestCompare); 55 CASE(16,TestSkippable); 56 #if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION 57 CASE(17,TestCustomComp); 58 CASE(18,TestCustomFCC); 59 #endif 60 CASE(19,TestFilteredNormalizer2Coverage); 61 default: name = ""; break; 62 } 63 } 64 65 /** 66 * Convert Java-style strings with \u Unicode escapes into UnicodeString objects 67 */ 68 static UnicodeString str(const char *input) 69 { 70 UnicodeString str(input, ""); // Invariant conversion 71 return str.unescape(); 72 } 73 74 75 BasicNormalizerTest::BasicNormalizerTest() 76 { 77 // canonTest 78 // Input Decomposed Composed 79 80 canonTests[0][0] = str("cat"); canonTests[0][1] = str("cat"); canonTests[0][2] = str("cat"); 81 82 canonTests[1][0] = str("\\u00e0ardvark"); canonTests[1][1] = str("a\\u0300ardvark"); canonTests[1][2] = str("\\u00e0ardvark"); 83 84 canonTests[2][0] = str("\\u1e0a"); canonTests[2][1] = str("D\\u0307"); canonTests[2][2] = str("\\u1e0a"); // D-dot_above 85 86 canonTests[3][0] = str("D\\u0307"); canonTests[3][1] = str("D\\u0307"); canonTests[3][2] = str("\\u1e0a"); // D dot_above 87 88 canonTests[4][0] = str("\\u1e0c\\u0307"); canonTests[4][1] = str("D\\u0323\\u0307"); canonTests[4][2] = str("\\u1e0c\\u0307"); // D-dot_below dot_above 89 90 canonTests[5][0] = str("\\u1e0a\\u0323"); canonTests[5][1] = str("D\\u0323\\u0307"); canonTests[5][2] = str("\\u1e0c\\u0307"); // D-dot_above dot_below 91 92 canonTests[6][0] = str("D\\u0307\\u0323"); canonTests[6][1] = str("D\\u0323\\u0307"); canonTests[6][2] = str("\\u1e0c\\u0307"); // D dot_below dot_above 93 94 canonTests[7][0] = str("\\u1e10\\u0307\\u0323"); canonTests[7][1] = str("D\\u0327\\u0323\\u0307"); canonTests[7][2] = str("\\u1e10\\u0323\\u0307"); // D dot_below cedilla dot_above 95 96 canonTests[8][0] = str("D\\u0307\\u0328\\u0323"); canonTests[8][1] = str("D\\u0328\\u0323\\u0307"); canonTests[8][2] = str("\\u1e0c\\u0328\\u0307"); // D dot_above ogonek dot_below 97 98 canonTests[9][0] = str("\\u1E14"); canonTests[9][1] = str("E\\u0304\\u0300"); canonTests[9][2] = str("\\u1E14"); // E-macron-grave 99 100 canonTests[10][0] = str("\\u0112\\u0300"); canonTests[10][1] = str("E\\u0304\\u0300"); canonTests[10][2] = str("\\u1E14"); // E-macron + grave 101 102 canonTests[11][0] = str("\\u00c8\\u0304"); canonTests[11][1] = str("E\\u0300\\u0304"); canonTests[11][2] = str("\\u00c8\\u0304"); // E-grave + macron 103 104 canonTests[12][0] = str("\\u212b"); canonTests[12][1] = str("A\\u030a"); canonTests[12][2] = str("\\u00c5"); // angstrom_sign 105 106 canonTests[13][0] = str("\\u00c5"); canonTests[13][1] = str("A\\u030a"); canonTests[13][2] = str("\\u00c5"); // A-ring 107 108 canonTests[14][0] = str("\\u00C4ffin"); canonTests[14][1] = str("A\\u0308ffin"); canonTests[14][2] = str("\\u00C4ffin"); 109 110 canonTests[15][0] = str("\\u00C4\\uFB03n"); canonTests[15][1] = str("A\\u0308\\uFB03n"); canonTests[15][2] = str("\\u00C4\\uFB03n"); 111 112 canonTests[16][0] = str("Henry IV"); canonTests[16][1] = str("Henry IV"); canonTests[16][2] = str("Henry IV"); 113 114 canonTests[17][0] = str("Henry \\u2163"); canonTests[17][1] = str("Henry \\u2163"); canonTests[17][2] = str("Henry \\u2163"); 115 116 canonTests[18][0] = str("\\u30AC"); canonTests[18][1] = str("\\u30AB\\u3099"); canonTests[18][2] = str("\\u30AC"); // ga (Katakana) 117 118 canonTests[19][0] = str("\\u30AB\\u3099"); canonTests[19][1] = str("\\u30AB\\u3099"); canonTests[19][2] = str("\\u30AC"); // ka + ten 119 120 canonTests[20][0] = str("\\uFF76\\uFF9E"); canonTests[20][1] = str("\\uFF76\\uFF9E"); canonTests[20][2] = str("\\uFF76\\uFF9E"); // hw_ka + hw_ten 121 122 canonTests[21][0] = str("\\u30AB\\uFF9E"); canonTests[21][1] = str("\\u30AB\\uFF9E"); canonTests[21][2] = str("\\u30AB\\uFF9E"); // ka + hw_ten 123 124 canonTests[22][0] = str("\\uFF76\\u3099"); canonTests[22][1] = str("\\uFF76\\u3099"); canonTests[22][2] = str("\\uFF76\\u3099"); // hw_ka + ten 125 126 canonTests[23][0] = str("A\\u0300\\u0316"); canonTests[23][1] = str("A\\u0316\\u0300"); canonTests[23][2] = str("\\u00C0\\u0316"); 127 128 /* compatTest */ 129 // Input Decomposed Composed 130 compatTests[0][0] = str("cat"); compatTests[0][1] = str("cat"); compatTests[0][2] = str("cat") ; 131 132 compatTests[1][0] = str("\\uFB4f"); compatTests[1][1] = str("\\u05D0\\u05DC"); compatTests[1][2] = str("\\u05D0\\u05DC"); // Alef-Lamed vs. Alef, Lamed 133 134 compatTests[2][0] = str("\\u00C4ffin"); compatTests[2][1] = str("A\\u0308ffin"); compatTests[2][2] = str("\\u00C4ffin") ; 135 136 compatTests[3][0] = str("\\u00C4\\uFB03n"); compatTests[3][1] = str("A\\u0308ffin"); compatTests[3][2] = str("\\u00C4ffin") ; // ffi ligature -> f + f + i 137 138 compatTests[4][0] = str("Henry IV"); compatTests[4][1] = str("Henry IV"); compatTests[4][2] = str("Henry IV") ; 139 140 compatTests[5][0] = str("Henry \\u2163"); compatTests[5][1] = str("Henry IV"); compatTests[5][2] = str("Henry IV") ; 141 142 compatTests[6][0] = str("\\u30AC"); compatTests[6][1] = str("\\u30AB\\u3099"); compatTests[6][2] = str("\\u30AC") ; // ga (Katakana) 143 144 compatTests[7][0] = str("\\u30AB\\u3099"); compatTests[7][1] = str("\\u30AB\\u3099"); compatTests[7][2] = str("\\u30AC") ; // ka + ten 145 146 compatTests[8][0] = str("\\uFF76\\u3099"); compatTests[8][1] = str("\\u30AB\\u3099"); compatTests[8][2] = str("\\u30AC") ; // hw_ka + ten 147 148 /* These two are broken in Unicode 2.1.2 but fixed in 2.1.5 and later */ 149 compatTests[9][0] = str("\\uFF76\\uFF9E"); compatTests[9][1] = str("\\u30AB\\u3099"); compatTests[9][2] = str("\\u30AC") ; // hw_ka + hw_ten 150 151 compatTests[10][0] = str("\\u30AB\\uFF9E"); compatTests[10][1] = str("\\u30AB\\u3099"); compatTests[10][2] = str("\\u30AC") ; // ka + hw_ten 152 153 /* Hangul Canonical */ 154 // Input Decomposed Composed 155 hangulCanon[0][0] = str("\\ud4db"); hangulCanon[0][1] = str("\\u1111\\u1171\\u11b6"); hangulCanon[0][2] = str("\\ud4db") ; 156 157 hangulCanon[1][0] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][1] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][2] = str("\\ud4db"); 158 } 159 160 BasicNormalizerTest::~BasicNormalizerTest() 161 { 162 } 163 164 void BasicNormalizerTest::TestPrevious() 165 { 166 Normalizer* norm = new Normalizer("", UNORM_NFD); 167 168 logln("testing decomp..."); 169 uint32_t i; 170 for (i = 0; i < ARRAY_LENGTH(canonTests); i++) { 171 backAndForth(norm, canonTests[i][0]); 172 } 173 174 logln("testing compose..."); 175 norm->setMode(UNORM_NFC); 176 for (i = 0; i < ARRAY_LENGTH(canonTests); i++) { 177 backAndForth(norm, canonTests[i][0]); 178 } 179 180 delete norm; 181 } 182 183 void BasicNormalizerTest::TestDecomp() 184 { 185 Normalizer* norm = new Normalizer("", UNORM_NFD); 186 iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 1); 187 staticTest(UNORM_NFD, 0, canonTests, ARRAY_LENGTH(canonTests), 1); 188 delete norm; 189 } 190 191 void BasicNormalizerTest::TestCompatDecomp() 192 { 193 Normalizer* norm = new Normalizer("", UNORM_NFKD); 194 iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 1); 195 196 staticTest(UNORM_NFKD, 0, 197 compatTests, ARRAY_LENGTH(compatTests), 1); 198 delete norm; 199 } 200 201 void BasicNormalizerTest::TestCanonCompose() 202 { 203 Normalizer* norm = new Normalizer("", UNORM_NFC); 204 iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 2); 205 206 staticTest(UNORM_NFC, 0, canonTests, 207 ARRAY_LENGTH(canonTests), 2); 208 delete norm; 209 } 210 211 void BasicNormalizerTest::TestCompatCompose() 212 { 213 Normalizer* norm = new Normalizer("", UNORM_NFKC); 214 iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 2); 215 216 staticTest(UNORM_NFKC, 0, 217 compatTests, ARRAY_LENGTH(compatTests), 2); 218 delete norm; 219 } 220 221 222 //------------------------------------------------------------------------------- 223 224 void BasicNormalizerTest::TestHangulCompose() 225 { 226 // Make sure that the static composition methods work 227 logln("Canonical composition..."); 228 staticTest(UNORM_NFC, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 2); 229 logln("Compatibility composition..."); 230 231 // Now try iterative composition.... 232 logln("Static composition..."); 233 Normalizer* norm = new Normalizer("", UNORM_NFC); 234 iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 2); 235 norm->setMode(UNORM_NFKC); 236 237 // And finally, make sure you can do it in reverse too 238 logln("Reverse iteration..."); 239 norm->setMode(UNORM_NFC); 240 for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) { 241 backAndForth(norm, hangulCanon[i][0]); 242 } 243 delete norm; 244 } 245 246 void BasicNormalizerTest::TestHangulDecomp() 247 { 248 // Make sure that the static decomposition methods work 249 logln("Canonical decomposition..."); 250 staticTest(UNORM_NFD, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 1); 251 logln("Compatibility decomposition..."); 252 253 // Now the iterative decomposition methods... 254 logln("Iterative decomposition..."); 255 Normalizer* norm = new Normalizer("", UNORM_NFD); 256 iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 1); 257 norm->setMode(UNORM_NFKD); 258 259 // And finally, make sure you can do it in reverse too 260 logln("Reverse iteration..."); 261 norm->setMode(UNORM_NFD); 262 for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) { 263 backAndForth(norm, hangulCanon[i][0]); 264 } 265 delete norm; 266 } 267 268 /** 269 * The Tibetan vowel sign AA, 0f71, was messed up prior to Unicode version 2.1.9. 270 */ 271 void BasicNormalizerTest::TestTibetan(void) { 272 UnicodeString decomp[1][3]; 273 decomp[0][0] = str("\\u0f77"); 274 decomp[0][1] = str("\\u0f77"); 275 decomp[0][2] = str("\\u0fb2\\u0f71\\u0f80"); 276 277 UnicodeString compose[1][3]; 278 compose[0][0] = str("\\u0fb2\\u0f71\\u0f80"); 279 compose[0][1] = str("\\u0fb2\\u0f71\\u0f80"); 280 compose[0][2] = str("\\u0fb2\\u0f71\\u0f80"); 281 282 staticTest(UNORM_NFD, 0, decomp, ARRAY_LENGTH(decomp), 1); 283 staticTest(UNORM_NFKD, 0, decomp, ARRAY_LENGTH(decomp), 2); 284 staticTest(UNORM_NFC, 0, compose, ARRAY_LENGTH(compose), 1); 285 staticTest(UNORM_NFKC, 0, compose, ARRAY_LENGTH(compose), 2); 286 } 287 288 /** 289 * Make sure characters in the CompositionExclusion.txt list do not get 290 * composed to. 291 */ 292 void BasicNormalizerTest::TestCompositionExclusion(void) { 293 // This list is generated from CompositionExclusion.txt. 294 // Update whenever the normalizer tables are updated. Note 295 // that we test all characters listed, even those that can be 296 // derived from the Unicode DB and are therefore commented 297 // out. 298 // ### TODO read composition exclusion from source/data/unidata file 299 // and test against that 300 UnicodeString EXCLUDED = str( 301 "\\u0340\\u0341\\u0343\\u0344\\u0374\\u037E\\u0387\\u0958" 302 "\\u0959\\u095A\\u095B\\u095C\\u095D\\u095E\\u095F\\u09DC" 303 "\\u09DD\\u09DF\\u0A33\\u0A36\\u0A59\\u0A5A\\u0A5B\\u0A5E" 304 "\\u0B5C\\u0B5D\\u0F43\\u0F4D\\u0F52\\u0F57\\u0F5C\\u0F69" 305 "\\u0F73\\u0F75\\u0F76\\u0F78\\u0F81\\u0F93\\u0F9D\\u0FA2" 306 "\\u0FA7\\u0FAC\\u0FB9\\u1F71\\u1F73\\u1F75\\u1F77\\u1F79" 307 "\\u1F7B\\u1F7D\\u1FBB\\u1FBE\\u1FC9\\u1FCB\\u1FD3\\u1FDB" 308 "\\u1FE3\\u1FEB\\u1FEE\\u1FEF\\u1FF9\\u1FFB\\u1FFD\\u2000" 309 "\\u2001\\u2126\\u212A\\u212B\\u2329\\u232A\\uF900\\uFA10" 310 "\\uFA12\\uFA15\\uFA20\\uFA22\\uFA25\\uFA26\\uFA2A\\uFB1F" 311 "\\uFB2A\\uFB2B\\uFB2C\\uFB2D\\uFB2E\\uFB2F\\uFB30\\uFB31" 312 "\\uFB32\\uFB33\\uFB34\\uFB35\\uFB36\\uFB38\\uFB39\\uFB3A" 313 "\\uFB3B\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46" 314 "\\uFB47\\uFB48\\uFB49\\uFB4A\\uFB4B\\uFB4C\\uFB4D\\uFB4E" 315 ); 316 for (int32_t i=0; i<EXCLUDED.length(); ++i) { 317 UnicodeString a(EXCLUDED.charAt(i)); 318 UnicodeString b; 319 UnicodeString c; 320 Normalizer::normalize(a, UNORM_NFKD, 0, b, status); 321 Normalizer::normalize(b, UNORM_NFC, 0, c, status); 322 if (c == a) { 323 errln("FAIL: " + hex(a) + " x DECOMP_COMPAT => " + 324 hex(b) + " x COMPOSE => " + 325 hex(c)); 326 } else if (verbose) { 327 logln("Ok: " + hex(a) + " x DECOMP_COMPAT => " + 328 hex(b) + " x COMPOSE => " + 329 hex(c)); 330 } 331 } 332 } 333 334 /** 335 * Test for a problem that showed up just before ICU 1.6 release 336 * having to do with combining characters with an index of zero. 337 * Such characters do not participate in any canonical 338 * decompositions. However, having an index of zero means that 339 * they all share one typeMask[] entry, that is, they all have to 340 * map to the same canonical class, which is not the case, in 341 * reality. 342 */ 343 void BasicNormalizerTest::TestZeroIndex(void) { 344 const char* DATA[] = { 345 // Expect col1 x COMPOSE_COMPAT => col2 346 // Expect col2 x DECOMP => col3 347 "A\\u0316\\u0300", "\\u00C0\\u0316", "A\\u0316\\u0300", 348 "A\\u0300\\u0316", "\\u00C0\\u0316", "A\\u0316\\u0300", 349 "A\\u0327\\u0300", "\\u00C0\\u0327", "A\\u0327\\u0300", 350 "c\\u0321\\u0327", "c\\u0321\\u0327", "c\\u0321\\u0327", 351 "c\\u0327\\u0321", "\\u00E7\\u0321", "c\\u0327\\u0321", 352 }; 353 int32_t DATA_length = (int32_t)(sizeof(DATA) / sizeof(DATA[0])); 354 355 for (int32_t i=0; i<DATA_length; i+=3) { 356 UErrorCode status = U_ZERO_ERROR; 357 UnicodeString a(DATA[i], ""); 358 a = a.unescape(); 359 UnicodeString b; 360 Normalizer::normalize(a, UNORM_NFKC, 0, b, status); 361 if (U_FAILURE(status)) { 362 dataerrln("Error calling normalize UNORM_NFKC: %s", u_errorName(status)); 363 } else { 364 UnicodeString exp(DATA[i+1], ""); 365 exp = exp.unescape(); 366 if (b == exp) { 367 logln((UnicodeString)"Ok: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b)); 368 } else { 369 errln((UnicodeString)"FAIL: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b) + 370 ", expect " + hex(exp)); 371 } 372 } 373 Normalizer::normalize(b, UNORM_NFD, 0, a, status); 374 if (U_FAILURE(status)) { 375 dataerrln("Error calling normalize UNORM_NFD: %s", u_errorName(status)); 376 } else { 377 UnicodeString exp = UnicodeString(DATA[i+2], "").unescape(); 378 if (a == exp) { 379 logln((UnicodeString)"Ok: " + hex(b) + " x DECOMP => " + hex(a)); 380 } else { 381 errln((UnicodeString)"FAIL: " + hex(b) + " x DECOMP => " + hex(a) + 382 ", expect " + hex(exp)); 383 } 384 } 385 } 386 } 387 388 /** 389 * Run a few specific cases that are failing for Verisign. 390 */ 391 void BasicNormalizerTest::TestVerisign(void) { 392 /* 393 > Their input: 394 > 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F 395 > Their output (supposedly from ICU): 396 > 05B8 05B1 05B9 0591 05C3 05B0 05AC 059F 397 > My output from charlint: 398 > 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F 399 400 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F => 05B1 05B8 05B9 0591 05C3 05B0 401 05AC 059F 402 403 U+05B8 18 E HEBREW POINT QAMATS 404 U+05B9 19 F HEBREW POINT HOLAM 405 U+05B1 11 HEBREW POINT HATAF SEGOL 406 U+0591 220 HEBREW ACCENT ETNAHTA 407 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 408 U+05B0 10 HEBREW POINT SHEVA 409 U+05AC 230 HEBREW ACCENT ILUY 410 U+059F 230 HEBREW ACCENT QARNEY PARA 411 412 U+05B1 11 HEBREW POINT HATAF SEGOL 413 U+05B8 18 HEBREW POINT QAMATS 414 U+05B9 19 HEBREW POINT HOLAM 415 U+0591 220 HEBREW ACCENT ETNAHTA 416 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 417 U+05B0 10 HEBREW POINT SHEVA 418 U+05AC 230 HEBREW ACCENT ILUY 419 U+059F 230 HEBREW ACCENT QARNEY PARA 420 421 Wrong result: 422 U+05B8 18 HEBREW POINT QAMATS 423 U+05B1 11 HEBREW POINT HATAF SEGOL 424 U+05B9 19 HEBREW POINT HOLAM 425 U+0591 220 HEBREW ACCENT ETNAHTA 426 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 427 U+05B0 10 HEBREW POINT SHEVA 428 U+05AC 230 HEBREW ACCENT ILUY 429 U+059F 230 HEBREW ACCENT QARNEY PARA 430 431 432 > Their input: 433 >0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD 434 >Their output (supposedly from ICU): 435 >0592 05B0 05B7 05BC 05A5 05C0 05AD 05C4 436 >My output from charlint: 437 >05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4 438 439 0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD => 05B0 05B7 05BC 05A5 0592 05C0 440 05AD 05C4 441 442 U+0592 230 HEBREW ACCENT SEGOL 443 U+05B7 17 HEBREW POINT PATAH 444 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 445 U+05A5 220 HEBREW ACCENT MERKHA 446 U+05B0 10 HEBREW POINT SHEVA 447 U+05C0 0 HEBREW PUNCTUATION PASEQ 448 U+05C4 230 HEBREW MARK UPPER DOT 449 U+05AD 222 HEBREW ACCENT DEHI 450 451 U+05B0 10 HEBREW POINT SHEVA 452 U+05B7 17 HEBREW POINT PATAH 453 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 454 U+05A5 220 HEBREW ACCENT MERKHA 455 U+0592 230 HEBREW ACCENT SEGOL 456 U+05C0 0 HEBREW PUNCTUATION PASEQ 457 U+05AD 222 HEBREW ACCENT DEHI 458 U+05C4 230 HEBREW MARK UPPER DOT 459 460 Wrong result: 461 U+0592 230 HEBREW ACCENT SEGOL 462 U+05B0 10 HEBREW POINT SHEVA 463 U+05B7 17 HEBREW POINT PATAH 464 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 465 U+05A5 220 HEBREW ACCENT MERKHA 466 U+05C0 0 HEBREW PUNCTUATION PASEQ 467 U+05AD 222 HEBREW ACCENT DEHI 468 U+05C4 230 HEBREW MARK UPPER DOT 469 */ 470 UnicodeString data[2][3]; 471 data[0][0] = str("\\u05B8\\u05B9\\u05B1\\u0591\\u05C3\\u05B0\\u05AC\\u059F"); 472 data[0][1] = str("\\u05B1\\u05B8\\u05B9\\u0591\\u05C3\\u05B0\\u05AC\\u059F"); 473 data[0][2] = str(""); 474 data[1][0] = str("\\u0592\\u05B7\\u05BC\\u05A5\\u05B0\\u05C0\\u05C4\\u05AD"); 475 data[1][1] = str("\\u05B0\\u05B7\\u05BC\\u05A5\\u0592\\u05C0\\u05AD\\u05C4"); 476 data[1][2] = str(""); 477 478 staticTest(UNORM_NFD, 0, data, ARRAY_LENGTH(data), 1); 479 staticTest(UNORM_NFC, 0, data, ARRAY_LENGTH(data), 1); 480 } 481 482 //------------------------------------------------------------------------ 483 // Internal utilities 484 // 485 486 UnicodeString BasicNormalizerTest::hex(UChar ch) { 487 UnicodeString result; 488 return appendHex(ch, 4, result); 489 } 490 491 UnicodeString BasicNormalizerTest::hex(const UnicodeString& s) { 492 UnicodeString result; 493 for (int i = 0; i < s.length(); ++i) { 494 if (i != 0) result += (UChar)0x2c/*,*/; 495 appendHex(s[i], 4, result); 496 } 497 return result; 498 } 499 500 501 inline static void insert(UnicodeString& dest, int pos, UChar32 ch) 502 { 503 dest.replace(pos, 0, ch); 504 } 505 506 void BasicNormalizerTest::backAndForth(Normalizer* iter, const UnicodeString& input) 507 { 508 UChar32 ch; 509 iter->setText(input, status); 510 511 // Run through the iterator forwards and stick it into a StringBuffer 512 UnicodeString forward; 513 for (ch = iter->first(); ch != iter->DONE; ch = iter->next()) { 514 forward += ch; 515 } 516 517 // Now do it backwards 518 UnicodeString reverse; 519 for (ch = iter->last(); ch != iter->DONE; ch = iter->previous()) { 520 insert(reverse, 0, ch); 521 } 522 523 if (forward != reverse) { 524 errln("Forward/reverse mismatch for input " + hex(input) 525 + ", forward: " + hex(forward) + ", backward: " + hex(reverse)); 526 } 527 } 528 529 void BasicNormalizerTest::staticTest(UNormalizationMode mode, int options, 530 UnicodeString tests[][3], int length, 531 int outCol) 532 { 533 for (int i = 0; i < length; i++) 534 { 535 UnicodeString& input = tests[i][0]; 536 UnicodeString& expect = tests[i][outCol]; 537 538 logln("Normalizing '" + input + "' (" + hex(input) + ")" ); 539 540 UnicodeString output; 541 Normalizer::normalize(input, mode, options, output, status); 542 543 if (output != expect) { 544 dataerrln(UnicodeString("ERROR: case ") + i + " normalized " + hex(input) + "\n" 545 + " expected " + hex(expect) + "\n" 546 + " static got " + hex(output) ); 547 } 548 } 549 } 550 551 void BasicNormalizerTest::iterateTest(Normalizer* iter, 552 UnicodeString tests[][3], int length, 553 int outCol) 554 { 555 for (int i = 0; i < length; i++) 556 { 557 UnicodeString& input = tests[i][0]; 558 UnicodeString& expect = tests[i][outCol]; 559 560 logln("Normalizing '" + input + "' (" + hex(input) + ")" ); 561 562 iter->setText(input, status); 563 assertEqual(input, expect, iter, UnicodeString("ERROR: case ") + i + " "); 564 } 565 } 566 567 void BasicNormalizerTest::assertEqual(const UnicodeString& input, 568 const UnicodeString& expected, 569 Normalizer* iter, 570 const UnicodeString& errPrefix) 571 { 572 UnicodeString result; 573 574 for (UChar32 ch = iter->first(); ch != iter->DONE; ch = iter->next()) { 575 result += ch; 576 } 577 if (result != expected) { 578 dataerrln(errPrefix + "normalized " + hex(input) + "\n" 579 + " expected " + hex(expected) + "\n" 580 + " iterate got " + hex(result) ); 581 } 582 } 583 584 // helper class for TestPreviousNext() 585 // simple UTF-32 character iterator 586 class UChar32Iterator { 587 public: 588 UChar32Iterator(const UChar32 *text, int32_t len, int32_t index) : 589 s(text), length(len), i(index) {} 590 591 UChar32 current() { 592 if(i<length) { 593 return s[i]; 594 } else { 595 return 0xffff; 596 } 597 } 598 599 UChar32 next() { 600 if(i<length) { 601 return s[i++]; 602 } else { 603 return 0xffff; 604 } 605 } 606 607 UChar32 previous() { 608 if(i>0) { 609 return s[--i]; 610 } else { 611 return 0xffff; 612 } 613 } 614 615 int32_t getIndex() { 616 return i; 617 } 618 private: 619 const UChar32 *s; 620 int32_t length, i; 621 }; 622 623 void 624 BasicNormalizerTest::TestPreviousNext(const UChar *src, int32_t srcLength, 625 const UChar32 *expect, int32_t expectLength, 626 const int32_t *expectIndex, // its length=expectLength+1 627 int32_t srcMiddle, int32_t expectMiddle, 628 const char *moves, 629 UNormalizationMode mode, 630 const char *name) { 631 // iterators 632 Normalizer iter(src, srcLength, mode); 633 634 // test getStaticClassID and getDynamicClassID 635 if(iter.getDynamicClassID() != Normalizer::getStaticClassID()) { 636 errln("getStaticClassID != getDynamicClassID for Normalizer."); 637 } 638 639 UChar32Iterator iter32(expect, expectLength, expectMiddle); 640 641 UChar32 c1, c2; 642 char m; 643 644 // initially set the indexes into the middle of the strings 645 iter.setIndexOnly(srcMiddle); 646 647 // move around and compare the iteration code points with 648 // the expected ones 649 const char *move=moves; 650 while((m=*move++)!=0) { 651 if(m=='-') { 652 c1=iter.previous(); 653 c2=iter32.previous(); 654 } else if(m=='0') { 655 c1=iter.current(); 656 c2=iter32.current(); 657 } else /* m=='+' */ { 658 c1=iter.next(); 659 c2=iter32.next(); 660 } 661 662 // compare results 663 if(c1!=c2) { 664 // copy the moves until the current (m) move, and terminate 665 char history[64]; 666 uprv_strcpy(history, moves); 667 history[move-moves]=0; 668 dataerrln("error: mismatch in Normalizer iteration (%s) at %s: " 669 "got c1=U+%04lx != expected c2=U+%04lx", 670 name, history, c1, c2); 671 break; 672 } 673 674 // compare indexes 675 if(iter.getIndex()!=expectIndex[iter32.getIndex()]) { 676 // copy the moves until the current (m) move, and terminate 677 char history[64]; 678 uprv_strcpy(history, moves); 679 history[move-moves]=0; 680 errln("error: index mismatch in Normalizer iteration (%s) at %s: " 681 "Normalizer index %ld expected %ld\n", 682 name, history, iter.getIndex(), expectIndex[iter32.getIndex()]); 683 break; 684 } 685 } 686 } 687 688 void 689 BasicNormalizerTest::TestPreviousNext() { 690 // src and expect strings 691 static const UChar src[]={ 692 U16_LEAD(0x2f999), U16_TRAIL(0x2f999), 693 U16_LEAD(0x1d15f), U16_TRAIL(0x1d15f), 694 0xc4, 695 0x1ed0 696 }; 697 static const UChar32 expect[]={ 698 0x831d, 699 0x1d158, 0x1d165, 700 0x41, 0x308, 701 0x4f, 0x302, 0x301 702 }; 703 704 // expected src indexes corresponding to expect indexes 705 static const int32_t expectIndex[]={ 706 0, 707 2, 2, 708 4, 4, 709 5, 5, 5, 710 6 // behind last character 711 }; 712 713 // src and expect strings for regression test for j2911 714 static const UChar src_j2911[]={ 715 U16_LEAD(0x2f999), U16_TRAIL(0x2f999), 716 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911 717 0xc4, 718 0x4f, 0x302, 0x301 719 }; 720 static const UChar32 expect_j2911[]={ 721 0x831d, 722 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911 723 0xc4, 724 0x1ed0 725 }; 726 727 // expected src indexes corresponding to expect indexes 728 static const int32_t expectIndex_j2911[]={ 729 0, 730 2, 3, 731 4, 732 5, 733 8 // behind last character 734 }; 735 736 // initial indexes into the src and expect strings 737 // for both sets of test data 738 enum { 739 SRC_MIDDLE=4, 740 EXPECT_MIDDLE=3, 741 SRC_MIDDLE_2=2, 742 EXPECT_MIDDLE_2=1 743 }; 744 745 // movement vector 746 // - for previous(), 0 for current(), + for next() 747 // for both sets of test data 748 static const char *const moves="0+0+0--0-0-+++0--+++++++0--------"; 749 750 TestPreviousNext(src, LENGTHOF(src), 751 expect, LENGTHOF(expect), 752 expectIndex, 753 SRC_MIDDLE, EXPECT_MIDDLE, 754 moves, UNORM_NFD, "basic"); 755 756 TestPreviousNext(src_j2911, LENGTHOF(src_j2911), 757 expect_j2911, LENGTHOF(expect_j2911), 758 expectIndex_j2911, 759 SRC_MIDDLE, EXPECT_MIDDLE, 760 moves, UNORM_NFKC, "j2911"); 761 762 // try again from different "middle" indexes 763 TestPreviousNext(src, LENGTHOF(src), 764 expect, LENGTHOF(expect), 765 expectIndex, 766 SRC_MIDDLE_2, EXPECT_MIDDLE_2, 767 moves, UNORM_NFD, "basic_2"); 768 769 TestPreviousNext(src_j2911, LENGTHOF(src_j2911), 770 expect_j2911, LENGTHOF(expect_j2911), 771 expectIndex_j2911, 772 SRC_MIDDLE_2, EXPECT_MIDDLE_2, 773 moves, UNORM_NFKC, "j2911_2"); 774 } 775 776 void BasicNormalizerTest::TestConcatenate() { 777 static const char *const 778 cases[][4]={ 779 /* mode, left, right, result */ 780 { 781 "C", 782 "re", 783 "\\u0301sum\\u00e9", 784 "r\\u00e9sum\\u00e9" 785 }, 786 { 787 "C", 788 "a\\u1100", 789 "\\u1161bcdefghijk", 790 "a\\uac00bcdefghijk" 791 }, 792 /* ### TODO: add more interesting cases */ 793 { 794 "D", 795 "\\u03B1\\u0345", 796 "\\u0C4D\\U000110BA\\U0001D169", 797 "\\u03B1\\U0001D169\\U000110BA\\u0C4D\\u0345" 798 } 799 }; 800 801 UnicodeString left, right, expect, result, r; 802 UErrorCode errorCode; 803 UNormalizationMode mode; 804 int32_t i; 805 806 /* test concatenation */ 807 for(i=0; i<(int32_t)(sizeof(cases)/sizeof(cases[0])); ++i) { 808 switch(*cases[i][0]) { 809 case 'C': mode=UNORM_NFC; break; 810 case 'D': mode=UNORM_NFD; break; 811 case 'c': mode=UNORM_NFKC; break; 812 case 'd': mode=UNORM_NFKD; break; 813 default: mode=UNORM_NONE; break; 814 } 815 816 left=UnicodeString(cases[i][1], "").unescape(); 817 right=UnicodeString(cases[i][2], "").unescape(); 818 expect=UnicodeString(cases[i][3], "").unescape(); 819 820 //result=r=UnicodeString(); 821 errorCode=U_ZERO_ERROR; 822 823 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 824 if(U_FAILURE(errorCode) || /*result!=r ||*/ result!=expect) { 825 dataerrln("error in Normalizer::concatenate(), cases[] fails with "+ 826 UnicodeString(u_errorName(errorCode))+", result==expect: expected: "+ 827 hex(expect)+" =========> got: " + hex(result)); 828 } 829 } 830 831 /* test error cases */ 832 833 /* left.getBuffer()==result.getBuffer() */ 834 result=r=expect=UnicodeString("zz", ""); 835 errorCode=U_UNEXPECTED_TOKEN; 836 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 837 if(errorCode!=U_UNEXPECTED_TOKEN || result!=r || !result.isBogus()) { 838 errln("error in Normalizer::concatenate(), violates UErrorCode protocol"); 839 } 840 841 left.setToBogus(); 842 errorCode=U_ZERO_ERROR; 843 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 844 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || result!=r || !result.isBogus()) { 845 errln("error in Normalizer::concatenate(), does not detect left.isBogus()"); 846 } 847 } 848 849 // reference implementation of Normalizer::compare 850 static int32_t 851 ref_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) { 852 UnicodeString r1, r2, t1, t2; 853 int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT); 854 855 if(options&U_COMPARE_IGNORE_CASE) { 856 Normalizer::decompose(s1, FALSE, normOptions, r1, errorCode); 857 Normalizer::decompose(s2, FALSE, normOptions, r2, errorCode); 858 859 r1.foldCase(options); 860 r2.foldCase(options); 861 } else { 862 r1=s1; 863 r2=s2; 864 } 865 866 Normalizer::decompose(r1, FALSE, normOptions, t1, errorCode); 867 Normalizer::decompose(r2, FALSE, normOptions, t2, errorCode); 868 869 if(options&U_COMPARE_CODE_POINT_ORDER) { 870 return t1.compareCodePointOrder(t2); 871 } else { 872 return t1.compare(t2); 873 } 874 } 875 876 // test wrapper for Normalizer::compare, sets UNORM_INPUT_IS_FCD appropriately 877 static int32_t 878 _norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) { 879 int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT); 880 881 if( UNORM_YES==Normalizer::quickCheck(s1, UNORM_FCD, normOptions, errorCode) && 882 UNORM_YES==Normalizer::quickCheck(s2, UNORM_FCD, normOptions, errorCode)) { 883 options|=UNORM_INPUT_IS_FCD; 884 } 885 886 return Normalizer::compare(s1, s2, options, errorCode); 887 } 888 889 // reference implementation of UnicodeString::caseCompare 890 static int32_t 891 ref_case_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options) { 892 UnicodeString t1, t2; 893 894 t1=s1; 895 t2=s2; 896 897 t1.foldCase(options); 898 t2.foldCase(options); 899 900 if(options&U_COMPARE_CODE_POINT_ORDER) { 901 return t1.compareCodePointOrder(t2); 902 } else { 903 return t1.compare(t2); 904 } 905 } 906 907 // reduce an integer to -1/0/1 908 static inline int32_t 909 _sign(int32_t value) { 910 if(value==0) { 911 return 0; 912 } else { 913 return (value>>31)|1; 914 } 915 } 916 917 static const char * 918 _signString(int32_t value) { 919 if(value<0) { 920 return "<0"; 921 } else if(value==0) { 922 return "=0"; 923 } else /* value>0 */ { 924 return ">0"; 925 } 926 } 927 928 void 929 BasicNormalizerTest::TestCompare() { 930 // test Normalizer::compare and unorm_compare (thinly wrapped by the former) 931 // by comparing it with its semantic equivalent 932 // since we trust the pieces, this is sufficient 933 934 // test each string with itself and each other 935 // each time with all options 936 static const char *const 937 strings[]={ 938 // some cases from NormalizationTest.txt 939 // 0..3 940 "D\\u031B\\u0307\\u0323", 941 "\\u1E0C\\u031B\\u0307", 942 "D\\u031B\\u0323\\u0307", 943 "d\\u031B\\u0323\\u0307", 944 945 // 4..6 946 "\\u00E4", 947 "a\\u0308", 948 "A\\u0308", 949 950 // Angstrom sign = A ring 951 // 7..10 952 "\\u212B", 953 "\\u00C5", 954 "A\\u030A", 955 "a\\u030A", 956 957 // 11.14 958 "a\\u059A\\u0316\\u302A\\u032Fb", 959 "a\\u302A\\u0316\\u032F\\u059Ab", 960 "a\\u302A\\u0316\\u032F\\u059Ab", 961 "A\\u059A\\u0316\\u302A\\u032Fb", 962 963 // from ICU case folding tests 964 // 15..20 965 "A\\u00df\\u00b5\\ufb03\\U0001040c\\u0131", 966 "ass\\u03bcffi\\U00010434i", 967 "\\u0061\\u0042\\u0131\\u03a3\\u00df\\ufb03\\ud93f\\udfff", 968 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udfff", 969 "\\u0041\\u0062\\u0131\\u03c3\\u0053\\u0073\\u0066\\u0046\\u0069\\ud93f\\udfff", 970 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udffd", 971 972 // U+d800 U+10001 see implementation comment in unorm_cmpEquivFold 973 // vs. U+10000 at bottom - code point order 974 // 21..22 975 "\\ud800\\ud800\\udc01", 976 "\\ud800\\udc00", 977 978 // other code point order tests from ustrtest.cpp 979 // 23..31 980 "\\u20ac\\ud801", 981 "\\u20ac\\ud800\\udc00", 982 "\\ud800", 983 "\\ud800\\uff61", 984 "\\udfff", 985 "\\uff61\\udfff", 986 "\\uff61\\ud800\\udc02", 987 "\\ud800\\udc02", 988 "\\ud84d\\udc56", 989 990 // long strings, see cnormtst.c/TestNormCoverage() 991 // equivalent if case-insensitive 992 // 32..33 993 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B" 994 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 995 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 996 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 997 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 998 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 999 "aaaaaaaaaaaaaaaaaazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" 1000 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" 1001 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" 1002 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" 1003 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B" 1004 "d\\u031B\\u0307\\u0323", 1005 1006 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa" 1007 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1008 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1009 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1010 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1011 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1012 "aaaaaaaaaaAAAAAAAAZZZZZZZZZZZZZZZZzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" 1013 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" 1014 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" 1015 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" 1016 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa" 1017 "\\u1E0C\\u031B\\u0307", 1018 1019 // some strings that may make a difference whether the compare function 1020 // case-folds or decomposes first 1021 // 34..41 1022 "\\u0360\\u0345\\u0334", 1023 "\\u0360\\u03b9\\u0334", 1024 1025 "\\u0360\\u1f80\\u0334", 1026 "\\u0360\\u03b1\\u0313\\u03b9\\u0334", 1027 1028 "\\u0360\\u1ffc\\u0334", 1029 "\\u0360\\u03c9\\u03b9\\u0334", 1030 1031 "a\\u0360\\u0345\\u0360\\u0345b", 1032 "a\\u0345\\u0360\\u0345\\u0360b", 1033 1034 // interesting cases for canonical caseless match with turkic i handling 1035 // 42..43 1036 "\\u00cc", 1037 "\\u0069\\u0300", 1038 1039 // strings with post-Unicode 3.2 normalization or normalization corrections 1040 // 44..45 1041 "\\u00e4\\u193b\\U0002f868", 1042 "\\u0061\\u193b\\u0308\\u36fc", 1043 1044 // empty string 1045 // 46 1046 "" 1047 }; 1048 1049 UnicodeString s[100]; // at least as many items as in strings[] ! 1050 1051 // all combinations of options 1052 // UNORM_INPUT_IS_FCD is set automatically if both input strings fulfill FCD conditions 1053 // set UNORM_UNICODE_3_2 in one additional combination 1054 static const struct { 1055 uint32_t options; 1056 const char *name; 1057 } opt[]={ 1058 { 0, "default" }, 1059 { U_COMPARE_CODE_POINT_ORDER, "c.p. order" }, 1060 { U_COMPARE_IGNORE_CASE, "ignore case" }, 1061 { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE, "c.p. order & ignore case" }, 1062 { U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "ignore case & special i" }, 1063 { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "c.p. order & ignore case & special i" }, 1064 { UNORM_UNICODE_3_2<<UNORM_COMPARE_NORM_OPTIONS_SHIFT, "Unicode 3.2" } 1065 }; 1066 1067 int32_t i, j, k, count=LENGTHOF(strings); 1068 int32_t result, refResult; 1069 1070 UErrorCode errorCode; 1071 1072 // create the UnicodeStrings 1073 for(i=0; i<count; ++i) { 1074 s[i]=UnicodeString(strings[i], "").unescape(); 1075 } 1076 1077 // test them each with each other 1078 for(i=0; i<count; ++i) { 1079 for(j=i; j<count; ++j) { 1080 for(k=0; k<LENGTHOF(opt); ++k) { 1081 // test Normalizer::compare 1082 errorCode=U_ZERO_ERROR; 1083 result=_norm_compare(s[i], s[j], opt[k].options, errorCode); 1084 refResult=ref_norm_compare(s[i], s[j], opt[k].options, errorCode); 1085 if(_sign(result)!=_sign(refResult)) { 1086 errln("Normalizer::compare(%d, %d, %s)%s should be %s %s", 1087 i, j, opt[k].name, _signString(result), _signString(refResult), 1088 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1089 } 1090 1091 // test UnicodeString::caseCompare - same internal implementation function 1092 if(opt[k].options&U_COMPARE_IGNORE_CASE) { 1093 errorCode=U_ZERO_ERROR; 1094 result=s[i].caseCompare(s[j], opt[k].options); 1095 refResult=ref_case_compare(s[i], s[j], opt[k].options); 1096 if(_sign(result)!=_sign(refResult)) { 1097 errln("UniStr::caseCompare(%d, %d, %s)%s should be %s %s", 1098 i, j, opt[k].name, _signString(result), _signString(refResult), 1099 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1100 } 1101 } 1102 } 1103 } 1104 } 1105 1106 // test cases with i and I to make sure Turkic works 1107 static const UChar iI[]={ 0x49, 0x69, 0x130, 0x131 }; 1108 UnicodeSet iSet, set; 1109 1110 UnicodeString s1, s2; 1111 1112 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode); 1113 if(U_FAILURE(errorCode) || !nfcImpl->ensureCanonIterData(errorCode)) { 1114 dataerrln("Normalizer2Factory::getNFCImpl().ensureCanonIterData() failed: %s", 1115 u_errorName(errorCode)); 1116 return; 1117 } 1118 1119 // collect all sets into one for contiguous output 1120 for(i=0; i<LENGTHOF(iI); ++i) { 1121 if(nfcImpl->getCanonStartSet(iI[i], iSet)) { 1122 set.addAll(iSet); 1123 } 1124 } 1125 1126 // test all of these precomposed characters 1127 const Normalizer2 *nfcNorm2=Normalizer2Factory::getNFCInstance(errorCode); 1128 UnicodeSetIterator it(set); 1129 while(it.next() && !it.isString()) { 1130 UChar32 c=it.getCodepoint(); 1131 if(!nfcNorm2->getDecomposition(c, s2)) { 1132 dataerrln("NFC.getDecomposition(i-composite U+%04lx) failed", (long)c); 1133 return; 1134 } 1135 1136 s1.setTo(c); 1137 for(k=0; k<LENGTHOF(opt); ++k) { 1138 // test Normalizer::compare 1139 errorCode=U_ZERO_ERROR; 1140 result=_norm_compare(s1, s2, opt[k].options, errorCode); 1141 refResult=ref_norm_compare(s1, s2, opt[k].options, errorCode); 1142 if(_sign(result)!=_sign(refResult)) { 1143 errln("Normalizer::compare(U+%04x with its NFD, %s)%s should be %s %s", 1144 c, opt[k].name, _signString(result), _signString(refResult), 1145 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1146 } 1147 1148 // test UnicodeString::caseCompare - same internal implementation function 1149 if(opt[k].options&U_COMPARE_IGNORE_CASE) { 1150 errorCode=U_ZERO_ERROR; 1151 result=s1.caseCompare(s2, opt[k].options); 1152 refResult=ref_case_compare(s1, s2, opt[k].options); 1153 if(_sign(result)!=_sign(refResult)) { 1154 errln("UniStr::caseCompare(U+%04x with its NFD, %s)%s should be %s %s", 1155 c, opt[k].name, _signString(result), _signString(refResult), 1156 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1157 } 1158 } 1159 } 1160 } 1161 1162 // test getDecomposition() for some characters that do not decompose 1163 if( nfcNorm2->getDecomposition(0x20, s2) || 1164 nfcNorm2->getDecomposition(0x4e00, s2) || 1165 nfcNorm2->getDecomposition(0x20002, s2) 1166 ) { 1167 errln("NFC.getDecomposition() returns TRUE for characters which do not have decompositions"); 1168 } 1169 1170 // test getRawDecomposition() for some characters that do not decompose 1171 if( nfcNorm2->getRawDecomposition(0x20, s2) || 1172 nfcNorm2->getRawDecomposition(0x4e00, s2) || 1173 nfcNorm2->getRawDecomposition(0x20002, s2) 1174 ) { 1175 errln("NFC.getRawDecomposition() returns TRUE for characters which do not have decompositions"); 1176 } 1177 1178 // test composePair() for some pairs of characters that do not compose 1179 if( nfcNorm2->composePair(0x20, 0x301)>=0 || 1180 nfcNorm2->composePair(0x61, 0x305)>=0 || 1181 nfcNorm2->composePair(0x1100, 0x1160)>=0 || 1182 nfcNorm2->composePair(0xac00, 0x11a7)>=0 1183 ) { 1184 errln("NFC.composePair() incorrectly composes some pairs of characters"); 1185 } 1186 1187 // test FilteredNormalizer2::getDecomposition() 1188 UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff]"), errorCode); 1189 FilteredNormalizer2 fn2(*nfcNorm2, filter); 1190 if( fn2.getDecomposition(0xe4, s1) || !fn2.getDecomposition(0x100, s2) || 1191 s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304 1192 ) { 1193 errln("FilteredNormalizer2(NFC, ^A0-FF).getDecomposition() failed"); 1194 } 1195 1196 // test FilteredNormalizer2::getRawDecomposition() 1197 if( fn2.getRawDecomposition(0xe4, s1) || !fn2.getRawDecomposition(0x100, s2) || 1198 s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304 1199 ) { 1200 errln("FilteredNormalizer2(NFC, ^A0-FF).getRawDecomposition() failed"); 1201 } 1202 1203 // test FilteredNormalizer2::composePair() 1204 if( 0x100!=fn2.composePair(0x41, 0x304) || 1205 fn2.composePair(0xc7, 0x301)>=0 // unfiltered result: U+1E08 1206 ) { 1207 errln("FilteredNormalizer2(NFC, ^A0-FF).composePair() failed"); 1208 } 1209 } 1210 1211 // verify that case-folding does not un-FCD strings 1212 int32_t 1213 BasicNormalizerTest::countFoldFCDExceptions(uint32_t foldingOptions) { 1214 UnicodeString s, fold, d; 1215 UChar32 c; 1216 int32_t count; 1217 uint8_t cc, trailCC, foldCC, foldTrailCC; 1218 UNormalizationCheckResult qcResult; 1219 int8_t category; 1220 UBool isNFD; 1221 UErrorCode errorCode; 1222 1223 logln("Test if case folding may un-FCD a string (folding options %04lx)", foldingOptions); 1224 1225 count=0; 1226 for(c=0; c<=0x10ffff; ++c) { 1227 errorCode = U_ZERO_ERROR; 1228 category=u_charType(c); 1229 if(category==U_UNASSIGNED) { 1230 continue; // skip unassigned code points 1231 } 1232 if(c==0xac00) { 1233 c=0xd7a3; // skip Hangul - no case folding there 1234 continue; 1235 } 1236 // skip Han blocks - no case folding there either 1237 if(c==0x3400) { 1238 c=0x4db5; 1239 continue; 1240 } 1241 if(c==0x4e00) { 1242 c=0x9fa5; 1243 continue; 1244 } 1245 if(c==0x20000) { 1246 c=0x2a6d6; 1247 continue; 1248 } 1249 1250 s.setTo(c); 1251 1252 // get leading and trailing cc for c 1253 Normalizer::decompose(s, FALSE, 0, d, errorCode); 1254 isNFD= s==d; 1255 cc=u_getCombiningClass(d.char32At(0)); 1256 trailCC=u_getCombiningClass(d.char32At(d.length()-1)); 1257 1258 // get leading and trailing cc for the case-folding of c 1259 s.foldCase(foldingOptions); 1260 Normalizer::decompose(s, FALSE, 0, d, errorCode); 1261 foldCC=u_getCombiningClass(d.char32At(0)); 1262 foldTrailCC=u_getCombiningClass(d.char32At(d.length()-1)); 1263 1264 qcResult=Normalizer::quickCheck(s, UNORM_FCD, errorCode); 1265 1266 if (U_FAILURE(errorCode)) { 1267 ++count; 1268 dataerrln("U+%04lx: Failed with error %s", u_errorName(errorCode)); 1269 } 1270 1271 // bad: 1272 // - character maps to empty string: adjacent characters may then need reordering 1273 // - folding has different leading/trailing cc's, and they don't become just 0 1274 // - folding itself is not FCD 1275 if( qcResult!=UNORM_YES || 1276 s.isEmpty() || 1277 (cc!=foldCC && foldCC!=0) || (trailCC!=foldTrailCC && foldTrailCC!=0) 1278 ) { 1279 ++count; 1280 dataerrln("U+%04lx: case-folding may un-FCD a string (folding options %04lx)", c, foldingOptions); 1281 dataerrln(" cc %02x trailCC %02x foldCC(U+%04lx) %02x foldTrailCC(U+%04lx) %02x quickCheck(folded)=%d", cc, trailCC, d.char32At(0), foldCC, d.char32At(d.length()-1), foldTrailCC, qcResult); 1282 continue; 1283 } 1284 1285 // also bad: 1286 // if a code point is in NFD but its case folding is not, then 1287 // unorm_compare will also fail 1288 if(isNFD && UNORM_YES!=Normalizer::quickCheck(s, UNORM_NFD, errorCode)) { 1289 ++count; 1290 errln("U+%04lx: case-folding un-NFDs this character (folding options %04lx)", c, foldingOptions); 1291 } 1292 } 1293 1294 logln("There are %ld code points for which case-folding may un-FCD a string (folding options %04lx)", count, foldingOptions); 1295 return count; 1296 } 1297 1298 void 1299 BasicNormalizerTest::FindFoldFCDExceptions() { 1300 int32_t count; 1301 1302 count=countFoldFCDExceptions(0); 1303 count+=countFoldFCDExceptions(U_FOLD_CASE_EXCLUDE_SPECIAL_I); 1304 if(count>0) { 1305 /* 1306 * If case-folding un-FCDs any strings, then unorm_compare() must be 1307 * re-implemented. 1308 * It currently assumes that one can check for FCD then case-fold 1309 * and then still have FCD strings for raw decomposition without reordering. 1310 */ 1311 dataerrln("error: There are %ld code points for which case-folding may un-FCD a string for all folding options.\n" 1312 "See comment in BasicNormalizerTest::FindFoldFCDExceptions()!", count); 1313 } 1314 } 1315 1316 static void 1317 initExpectedSkippables(UnicodeSet skipSets[UNORM_MODE_COUNT], UErrorCode &errorCode) { 1318 skipSets[UNORM_NFD].applyPattern( 1319 UNICODE_STRING_SIMPLE("[[:NFD_QC=Yes:]&[:ccc=0:]]"), errorCode); 1320 skipSets[UNORM_NFC].applyPattern( 1321 UNICODE_STRING_SIMPLE("[[:NFC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode); 1322 skipSets[UNORM_NFKD].applyPattern( 1323 UNICODE_STRING_SIMPLE("[[:NFKD_QC=Yes:]&[:ccc=0:]]"), errorCode); 1324 skipSets[UNORM_NFKC].applyPattern( 1325 UNICODE_STRING_SIMPLE("[[:NFKC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode); 1326 1327 // Remove from the NFC and NFKC sets all those characters that change 1328 // when a back-combining character is added. 1329 // First, get all of the back-combining characters and their combining classes. 1330 UnicodeSet combineBack("[:NFC_QC=Maybe:]", errorCode); 1331 int32_t numCombineBack=combineBack.size(); 1332 int32_t *combineBackCharsAndCc=new int32_t[numCombineBack*2]; 1333 UnicodeSetIterator iter(combineBack); 1334 for(int32_t i=0; i<numCombineBack; ++i) { 1335 iter.next(); 1336 UChar32 c=iter.getCodepoint(); 1337 combineBackCharsAndCc[2*i]=c; 1338 combineBackCharsAndCc[2*i+1]=u_getCombiningClass(c); 1339 } 1340 1341 // We need not look at control codes, Han characters nor Hangul LVT syllables because they 1342 // do not combine forward. LV syllables are already removed. 1343 UnicodeSet notInteresting("[[:C:][:Unified_Ideograph:][:HST=LVT:]]", errorCode); 1344 LocalPointer<UnicodeSet> unsure(&((UnicodeSet *)(skipSets[UNORM_NFC].clone()))->removeAll(notInteresting)); 1345 // System.out.format("unsure.size()=%d\n", unsure.size()); 1346 1347 // For each character about which we are unsure, see if it changes when we add 1348 // one of the back-combining characters. 1349 const Normalizer2 *norm2=Normalizer2::getNFCInstance(errorCode); 1350 UnicodeString s; 1351 iter.reset(*unsure); 1352 while(iter.next()) { 1353 UChar32 c=iter.getCodepoint(); 1354 s.setTo(c); 1355 int32_t cLength=s.length(); 1356 int32_t tccc=u_getIntPropertyValue(c, UCHAR_TRAIL_CANONICAL_COMBINING_CLASS); 1357 for(int32_t i=0; i<numCombineBack; ++i) { 1358 // If c's decomposition ends with a character with non-zero combining class, then 1359 // c can only change if it combines with a character with a non-zero combining class. 1360 int32_t cc2=combineBackCharsAndCc[2*i+1]; 1361 if(tccc==0 || cc2!=0) { 1362 UChar32 c2=combineBackCharsAndCc[2*i]; 1363 s.append(c2); 1364 if(!norm2->isNormalized(s, errorCode)) { 1365 // System.out.format("remove U+%04x (tccc=%d) + U+%04x (cc=%d)\n", c, tccc, c2, cc2); 1366 skipSets[UNORM_NFC].remove(c); 1367 skipSets[UNORM_NFKC].remove(c); 1368 break; 1369 } 1370 s.truncate(cLength); 1371 } 1372 } 1373 } 1374 delete [] combineBackCharsAndCc; 1375 } 1376 1377 void 1378 BasicNormalizerTest::TestSkippable() { 1379 UnicodeSet diff, skipSets[UNORM_MODE_COUNT], expectSets[UNORM_MODE_COUNT]; 1380 UnicodeString s, pattern; 1381 1382 /* build NF*Skippable sets from runtime data */ 1383 IcuTestErrorCode errorCode(*this, "TestSkippable"); 1384 skipSets[UNORM_NFD].applyPattern(UNICODE_STRING_SIMPLE("[:NFD_Inert:]"), errorCode); 1385 skipSets[UNORM_NFKD].applyPattern(UNICODE_STRING_SIMPLE("[:NFKD_Inert:]"), errorCode); 1386 skipSets[UNORM_NFC].applyPattern(UNICODE_STRING_SIMPLE("[:NFC_Inert:]"), errorCode); 1387 skipSets[UNORM_NFKC].applyPattern(UNICODE_STRING_SIMPLE("[:NFKC_Inert:]"), errorCode); 1388 if(errorCode.logDataIfFailureAndReset("UnicodeSet(NF..._Inert) failed")) { 1389 return; 1390 } 1391 1392 /* get expected sets from hardcoded patterns */ 1393 initExpectedSkippables(expectSets, errorCode); 1394 errorCode.assertSuccess(); 1395 1396 for(int32_t i=UNORM_NONE; i<UNORM_MODE_COUNT; ++i) { 1397 if(skipSets[i]!=expectSets[i]) { 1398 errln("error: TestSkippable skipSets[%d]!=expectedSets[%d]\n", i, i); 1399 // Note: This used to depend on hardcoded UnicodeSet patterns generated by 1400 // Mark's unicodetools.com.ibm.text.UCD.NFSkippable, by 1401 // running com.ibm.text.UCD.Main with the option NFSkippable. 1402 // Since ICU 4.6/Unicode 6, we are generating the 1403 // expectSets ourselves in initSkippables(). 1404 1405 s=UNICODE_STRING_SIMPLE("skip-expect="); 1406 (diff=skipSets[i]).removeAll(expectSets[i]).toPattern(pattern, TRUE); 1407 s.append(pattern); 1408 1409 pattern.remove(); 1410 s.append(UNICODE_STRING_SIMPLE("\n\nexpect-skip=")); 1411 (diff=expectSets[i]).removeAll(skipSets[i]).toPattern(pattern, TRUE); 1412 s.append(pattern); 1413 s.append(UNICODE_STRING_SIMPLE("\n\n")); 1414 1415 errln(s); 1416 } 1417 } 1418 } 1419 1420 struct StringPair { const char *input, *expected; }; 1421 1422 void 1423 BasicNormalizerTest::TestCustomComp() { 1424 static const StringPair pairs[]={ 1425 { "\\uD801\\uE000\\uDFFE", "" }, 1426 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" }, 1427 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" }, 1428 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE002\\U000110B9\\u0327\\u0345" }, 1429 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" }, 1430 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" }, 1431 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" }, 1432 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" } 1433 }; 1434 IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomComp"); 1435 const Normalizer2 *customNorm2= 1436 Normalizer2::getInstance(loadTestData(errorCode), "testnorm", 1437 UNORM2_COMPOSE, errorCode); 1438 if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) { 1439 return; 1440 } 1441 for(int32_t i=0; i<LENGTHOF(pairs); ++i) { 1442 const StringPair &pair=pairs[i]; 1443 UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape(); 1444 UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape(); 1445 UnicodeString result=customNorm2->normalize(input, errorCode); 1446 if(result!=expected) { 1447 errln("custom compose Normalizer2 did not normalize input %d as expected", i); 1448 } 1449 } 1450 } 1451 1452 void 1453 BasicNormalizerTest::TestCustomFCC() { 1454 static const StringPair pairs[]={ 1455 { "\\uD801\\uE000\\uDFFE", "" }, 1456 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" }, 1457 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" }, 1458 // The following expected result is different from CustomComp 1459 // because of only-contiguous composition. 1460 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE001\\U000110B9\\u0327\\u0308\\u0345" }, 1461 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" }, 1462 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" }, 1463 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" }, 1464 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" } 1465 }; 1466 IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomFCC"); 1467 const Normalizer2 *customNorm2= 1468 Normalizer2::getInstance(loadTestData(errorCode), "testnorm", 1469 UNORM2_COMPOSE_CONTIGUOUS, errorCode); 1470 if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) { 1471 return; 1472 } 1473 for(int32_t i=0; i<LENGTHOF(pairs); ++i) { 1474 const StringPair &pair=pairs[i]; 1475 UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape(); 1476 UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape(); 1477 UnicodeString result=customNorm2->normalize(input, errorCode); 1478 if(result!=expected) { 1479 errln("custom FCC Normalizer2 did not normalize input %d as expected", i); 1480 } 1481 } 1482 } 1483 1484 /* Improve code coverage of Normalizer2 */ 1485 void 1486 BasicNormalizerTest::TestFilteredNormalizer2Coverage() { 1487 UErrorCode errorCode = U_ZERO_ERROR; 1488 const Normalizer2 *nfcNorm2=Normalizer2Factory::getNFCInstance(errorCode); 1489 if (U_FAILURE(errorCode)) { 1490 dataerrln("Normalizer2Factory::getNFCInstance() call failed - %s", u_errorName(status)); 1491 return; 1492 } 1493 UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff\\u0310-\\u031f]"), errorCode); 1494 FilteredNormalizer2 fn2(*nfcNorm2, filter); 1495 1496 UChar32 char32 = 0x0054; 1497 1498 if (fn2.isInert(char32)) { 1499 errln("FilteredNormalizer2.isInert() failed."); 1500 } 1501 1502 if (fn2.hasBoundaryAfter(char32)) { 1503 errln("FilteredNormalizer2.hasBoundaryAfter() failed."); 1504 } 1505 1506 UChar32 c; 1507 for(c=0; c<=0x3ff; ++c) { 1508 uint8_t expectedCC= filter.contains(c) ? nfcNorm2->getCombiningClass(c) : 0; 1509 uint8_t cc=fn2.getCombiningClass(c); 1510 if(cc!=expectedCC) { 1511 errln( 1512 UnicodeString("FilteredNormalizer2(NFC, ^A0-FF,310-31F).getCombiningClass(U+")+ 1513 hex(c)+ 1514 ")==filtered NFC.getCC()"); 1515 } 1516 } 1517 1518 UnicodeString newString1 = UNICODE_STRING_SIMPLE("[^\\u0100-\\u01ff]"); 1519 UnicodeString newString2 = UNICODE_STRING_SIMPLE("[^\\u0200-\\u02ff]"); 1520 fn2.append(newString1, newString2, errorCode); 1521 if (U_FAILURE(errorCode)) { 1522 errln("FilteredNormalizer2.append() failed."); 1523 } 1524 } 1525 1526 #endif /* #if !UCONFIG_NO_NORMALIZATION */ 1527
