1 /* 2 ************************************************************************ 3 * Copyright (c) 1997-2009, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ************************************************************************ 6 */ 7 8 #include "unicode/utypes.h" 9 10 #if !UCONFIG_NO_NORMALIZATION 11 12 #include "unicode/uchar.h" 13 #include "unicode/normlzr.h" 14 #include "unicode/uniset.h" 15 #include "unicode/putil.h" 16 #include "unormimp.h" 17 #include "cstring.h" 18 #include "filestrm.h" 19 #include "normconf.h" 20 #include <stdio.h> 21 22 #define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) 23 24 #define CASE(id,test) case id: \ 25 name = #test; \ 26 if (exec) { \ 27 logln(#test "---"); \ 28 logln((UnicodeString)""); \ 29 test(); \ 30 } \ 31 break 32 33 void NormalizerConformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) { 34 switch (index) { 35 CASE(0, TestConformance); 36 CASE(1, TestConformance32); 37 // CASE(2, TestCase6); 38 default: name = ""; break; 39 } 40 } 41 42 #define FIELD_COUNT 5 43 44 NormalizerConformanceTest::NormalizerConformanceTest() : 45 normalizer(UnicodeString(), UNORM_NFC) {} 46 47 NormalizerConformanceTest::~NormalizerConformanceTest() {} 48 49 // more interesting conformance test cases, not in the unicode.org NormalizationTest.txt 50 static const char *moreCases[]={ 51 // Markus 2001aug30 52 "0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0", 53 54 // Markus 2001oct26 - test edge case for iteration: U+0f73.cc==0 but decomposition.lead.cc==129 55 "0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1" 56 }; 57 58 void NormalizerConformanceTest::compare(const UnicodeString& s1, const UnicodeString& s2){ 59 UErrorCode status=U_ZERO_ERROR; 60 // TODO: Re-enable this tests after UTC fixes UAX 21 61 if(s1.indexOf((UChar32)0x0345)>=0)return; 62 if(Normalizer::compare(s1,s2,U_FOLD_CASE_DEFAULT,status)!=0){ 63 errln("Normalizer::compare() failed for s1: " + prettify(s1) + " s2: " +prettify(s2)); 64 } 65 } 66 67 FileStream * 68 NormalizerConformanceTest::openNormalizationTestFile(const char *filename) { 69 char unidataPath[2000]; 70 const char *folder; 71 FileStream *input; 72 UErrorCode errorCode; 73 74 // look inside ICU_DATA first 75 folder=pathToDataDirectory(); 76 if(folder!=NULL) { 77 strcpy(unidataPath, folder); 78 strcat(unidataPath, "unidata" U_FILE_SEP_STRING); 79 strcat(unidataPath, filename); 80 input=T_FileStream_open(unidataPath, "rb"); 81 if(input!=NULL) { 82 return input; 83 } 84 } 85 86 // find icu/source/data/unidata relative to the test data 87 errorCode=U_ZERO_ERROR; 88 folder=loadTestData(errorCode); 89 if(U_SUCCESS(errorCode)) { 90 strcpy(unidataPath, folder); 91 strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." 92 U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." 93 U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING); 94 strcat(unidataPath, filename); 95 input=T_FileStream_open(unidataPath, "rb"); 96 if(input!=NULL) { 97 return input; 98 } 99 } 100 101 // look in icu/source/test/testdata/out/build 102 errorCode=U_ZERO_ERROR; 103 folder=loadTestData(errorCode); 104 if(U_SUCCESS(errorCode)) { 105 strcpy(unidataPath, folder); 106 strcat(unidataPath, U_FILE_SEP_STRING); 107 strcat(unidataPath, filename); 108 input=T_FileStream_open(unidataPath, "rb"); 109 if(input!=NULL) { 110 return input; 111 } 112 } 113 114 // look in icu/source/test/testdata 115 errorCode=U_ZERO_ERROR; 116 folder=loadTestData(errorCode); 117 if(U_SUCCESS(errorCode)) { 118 strcpy(unidataPath, folder); 119 strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING); 120 strcat(unidataPath, filename); 121 input=T_FileStream_open(unidataPath, "rb"); 122 if(input!=NULL) { 123 return input; 124 } 125 } 126 127 // find icu/source/data/unidata relative to U_TOPSRCDIR 128 #if defined(U_TOPSRCDIR) 129 strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING); 130 strcat(unidataPath, filename); 131 input=T_FileStream_open(unidataPath, "rb"); 132 if(input!=NULL) { 133 return input; 134 } 135 136 strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING); 137 strcat(unidataPath, filename); 138 input=T_FileStream_open(unidataPath, "rb"); 139 if(input!=NULL) { 140 return input; 141 } 142 #endif 143 144 dataerrln("Failed to open %s", filename); 145 return NULL; 146 } 147 148 /** 149 * Test the conformance of Normalizer to 150 * http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt 151 */ 152 void NormalizerConformanceTest::TestConformance() { 153 TestConformance(openNormalizationTestFile("NormalizationTest.txt"), 0); 154 } 155 156 void NormalizerConformanceTest::TestConformance32() { 157 TestConformance(openNormalizationTestFile("NormalizationTest-3.2.0.txt"), UNORM_UNICODE_3_2); 158 } 159 160 void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) { 161 enum { BUF_SIZE = 1024 }; 162 char lineBuf[BUF_SIZE]; 163 UnicodeString fields[FIELD_COUNT]; 164 UErrorCode status = U_ZERO_ERROR; 165 int32_t passCount = 0; 166 int32_t failCount = 0; 167 UChar32 c; 168 169 if(input==NULL) { 170 return; 171 } 172 173 // UnicodeSet for all code points that are not mentioned in NormalizationTest.txt 174 UnicodeSet other(0, 0x10ffff); 175 176 int32_t count, countMoreCases = sizeof(moreCases)/sizeof(moreCases[0]); 177 for (count = 1;;++count) { 178 if (!T_FileStream_eof(input)) { 179 T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf)); 180 } else { 181 // once NormalizationTest.txt is finished, use moreCases[] 182 if(count > countMoreCases) { 183 count = 0; 184 } else if(count == countMoreCases) { 185 // all done 186 break; 187 } 188 uprv_strcpy(lineBuf, moreCases[count]); 189 } 190 if (lineBuf[0] == 0 || lineBuf[0] == '\n' || lineBuf[0] == '\r') continue; 191 192 // Expect 5 columns of this format: 193 // 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments> 194 195 // Parse out the comment. 196 if (lineBuf[0] == '#') continue; 197 198 // Read separator lines starting with '@' 199 if (lineBuf[0] == '@') { 200 logln(lineBuf); 201 continue; 202 } 203 204 // Parse out the fields 205 if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) { 206 errln((UnicodeString)"Unable to parse line " + count); 207 break; // Syntax error 208 } 209 210 // Remove a single code point from the "other" UnicodeSet 211 if(fields[0].length()==fields[0].moveIndex32(0, 1)) { 212 c=fields[0].char32At(0); 213 if(0xac20<=c && c<=0xd73f && quick) { 214 // not an exhaustive test run: skip most Hangul syllables 215 if(c==0xac20) { 216 other.remove(0xac20, 0xd73f); 217 } 218 continue; 219 } 220 other.remove(c); 221 } 222 223 if (checkConformance(fields, lineBuf, options, status)) { 224 ++passCount; 225 } else { 226 ++failCount; 227 if(status == U_FILE_ACCESS_ERROR) { 228 errln("Something is wrong with the normalizer, skipping the rest of the test."); 229 break; 230 } 231 } 232 if ((count % 1000) == 0) { 233 logln("Line %d", count); 234 } 235 } 236 237 T_FileStream_close(input); 238 239 /* 240 * Test that all characters that are not mentioned 241 * as single code points in column 1 242 * do not change under any normalization. 243 */ 244 245 // remove U+ffff because that is the end-of-iteration sentinel value 246 other.remove(0xffff); 247 248 for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) { 249 if(0x30000<=c && c<0xe0000) { 250 c=0xe0000; 251 } 252 if(!other.contains(c)) { 253 continue; 254 } 255 256 fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c); 257 sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c); 258 259 if (checkConformance(fields, lineBuf, options, status)) { 260 ++passCount; 261 } else { 262 ++failCount; 263 if(status == U_FILE_ACCESS_ERROR) { 264 errln("Something is wrong with the normalizer, skipping the rest of the test."); 265 break; 266 } 267 } 268 if ((c % 0x1000) == 0) { 269 logln("Code point U+%04lx", c); 270 } 271 } 272 273 if (failCount != 0) { 274 errln((UnicodeString)"Total: " + failCount + " lines/code points failed, " + 275 passCount + " lines/code points passed"); 276 } else { 277 logln((UnicodeString)"Total: " + passCount + " lines/code points passed"); 278 } 279 } 280 281 /** 282 * Verify the conformance of the given line of the Unicode 283 * normalization (UTR 15) test suite file. For each line, 284 * there are five columns, corresponding to field[0]..field[4]. 285 * 286 * The following invariants must be true for all conformant implementations 287 * c2 == NFC(c1) == NFC(c2) == NFC(c3) 288 * c3 == NFD(c1) == NFD(c2) == NFD(c3) 289 * c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) 290 * c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) 291 * 292 * @param field the 5 columns 293 * @param line the source line from the test suite file 294 * @return true if the test passes 295 */ 296 UBool NormalizerConformanceTest::checkConformance(const UnicodeString* field, 297 const char *line, 298 int32_t options, 299 UErrorCode &status) { 300 UBool pass = TRUE, result; 301 //UErrorCode status = U_ZERO_ERROR; 302 UnicodeString out, fcd; 303 int32_t fieldNum; 304 305 for (int32_t i=0; i<FIELD_COUNT; ++i) { 306 fieldNum = i+1; 307 if (i<3) { 308 Normalizer::normalize(field[i], UNORM_NFC, options, out, status); 309 pass &= assertEqual("C", field[i], out, field[1], "c2!=C(c", fieldNum); 310 iterativeNorm(field[i], UNORM_NFC, options, out, +1); 311 pass &= assertEqual("C(+1)", field[i], out, field[1], "c2!=C(c", fieldNum); 312 iterativeNorm(field[i], UNORM_NFC, options, out, -1); 313 pass &= assertEqual("C(-1)", field[i], out, field[1], "c2!=C(c", fieldNum); 314 315 Normalizer::normalize(field[i], UNORM_NFD, options, out, status); 316 pass &= assertEqual("D", field[i], out, field[2], "c3!=D(c", fieldNum); 317 iterativeNorm(field[i], UNORM_NFD, options, out, +1); 318 pass &= assertEqual("D(+1)", field[i], out, field[2], "c3!=D(c", fieldNum); 319 iterativeNorm(field[i], UNORM_NFD, options, out, -1); 320 pass &= assertEqual("D(-1)", field[i], out, field[2], "c3!=D(c", fieldNum); 321 } 322 Normalizer::normalize(field[i], UNORM_NFKC, options, out, status); 323 pass &= assertEqual("KC", field[i], out, field[3], "c4!=KC(c", fieldNum); 324 iterativeNorm(field[i], UNORM_NFKC, options, out, +1); 325 pass &= assertEqual("KC(+1)", field[i], out, field[3], "c4!=KC(c", fieldNum); 326 iterativeNorm(field[i], UNORM_NFKC, options, out, -1); 327 pass &= assertEqual("KC(-1)", field[i], out, field[3], "c4!=KC(c", fieldNum); 328 329 Normalizer::normalize(field[i], UNORM_NFKD, options, out, status); 330 pass &= assertEqual("KD", field[i], out, field[4], "c5!=KD(c", fieldNum); 331 iterativeNorm(field[i], UNORM_NFKD, options, out, +1); 332 pass &= assertEqual("KD(+1)", field[i], out, field[4], "c5!=KD(c", fieldNum); 333 iterativeNorm(field[i], UNORM_NFKD, options, out, -1); 334 pass &= assertEqual("KD(-1)", field[i], out, field[4], "c5!=KD(c", fieldNum); 335 } 336 compare(field[1],field[2]); 337 compare(field[0],field[1]); 338 // test quick checks 339 if(UNORM_NO == Normalizer::quickCheck(field[1], UNORM_NFC, options, status)) { 340 errln("Normalizer error: quickCheck(NFC(s), UNORM_NFC) is UNORM_NO"); 341 pass = FALSE; 342 } 343 if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_NFD, options, status)) { 344 errln("Normalizer error: quickCheck(NFD(s), UNORM_NFD) is UNORM_NO"); 345 pass = FALSE; 346 } 347 if(UNORM_NO == Normalizer::quickCheck(field[3], UNORM_NFKC, options, status)) { 348 errln("Normalizer error: quickCheck(NFKC(s), UNORM_NFKC) is UNORM_NO"); 349 pass = FALSE; 350 } 351 if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_NFKD, options, status)) { 352 errln("Normalizer error: quickCheck(NFKD(s), UNORM_NFKD) is UNORM_NO"); 353 pass = FALSE; 354 } 355 356 // branch on options==0 for better code coverage 357 if(options==0) { 358 result = Normalizer::isNormalized(field[1], UNORM_NFC, status); 359 } else { 360 result = Normalizer::isNormalized(field[1], UNORM_NFC, options, status); 361 } 362 if(!result) { 363 errln("Normalizer error: isNormalized(NFC(s), UNORM_NFC) is FALSE"); 364 pass = FALSE; 365 } 366 if(field[0]!=field[1] && Normalizer::isNormalized(field[0], UNORM_NFC, options, status)) { 367 errln("Normalizer error: isNormalized(s, UNORM_NFC) is TRUE"); 368 pass = FALSE; 369 } 370 if(!Normalizer::isNormalized(field[3], UNORM_NFKC, options, status)) { 371 errln("Normalizer error: isNormalized(NFKC(s), UNORM_NFKC) is FALSE"); 372 pass = FALSE; 373 } 374 if(field[0]!=field[3] && Normalizer::isNormalized(field[0], UNORM_NFKC, options, status)) { 375 errln("Normalizer error: isNormalized(s, UNORM_NFKC) is TRUE"); 376 pass = FALSE; 377 } 378 379 // test FCD quick check and "makeFCD" 380 Normalizer::normalize(field[0], UNORM_FCD, options, fcd, status); 381 if(UNORM_NO == Normalizer::quickCheck(fcd, UNORM_FCD, options, status)) { 382 errln("Normalizer error: quickCheck(FCD(s), UNORM_FCD) is UNORM_NO"); 383 pass = FALSE; 384 } 385 if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_FCD, options, status)) { 386 errln("Normalizer error: quickCheck(NFD(s), UNORM_FCD) is UNORM_NO"); 387 pass = FALSE; 388 } 389 if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_FCD, options, status)) { 390 errln("Normalizer error: quickCheck(NFKD(s), UNORM_FCD) is UNORM_NO"); 391 pass = FALSE; 392 } 393 394 Normalizer::normalize(fcd, UNORM_NFD, options, out, status); 395 if(out != field[2]) { 396 errln("Normalizer error: NFD(FCD(s))!=NFD(s)"); 397 pass = FALSE; 398 } 399 400 if (U_FAILURE(status)) { 401 errln("Normalizer::normalize returned error status"); 402 pass = FALSE; 403 } 404 405 if(field[0]!=field[2]) { 406 // two strings that are canonically equivalent must test 407 // equal under a canonical caseless match 408 // see UAX #21 Case Mappings and Jitterbug 2021 and 409 // Unicode Technical Committee meeting consensus 92-C31 410 int32_t rc; 411 412 status=U_ZERO_ERROR; 413 rc=Normalizer::compare(field[0], field[2], (options<<UNORM_COMPARE_NORM_OPTIONS_SHIFT)|U_COMPARE_IGNORE_CASE, status); 414 if(U_FAILURE(status)) { 415 errln("Normalizer::compare(case-insensitive) sets %s", u_errorName(status)); 416 pass=FALSE; 417 } else if(rc!=0) { 418 errln("Normalizer::compare(original, NFD, case-insensitive) returned %d instead of 0 for equal", rc); 419 pass=FALSE; 420 } 421 } 422 423 if (!pass) { 424 errln("FAIL: %s", line); 425 } 426 return pass; 427 } 428 429 /** 430 * Do a normalization using the iterative API in the given direction. 431 * @param dir either +1 or -1 432 */ 433 void NormalizerConformanceTest::iterativeNorm(const UnicodeString& str, 434 UNormalizationMode mode, int32_t options, 435 UnicodeString& result, 436 int8_t dir) { 437 UErrorCode status = U_ZERO_ERROR; 438 normalizer.setText(str, status); 439 normalizer.setMode(mode); 440 normalizer.setOption(-1, 0); // reset all options 441 normalizer.setOption(options, 1); // set desired options 442 result.truncate(0); 443 if (U_FAILURE(status)) { 444 return; 445 } 446 UChar32 ch; 447 if (dir > 0) { 448 for (ch = normalizer.first(); ch != Normalizer::DONE; 449 ch = normalizer.next()) { 450 result.append(ch); 451 } 452 } else { 453 for (ch = normalizer.last(); ch != Normalizer::DONE; 454 ch = normalizer.previous()) { 455 result.insert(0, ch); 456 } 457 } 458 } 459 460 /** 461 * @param op name of normalization form, e.g., "KC" 462 * @param s string being normalized 463 * @param got value received 464 * @param exp expected value 465 * @param msg description of this test 466 * @param return true if got == exp 467 */ 468 UBool NormalizerConformanceTest::assertEqual(const char *op, 469 const UnicodeString& s, 470 const UnicodeString& got, 471 const UnicodeString& exp, 472 const char *msg, 473 int32_t field) 474 { 475 if (exp == got) 476 return TRUE; 477 478 char *sChars, *gotChars, *expChars; 479 UnicodeString sPretty(prettify(s)); 480 UnicodeString gotPretty(prettify(got)); 481 UnicodeString expPretty(prettify(exp)); 482 483 sChars = new char[sPretty.length() + 1]; 484 gotChars = new char[gotPretty.length() + 1]; 485 expChars = new char[expPretty.length() + 1]; 486 487 sPretty.extract(0, sPretty.length(), sChars, sPretty.length() + 1); 488 sChars[sPretty.length()] = 0; 489 gotPretty.extract(0, gotPretty.length(), gotChars, gotPretty.length() + 1); 490 gotChars[gotPretty.length()] = 0; 491 expPretty.extract(0, expPretty.length(), expChars, expPretty.length() + 1); 492 expChars[expPretty.length()] = 0; 493 494 errln(" %s%d)%s(%s)=%s, exp. %s", msg, field, op, sChars, gotChars, expChars); 495 496 delete []sChars; 497 delete []gotChars; 498 delete []expChars; 499 return FALSE; 500 } 501 502 /** 503 * Split a string into pieces based on the given delimiter 504 * character. Then, parse the resultant fields from hex into 505 * characters. That is, "0040 0400;0C00;0899" -> new String[] { 506 * "\u0040\u0400", "\u0C00", "\u0899" }. The output is assumed to 507 * be of the proper length already, and exactly output.length 508 * fields are parsed. If there are too few an exception is 509 * thrown. If there are too many the extras are ignored. 510 * 511 * @return FALSE upon failure 512 */ 513 UBool NormalizerConformanceTest::hexsplit(const char *s, char delimiter, 514 UnicodeString output[], int32_t outputLength) { 515 const char *t = s; 516 char *end = NULL; 517 UChar32 c; 518 int32_t i; 519 for (i=0; i<outputLength; ++i) { 520 // skip whitespace 521 while(*t == ' ' || *t == '\t') { 522 ++t; 523 } 524 525 // read a sequence of code points 526 output[i].remove(); 527 for(;;) { 528 c = (UChar32)uprv_strtoul(t, &end, 16); 529 530 if( (char *)t == end || 531 (uint32_t)c > 0x10ffff || 532 (*end != ' ' && *end != '\t' && *end != delimiter) 533 ) { 534 errln(UnicodeString("Bad field ", "") + (i + 1) + " in " + UnicodeString(s, "")); 535 return FALSE; 536 } 537 538 output[i].append(c); 539 540 t = (const char *)end; 541 542 // skip whitespace 543 while(*t == ' ' || *t == '\t') { 544 ++t; 545 } 546 547 if(*t == delimiter) { 548 ++t; 549 break; 550 } 551 if(*t == 0) { 552 if((i + 1) == outputLength) { 553 return TRUE; 554 } else { 555 errln(UnicodeString("Missing field(s) in ", "") + s + " only " + (i + 1) + " out of " + outputLength); 556 return FALSE; 557 } 558 } 559 } 560 } 561 return TRUE; 562 } 563 564 // Specific tests for debugging. These are generally failures taken from 565 // the conformance file, but culled out to make debugging easier. 566 567 void NormalizerConformanceTest::TestCase6(void) { 568 _testOneLine("0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301;"); 569 } 570 571 void NormalizerConformanceTest::_testOneLine(const char *line) { 572 UErrorCode status = U_ZERO_ERROR; 573 UnicodeString fields[FIELD_COUNT]; 574 if (!hexsplit(line, ';', fields, FIELD_COUNT)) { 575 errln((UnicodeString)"Unable to parse line " + line); 576 } else { 577 checkConformance(fields, line, 0, status); 578 } 579 } 580 581 #endif /* #if !UCONFIG_NO_NORMALIZATION */ 582