1 /* 2 ********************************************************************** 3 * Copyright (C) 1999-2012, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 * Date Name Description 7 * 12/09/99 aliu Ported from Java. 8 ********************************************************************** 9 */ 10 11 #include "unicode/utypes.h" 12 13 #if !UCONFIG_NO_COLLATION 14 15 #include "thcoll.h" 16 #include "unicode/utypes.h" 17 #include "unicode/coll.h" 18 #include "unicode/localpointer.h" 19 #include "unicode/sortkey.h" 20 #include "unicode/ustring.h" 21 #include "cstring.h" 22 #include "filestrm.h" 23 #include "textfile.h" 24 25 /** 26 * The TestDictionary test expects a file of this name, with this 27 * encoding, to be present in the directory $ICU/source/test/testdata. 28 */ 29 //#define TEST_FILE "th18057.txt" 30 31 /** 32 * This is the most failures we show in TestDictionary. If this number 33 * is < 0, we show all failures. 34 */ 35 #define MAX_FAILURES_TO_SHOW -1 36 37 CollationThaiTest::CollationThaiTest() { 38 UErrorCode status = U_ZERO_ERROR; 39 coll = Collator::createInstance(Locale("th", "TH", ""), status); 40 if (coll && U_SUCCESS(status)) { 41 //coll->setStrength(Collator::TERTIARY); 42 } else { 43 delete coll; 44 coll = 0; 45 } 46 } 47 48 CollationThaiTest::~CollationThaiTest() { 49 delete coll; 50 } 51 52 void CollationThaiTest::runIndexedTest(int32_t index, UBool exec, const char* &name, 53 char* /*par*/) { 54 55 if((!coll) && exec) { 56 dataerrln(__FILE__ " cannot test - failed to create collator."); 57 name = "some test"; 58 return; 59 } 60 61 switch (index) { 62 TESTCASE(0,TestDictionary); 63 TESTCASE(1,TestCornerCases); 64 TESTCASE(2,TestNamesList); 65 TESTCASE(3,TestInvalidThai); 66 TESTCASE(4,TestReordering); 67 default: name = ""; break; 68 } 69 } 70 71 /** 72 * Read the external names list, and confirms that the collator 73 * gets the same results when comparing lines one to another 74 * using regular and iterative comparison. 75 */ 76 void CollationThaiTest::TestNamesList(void) { 77 if (coll == 0) { 78 errln("Error: could not construct Thai collator"); 79 return; 80 } 81 82 UErrorCode ec = U_ZERO_ERROR; 83 TextFile names("TestNames_Thai.txt", "UTF16LE", ec); 84 if (U_FAILURE(ec)) { 85 logln("Can't open TestNames_Thai.txt: %s; skipping test", 86 u_errorName(ec)); 87 return; 88 } 89 90 // 91 // Loop through each word in the dictionary and compare it to the previous 92 // word. They should be in sorted order. 93 // 94 UnicodeString lastWord, word; 95 //int32_t failed = 0; 96 int32_t wordCount = 0; 97 while (names.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) { 98 99 // Show the first 8 words being compared, so we can see what's happening 100 ++wordCount; 101 if (wordCount <= 8) { 102 UnicodeString str; 103 logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str)); 104 } 105 106 if (lastWord.length() > 0) { 107 Collator::EComparisonResult result = coll->compare(lastWord, word); 108 doTest(coll, lastWord, word, result); 109 } 110 lastWord = word; 111 } 112 113 assertSuccess("readLine", ec); 114 115 logln((UnicodeString)"Words checked: " + wordCount); 116 } 117 118 /** 119 * Read the external dictionary file, which is already in proper 120 * sorted order, and confirm that the collator compares each line as 121 * preceding the following line. 122 */ 123 void CollationThaiTest::TestDictionary(void) { 124 if (coll == 0) { 125 errln("Error: could not construct Thai collator"); 126 return; 127 } 128 129 UErrorCode ec = U_ZERO_ERROR; 130 TextFile riwords("riwords.txt", "UTF8", ec); 131 if (U_FAILURE(ec)) { 132 logln("Can't open riwords.txt: %s; skipping test", 133 u_errorName(ec)); 134 return; 135 } 136 137 // 138 // Loop through each word in the dictionary and compare it to the previous 139 // word. They should be in sorted order. 140 // 141 UnicodeString lastWord, word; 142 int32_t failed = 0; 143 int32_t wordCount = 0; 144 while (riwords.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) { 145 146 // Show the first 8 words being compared, so we can see what's happening 147 ++wordCount; 148 if (wordCount <= 8) { 149 UnicodeString str; 150 logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str)); 151 } 152 153 if (lastWord.length() > 0) { 154 int32_t result = coll->compare(lastWord, word); 155 156 if (result > 0) { 157 failed++; 158 if (MAX_FAILURES_TO_SHOW < 0 || failed <= MAX_FAILURES_TO_SHOW) { 159 UnicodeString str; 160 UnicodeString msg = 161 UnicodeString("--------------------------------------------\n") 162 + riwords.getLineNumber() 163 + " compare(" + IntlTest::prettify(lastWord, str); 164 msg += UnicodeString(", ") 165 + IntlTest::prettify(word, str) + ") returned " + result 166 + ", expected -1\n"; 167 UErrorCode status = U_ZERO_ERROR; 168 CollationKey k1, k2; 169 coll->getCollationKey(lastWord, k1, status); 170 coll->getCollationKey(word, k2, status); 171 if (U_FAILURE(status)) { 172 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status)); 173 return; 174 } 175 msg.append("key1: ").append(prettify(k1, str)).append("\n"); 176 msg.append("key2: ").append(prettify(k2, str)); 177 errln(msg); 178 } 179 } 180 } 181 lastWord = word; 182 } 183 184 assertSuccess("readLine", ec); 185 186 if (failed != 0) { 187 if (failed > MAX_FAILURES_TO_SHOW) { 188 errln((UnicodeString)"Too many failures; only the first " + 189 MAX_FAILURES_TO_SHOW + " failures were shown"); 190 } 191 errln((UnicodeString)"Summary: " + failed + " of " + (riwords.getLineNumber() - 1) + 192 " comparisons failed"); 193 } 194 195 logln((UnicodeString)"Words checked: " + wordCount); 196 } 197 198 /** 199 * Odd corner conditions taken from "How to Sort Thai Without Rewriting Sort", 200 * by Doug Cooper, http://seasrc.th.net/paper/thaisort.zip 201 */ 202 void CollationThaiTest::TestCornerCases(void) { 203 const char* TESTS[] = { 204 // Shorter words precede longer 205 "\\u0e01", "<", "\\u0e01\\u0e01", 206 207 // Tone marks are considered after letters (i.e. are primary ignorable) 208 "\\u0e01\\u0e32", "<", "\\u0e01\\u0e49\\u0e32", 209 210 // ditto for other over-marks 211 "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e4c", 212 213 // commonly used mark-in-context order. 214 // In effect, marks are sorted after each syllable. 215 "\\u0e01\\u0e32\\u0e01\\u0e49\\u0e32", "<", "\\u0e01\\u0e48\\u0e32\\u0e01\\u0e49\\u0e32", 216 217 // Hyphens and other punctuation follow whitespace but come before letters 218 "\\u0e01\\u0e32", "=", "\\u0e01\\u0e32-", 219 "\\u0e01\\u0e32-", "<", "\\u0e01\\u0e32\\u0e01\\u0e32", 220 221 // Doubler follows an indentical word without the doubler 222 "\\u0e01\\u0e32", "=", "\\u0e01\\u0e32\\u0e46", 223 "\\u0e01\\u0e32\\u0e46", "<", "\\u0e01\\u0e32\\u0e01\\u0e32", 224 225 226 // \\u0e45 after either \\u0e24 or \\u0e26 is treated as a single 227 // combining character, similar to "c < ch" in traditional spanish. 228 // TODO: beef up this case 229 "\\u0e24\\u0e29\\u0e35", "<", "\\u0e24\\u0e45\\u0e29\\u0e35", 230 "\\u0e26\\u0e29\\u0e35", "<", "\\u0e26\\u0e45\\u0e29\\u0e35", 231 232 // Vowels reorder, should compare \\u0e2d and \\u0e34 233 "\\u0e40\\u0e01\\u0e2d", "<", "\\u0e40\\u0e01\\u0e34", 234 235 // Tones are compared after the rest of the word (e.g. primary ignorable) 236 "\\u0e01\\u0e32\\u0e01\\u0e48\\u0e32", "<", "\\u0e01\\u0e49\\u0e32\\u0e01\\u0e32", 237 238 // Periods are ignored entirely 239 "\\u0e01.\\u0e01.", "<", "\\u0e01\\u0e32", 240 }; 241 const int32_t TESTS_length = (int32_t)(sizeof(TESTS)/sizeof(TESTS[0])); 242 243 if (coll == 0) { 244 errln("Error: could not construct Thai collator"); 245 return; 246 } 247 compareArray(*coll, TESTS, TESTS_length); 248 } 249 250 //------------------------------------------------------------------------ 251 // Internal utilities 252 //------------------------------------------------------------------------ 253 254 void CollationThaiTest::compareArray(Collator& c, const char* tests[], 255 int32_t testsLength) { 256 for (int32_t i = 0; i < testsLength; i += 3) { 257 258 Collator::EComparisonResult expect; 259 if (tests[i+1][0] == '<') { 260 expect = Collator::LESS; 261 } else if (tests[i+1][0] == '>') { 262 expect = Collator::GREATER; 263 } else if (tests[i+1][0] == '=') { 264 expect = Collator::EQUAL; 265 } else { 266 // expect = Integer.decode(tests[i+1]).intValue(); 267 errln((UnicodeString)"Error: unknown operator " + tests[i+1]); 268 return; 269 } 270 271 UnicodeString s1, s2; 272 parseChars(s1, tests[i]); 273 parseChars(s2, tests[i+2]); 274 275 doTest(&c, s1, s2, expect); 276 #if 0 277 UErrorCode status = U_ZERO_ERROR; 278 int32_t result = c.compare(s1, s2); 279 if (sign(result) != sign(expect)) 280 { 281 UnicodeString t1, t2; 282 errln(UnicodeString("") + 283 i/3 + ": compare(" + IntlTest::prettify(s1, t1) 284 + " , " + IntlTest::prettify(s2, t2) 285 + ") got " + result + "; expected " + expect); 286 287 CollationKey k1, k2; 288 c.getCollationKey(s1, k1, status); 289 c.getCollationKey(s2, k2, status); 290 if (U_FAILURE(status)) { 291 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status)); 292 return; 293 } 294 errln((UnicodeString)" key1: " + prettify(k1, t1) ); 295 errln((UnicodeString)" key2: " + prettify(k2, t2) ); 296 } 297 else 298 { 299 // Collator.compare worked OK; now try the collation keys 300 CollationKey k1, k2; 301 c.getCollationKey(s1, k1, status); 302 c.getCollationKey(s2, k2, status); 303 if (U_FAILURE(status)) { 304 errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status)); 305 return; 306 } 307 308 result = k1.compareTo(k2); 309 if (sign(result) != sign(expect)) { 310 UnicodeString t1, t2; 311 errln(UnicodeString("") + 312 i/3 + ": key(" + IntlTest::prettify(s1, t1) 313 + ").compareTo(key(" + IntlTest::prettify(s2, t2) 314 + ")) got " + result + "; expected " + expect); 315 316 errln((UnicodeString)" " + prettify(k1, t1) + " vs. " + prettify(k2, t2)); 317 } 318 } 319 #endif 320 } 321 } 322 323 int8_t CollationThaiTest::sign(int32_t i) { 324 if (i < 0) return -1; 325 if (i > 0) return 1; 326 return 0; 327 } 328 329 /** 330 * Set a UnicodeString corresponding to the given string. Use 331 * UnicodeString and the default converter, unless we see the sequence 332 * "\\u", in which case we interpret the subsequent escape. 333 */ 334 UnicodeString& CollationThaiTest::parseChars(UnicodeString& result, 335 const char* chars) { 336 return result = CharsToUnicodeString(chars); 337 } 338 339 UCollator *thaiColl = NULL; 340 341 U_CDECL_BEGIN 342 static int U_CALLCONV 343 StrCmp(const void *p1, const void *p2) { 344 return ucol_strcoll(thaiColl, *(UChar **) p1, -1, *(UChar **)p2, -1); 345 } 346 U_CDECL_END 347 348 349 #define LINES 6 350 351 void CollationThaiTest::TestInvalidThai(void) { 352 const char *tests[LINES] = { 353 "\\u0E44\\u0E01\\u0E44\\u0E01", 354 "\\u0E44\\u0E01\\u0E01\\u0E44", 355 "\\u0E01\\u0E44\\u0E01\\u0E44", 356 "\\u0E01\\u0E01\\u0E44\\u0E44", 357 "\\u0E44\\u0E44\\u0E01\\u0E01", 358 "\\u0E01\\u0E44\\u0E44\\u0E01", 359 }; 360 361 UChar strings[LINES][20]; 362 363 UChar *toSort[LINES]; 364 365 int32_t i = 0, j = 0, len = 0; 366 367 UErrorCode coll_status = U_ZERO_ERROR; 368 UnicodeString iteratorText; 369 370 thaiColl = ucol_open ("th_TH", &coll_status); 371 if (U_FAILURE(coll_status)) { 372 errln("Error opening Thai collator: %s", u_errorName(coll_status)); 373 return; 374 } 375 376 CollationElementIterator* c = ((RuleBasedCollator *)coll)->createCollationElementIterator( iteratorText ); 377 378 for(i = 0; i < (int32_t)(sizeof(tests)/sizeof(tests[0])); i++) { 379 len = u_unescape(tests[i], strings[i], 20); 380 strings[i][len] = 0; 381 toSort[i] = strings[i]; 382 } 383 384 qsort (toSort, LINES, sizeof (UChar *), StrCmp); 385 386 for (i=0; i < LINES; i++) 387 { 388 logln("%i", i); 389 for (j=i+1; j < LINES; j++) { 390 if (ucol_strcoll (thaiColl, toSort[i], -1, toSort[j], -1) == UCOL_GREATER) 391 { 392 // inconsistency ordering found! 393 errln("Inconsistent ordering between strings %i and %i", i, j); 394 } 395 } 396 iteratorText.setTo(toSort[i]); 397 c->setText(iteratorText, coll_status); 398 backAndForth(*c); 399 } 400 401 402 ucol_close(thaiColl); 403 delete c; 404 } 405 406 void CollationThaiTest::TestReordering(void) { 407 // Until UCA 4.1, the collation code swapped Thai/Lao prevowels with the following consonants, 408 // resulting in consonant+prevowel == prevowel+consonant. 409 // From UCA 5.0 on, there are order-reversing contractions for prevowel+consonant. 410 // From UCA 5.0 until UCA 6.1, there was a tertiary difference between 411 // consonant+prevowel and prevowel+consonant. 412 // In UCA 6.2, they compare equal again. 413 // The test was modified to using a collator with strength=secondary, 414 // ignoring possible tertiary differences. 415 const char *tests[] = { 416 "\\u0E41c\\u0301", "=", "\\u0E41\\u0107", // composition 417 "\\u0E41\\U0001D7CE", "<", "\\u0E41\\U0001D7CF", // supplementaries 418 "\\u0E41\\U0001D15F", "=", "\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary 419 "\\u0E41\\U0002F802", "=", "\\u0E41\\u4E41", // supplementary composition decomps to BMP 420 "\\u0E41\\u0301", "=", "\\u0E41\\u0301", // unsafe (just checking backwards iteration) 421 "\\u0E41\\u0301\\u0316", "=", "\\u0E41\\u0316\\u0301", 422 423 "\\u0e24\\u0e41", "=", "\\u0e41\\u0e24", // exiting contraction bug 424 "\\u0e3f\\u0e3f\\u0e24\\u0e41", "=", "\\u0e3f\\u0e3f\\u0e41\\u0e24", 425 426 "abc\\u0E41c\\u0301", "=", "abc\\u0E41\\u0107", // composition 427 "abc\\u0E41\\U0001D000", "<", "abc\\u0E41\\U0001D001", // supplementaries 428 "abc\\u0E41\\U0001D15F", "=", "abc\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary 429 "abc\\u0E41\\U0002F802", "=", "abc\\u0E41\\u4E41", // supplementary composition decomps to BMP 430 "abc\\u0E41\\u0301", "=", "abc\\u0E41\\u0301", // unsafe (just checking backwards iteration) 431 "abc\\u0E41\\u0301\\u0316", "=", "abc\\u0E41\\u0316\\u0301", 432 433 "\\u0E41c\\u0301abc", "=", "\\u0E41\\u0107abc", // composition 434 "\\u0E41\\U0001D000abc", "<", "\\u0E41\\U0001D001abc", // supplementaries 435 "\\u0E41\\U0001D15Fabc", "=", "\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary 436 "\\u0E41\\U0002F802abc", "=", "\\u0E41\\u4E41abc", // supplementary composition decomps to BMP 437 "\\u0E41\\u0301abc", "=", "\\u0E41\\u0301abc", // unsafe (just checking backwards iteration) 438 "\\u0E41\\u0301\\u0316abc", "=", "\\u0E41\\u0316\\u0301abc", 439 440 "abc\\u0E41c\\u0301abc", "=", "abc\\u0E41\\u0107abc", // composition 441 "abc\\u0E41\\U0001D000abc", "<", "abc\\u0E41\\U0001D001abc", // supplementaries 442 "abc\\u0E41\\U0001D15Fabc", "=", "abc\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary 443 "abc\\u0E41\\U0002F802abc", "=", "abc\\u0E41\\u4E41abc", // supplementary composition decomps to BMP 444 "abc\\u0E41\\u0301abc", "=", "abc\\u0E41\\u0301abc", // unsafe (just checking backwards iteration) 445 "abc\\u0E41\\u0301\\u0316abc", "=", "abc\\u0E41\\u0316\\u0301abc", 446 }; 447 448 LocalPointer<Collator> coll2(coll->clone()); 449 UErrorCode status = U_ZERO_ERROR; 450 coll2->setAttribute(UCOL_STRENGTH, UCOL_SECONDARY, status); 451 if(U_FAILURE(status)) { 452 errln("Unable to set the Thai collator clone to secondary strength"); 453 return; 454 } 455 compareArray(*coll2, tests, sizeof(tests)/sizeof(tests[0])); 456 457 const char *rule = "& c < ab"; 458 const char *testcontraction[] = { "\\u0E41ab", ">", "\\u0E41c"}; // After UCA 4.1 Thai are normal so won't break a contraction 459 UnicodeString rules; 460 parseChars(rules, rule); 461 LocalPointer<RuleBasedCollator> rcoll(new RuleBasedCollator(rules, status)); 462 if(U_SUCCESS(status)) { 463 compareArray(*rcoll, testcontraction, 3); 464 } else { 465 errln("Couldn't instantiate collator from rules"); 466 } 467 468 } 469 470 471 #endif /* #if !UCONFIG_NO_COLLATION */ 472
