1 /******************************************************************** 2 * COPYRIGHT: 3 * Copyright (c) 1997-2011, International Business Machines Corporation and 4 * others. All Rights Reserved. 5 ********************************************************************/ 6 /******************************************************************************** 7 * 8 * File CITERTST.C 9 * 10 * Modification History: 11 * Date Name Description 12 * Madhu Katragadda Ported for C API 13 * 02/19/01 synwee Modified test case for new collation iterator 14 *********************************************************************************/ 15 /* 16 * Collation Iterator tests. 17 * (Let me reiterate my position...) 18 */ 19 20 #include "unicode/utypes.h" 21 22 #if !UCONFIG_NO_COLLATION 23 24 #include "unicode/ucol.h" 25 #include "unicode/ucoleitr.h" 26 #include "unicode/uloc.h" 27 #include "unicode/uchar.h" 28 #include "unicode/ustring.h" 29 #include "unicode/putil.h" 30 #include "callcoll.h" 31 #include "cmemory.h" 32 #include "cintltst.h" 33 #include "citertst.h" 34 #include "ccolltst.h" 35 #include "filestrm.h" 36 #include "cstring.h" 37 #include "ucol_imp.h" 38 #include "ucol_tok.h" 39 #include "uparse.h" 40 #include <stdio.h> 41 42 extern uint8_t ucol_uprv_getCaseBits(const UChar *, uint32_t, UErrorCode *); 43 44 void addCollIterTest(TestNode** root) 45 { 46 addTest(root, &TestPrevious, "tscoll/citertst/TestPrevious"); 47 addTest(root, &TestOffset, "tscoll/citertst/TestOffset"); 48 addTest(root, &TestSetText, "tscoll/citertst/TestSetText"); 49 addTest(root, &TestMaxExpansion, "tscoll/citertst/TestMaxExpansion"); 50 addTest(root, &TestUnicodeChar, "tscoll/citertst/TestUnicodeChar"); 51 addTest(root, &TestNormalizedUnicodeChar, 52 "tscoll/citertst/TestNormalizedUnicodeChar"); 53 addTest(root, &TestNormalization, "tscoll/citertst/TestNormalization"); 54 addTest(root, &TestBug672, "tscoll/citertst/TestBug672"); 55 addTest(root, &TestBug672Normalize, "tscoll/citertst/TestBug672Normalize"); 56 addTest(root, &TestSmallBuffer, "tscoll/citertst/TestSmallBuffer"); 57 addTest(root, &TestCEs, "tscoll/citertst/TestCEs"); 58 addTest(root, &TestDiscontiguos, "tscoll/citertst/TestDiscontiguos"); 59 addTest(root, &TestCEBufferOverflow, "tscoll/citertst/TestCEBufferOverflow"); 60 addTest(root, &TestCEValidity, "tscoll/citertst/TestCEValidity"); 61 addTest(root, &TestSortKeyValidity, "tscoll/citertst/TestSortKeyValidity"); 62 addTest(root, &TestSearchCollatorElements, "tscoll/citertst/TestSearchCollatorElements"); 63 } 64 65 /* The locales we support */ 66 67 static const char * LOCALES[] = {"en_AU", "en_BE", "en_CA"}; 68 69 static void TestBug672() { 70 UErrorCode status = U_ZERO_ERROR; 71 UChar pattern[20]; 72 UChar text[50]; 73 int i; 74 int result[3][3]; 75 76 u_uastrcpy(pattern, "resume"); 77 u_uastrcpy(text, "Time to resume updating my resume."); 78 79 for (i = 0; i < 3; ++ i) { 80 UCollator *coll = ucol_open(LOCALES[i], &status); 81 UCollationElements *pitr = ucol_openElements(coll, pattern, -1, 82 &status); 83 UCollationElements *titer = ucol_openElements(coll, text, -1, 84 &status); 85 if (U_FAILURE(status)) { 86 log_err_status(status, "ERROR: in creation of either the collator or the collation iterator :%s\n", 87 myErrorName(status)); 88 return; 89 } 90 91 log_verbose("locale tested %s\n", LOCALES[i]); 92 93 while (ucol_next(pitr, &status) != UCOL_NULLORDER && 94 U_SUCCESS(status)) { 95 } 96 if (U_FAILURE(status)) { 97 log_err("ERROR: reversing collation iterator :%s\n", 98 myErrorName(status)); 99 return; 100 } 101 ucol_reset(pitr); 102 103 ucol_setOffset(titer, u_strlen(pattern), &status); 104 if (U_FAILURE(status)) { 105 log_err("ERROR: setting offset in collator :%s\n", 106 myErrorName(status)); 107 return; 108 } 109 result[i][0] = ucol_getOffset(titer); 110 log_verbose("Text iterator set to offset %d\n", result[i][0]); 111 112 /* Use previous() */ 113 ucol_previous(titer, &status); 114 result[i][1] = ucol_getOffset(titer); 115 log_verbose("Current offset %d after previous\n", result[i][1]); 116 117 /* Add one to index */ 118 log_verbose("Adding one to current offset...\n"); 119 ucol_setOffset(titer, ucol_getOffset(titer) + 1, &status); 120 if (U_FAILURE(status)) { 121 log_err("ERROR: setting offset in collator :%s\n", 122 myErrorName(status)); 123 return; 124 } 125 result[i][2] = ucol_getOffset(titer); 126 log_verbose("Current offset in text = %d\n", result[i][2]); 127 ucol_closeElements(pitr); 128 ucol_closeElements(titer); 129 ucol_close(coll); 130 } 131 132 if (uprv_memcmp(result[0], result[1], 3) != 0 || 133 uprv_memcmp(result[1], result[2], 3) != 0) { 134 log_err("ERROR: Different locales have different offsets at the same character\n"); 135 } 136 } 137 138 139 140 /* Running this test with normalization enabled showed up a bug in the incremental 141 normalization code. */ 142 static void TestBug672Normalize() { 143 UErrorCode status = U_ZERO_ERROR; 144 UChar pattern[20]; 145 UChar text[50]; 146 int i; 147 int result[3][3]; 148 149 u_uastrcpy(pattern, "resume"); 150 u_uastrcpy(text, "Time to resume updating my resume."); 151 152 for (i = 0; i < 3; ++ i) { 153 UCollator *coll = ucol_open(LOCALES[i], &status); 154 UCollationElements *pitr = NULL; 155 UCollationElements *titer = NULL; 156 157 ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); 158 159 pitr = ucol_openElements(coll, pattern, -1, &status); 160 titer = ucol_openElements(coll, text, -1, &status); 161 if (U_FAILURE(status)) { 162 log_err_status(status, "ERROR: in creation of either the collator or the collation iterator :%s\n", 163 myErrorName(status)); 164 return; 165 } 166 167 log_verbose("locale tested %s\n", LOCALES[i]); 168 169 while (ucol_next(pitr, &status) != UCOL_NULLORDER && 170 U_SUCCESS(status)) { 171 } 172 if (U_FAILURE(status)) { 173 log_err("ERROR: reversing collation iterator :%s\n", 174 myErrorName(status)); 175 return; 176 } 177 ucol_reset(pitr); 178 179 ucol_setOffset(titer, u_strlen(pattern), &status); 180 if (U_FAILURE(status)) { 181 log_err("ERROR: setting offset in collator :%s\n", 182 myErrorName(status)); 183 return; 184 } 185 result[i][0] = ucol_getOffset(titer); 186 log_verbose("Text iterator set to offset %d\n", result[i][0]); 187 188 /* Use previous() */ 189 ucol_previous(titer, &status); 190 result[i][1] = ucol_getOffset(titer); 191 log_verbose("Current offset %d after previous\n", result[i][1]); 192 193 /* Add one to index */ 194 log_verbose("Adding one to current offset...\n"); 195 ucol_setOffset(titer, ucol_getOffset(titer) + 1, &status); 196 if (U_FAILURE(status)) { 197 log_err("ERROR: setting offset in collator :%s\n", 198 myErrorName(status)); 199 return; 200 } 201 result[i][2] = ucol_getOffset(titer); 202 log_verbose("Current offset in text = %d\n", result[i][2]); 203 ucol_closeElements(pitr); 204 ucol_closeElements(titer); 205 ucol_close(coll); 206 } 207 208 if (uprv_memcmp(result[0], result[1], 3) != 0 || 209 uprv_memcmp(result[1], result[2], 3) != 0) { 210 log_err("ERROR: Different locales have different offsets at the same character\n"); 211 } 212 } 213 214 215 216 217 /** 218 * Test for CollationElementIterator previous and next for the whole set of 219 * unicode characters. 220 */ 221 static void TestUnicodeChar() 222 { 223 UChar source[0x100]; 224 UCollator *en_us; 225 UCollationElements *iter; 226 UErrorCode status = U_ZERO_ERROR; 227 UChar codepoint; 228 229 UChar *test; 230 en_us = ucol_open("en_US", &status); 231 if (U_FAILURE(status)){ 232 log_err_status(status, "ERROR: in creation of collation data using ucol_open()\n %s\n", 233 myErrorName(status)); 234 return; 235 } 236 237 for (codepoint = 1; codepoint < 0xFFFE;) 238 { 239 test = source; 240 241 while (codepoint % 0xFF != 0) 242 { 243 if (u_isdefined(codepoint)) 244 *(test ++) = codepoint; 245 codepoint ++; 246 } 247 248 if (u_isdefined(codepoint)) 249 *(test ++) = codepoint; 250 251 if (codepoint != 0xFFFF) 252 codepoint ++; 253 254 *test = 0; 255 iter=ucol_openElements(en_us, source, u_strlen(source), &status); 256 if(U_FAILURE(status)){ 257 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 258 myErrorName(status)); 259 ucol_close(en_us); 260 return; 261 } 262 /* A basic test to see if it's working at all */ 263 log_verbose("codepoint testing %x\n", codepoint); 264 backAndForth(iter); 265 ucol_closeElements(iter); 266 267 /* null termination test */ 268 iter=ucol_openElements(en_us, source, -1, &status); 269 if(U_FAILURE(status)){ 270 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 271 myErrorName(status)); 272 ucol_close(en_us); 273 return; 274 } 275 /* A basic test to see if it's working at all */ 276 backAndForth(iter); 277 ucol_closeElements(iter); 278 } 279 280 ucol_close(en_us); 281 } 282 283 /** 284 * Test for CollationElementIterator previous and next for the whole set of 285 * unicode characters with normalization on. 286 */ 287 static void TestNormalizedUnicodeChar() 288 { 289 UChar source[0x100]; 290 UCollator *th_th; 291 UCollationElements *iter; 292 UErrorCode status = U_ZERO_ERROR; 293 UChar codepoint; 294 295 UChar *test; 296 /* thai should have normalization on */ 297 th_th = ucol_open("th_TH", &status); 298 if (U_FAILURE(status)){ 299 log_err_status(status, "ERROR: in creation of thai collation using ucol_open()\n %s\n", 300 myErrorName(status)); 301 return; 302 } 303 304 for (codepoint = 1; codepoint < 0xFFFE;) 305 { 306 test = source; 307 308 while (codepoint % 0xFF != 0) 309 { 310 if (u_isdefined(codepoint)) 311 *(test ++) = codepoint; 312 codepoint ++; 313 } 314 315 if (u_isdefined(codepoint)) 316 *(test ++) = codepoint; 317 318 if (codepoint != 0xFFFF) 319 codepoint ++; 320 321 *test = 0; 322 iter=ucol_openElements(th_th, source, u_strlen(source), &status); 323 if(U_FAILURE(status)){ 324 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 325 myErrorName(status)); 326 ucol_close(th_th); 327 return; 328 } 329 330 backAndForth(iter); 331 ucol_closeElements(iter); 332 333 iter=ucol_openElements(th_th, source, -1, &status); 334 if(U_FAILURE(status)){ 335 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 336 myErrorName(status)); 337 ucol_close(th_th); 338 return; 339 } 340 341 backAndForth(iter); 342 ucol_closeElements(iter); 343 } 344 345 ucol_close(th_th); 346 } 347 348 /** 349 * Test the incremental normalization 350 */ 351 static void TestNormalization() 352 { 353 UErrorCode status = U_ZERO_ERROR; 354 const char *str = 355 "&a < \\u0300\\u0315 < A\\u0300\\u0315 < \\u0316\\u0315B < \\u0316\\u0300\\u0315"; 356 UCollator *coll; 357 UChar rule[50]; 358 int rulelen = u_unescape(str, rule, 50); 359 int count = 0; 360 const char *testdata[] = 361 {"\\u1ED9", "o\\u0323\\u0302", 362 "\\u0300\\u0315", "\\u0315\\u0300", 363 "A\\u0300\\u0315B", "A\\u0315\\u0300B", 364 "A\\u0316\\u0315B", "A\\u0315\\u0316B", 365 "\\u0316\\u0300\\u0315", "\\u0315\\u0300\\u0316", 366 "A\\u0316\\u0300\\u0315B", "A\\u0315\\u0300\\u0316B", 367 "\\u0316\\u0315\\u0300", "A\\u0316\\u0315\\u0300B"}; 368 int32_t srclen; 369 UChar source[10]; 370 UCollationElements *iter; 371 372 coll = ucol_openRules(rule, rulelen, UCOL_ON, UCOL_TERTIARY, NULL, &status); 373 ucol_setAttribute(coll, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); 374 if (U_FAILURE(status)){ 375 log_err_status(status, "ERROR: in creation of collator using ucol_openRules()\n %s\n", 376 myErrorName(status)); 377 return; 378 } 379 380 srclen = u_unescape(testdata[0], source, 10); 381 iter = ucol_openElements(coll, source, srclen, &status); 382 backAndForth(iter); 383 ucol_closeElements(iter); 384 385 srclen = u_unescape(testdata[1], source, 10); 386 iter = ucol_openElements(coll, source, srclen, &status); 387 backAndForth(iter); 388 ucol_closeElements(iter); 389 390 while (count < 12) { 391 srclen = u_unescape(testdata[count], source, 10); 392 iter = ucol_openElements(coll, source, srclen, &status); 393 394 if (U_FAILURE(status)){ 395 log_err("ERROR: in creation of collator element iterator\n %s\n", 396 myErrorName(status)); 397 return; 398 } 399 backAndForth(iter); 400 ucol_closeElements(iter); 401 402 iter = ucol_openElements(coll, source, -1, &status); 403 404 if (U_FAILURE(status)){ 405 log_err("ERROR: in creation of collator element iterator\n %s\n", 406 myErrorName(status)); 407 return; 408 } 409 backAndForth(iter); 410 ucol_closeElements(iter); 411 count ++; 412 } 413 ucol_close(coll); 414 } 415 416 /** 417 * Test for CollationElementIterator.previous() 418 * 419 * @bug 4108758 - Make sure it works with contracting characters 420 * 421 */ 422 static void TestPrevious() 423 { 424 UCollator *coll=NULL; 425 UChar rule[50]; 426 UChar *source; 427 UCollator *c1, *c2, *c3; 428 UCollationElements *iter; 429 UErrorCode status = U_ZERO_ERROR; 430 UChar test1[50]; 431 UChar test2[50]; 432 433 u_uastrcpy(test1, "What subset of all possible test cases?"); 434 u_uastrcpy(test2, "has the highest probability of detecting"); 435 coll = ucol_open("en_US", &status); 436 437 iter=ucol_openElements(coll, test1, u_strlen(test1), &status); 438 log_verbose("English locale testing back and forth\n"); 439 if(U_FAILURE(status)){ 440 log_err_status(status, "ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 441 myErrorName(status)); 442 ucol_close(coll); 443 return; 444 } 445 /* A basic test to see if it's working at all */ 446 backAndForth(iter); 447 ucol_closeElements(iter); 448 ucol_close(coll); 449 450 /* Test with a contracting character sequence */ 451 u_uastrcpy(rule, "&a,A < b,B < c,C, d,D < z,Z < ch,cH,Ch,CH"); 452 c1 = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL, &status); 453 454 log_verbose("Contraction rule testing back and forth with no normalization\n"); 455 456 if (c1 == NULL || U_FAILURE(status)) 457 { 458 log_err("Couldn't create a RuleBasedCollator with a contracting sequence\n %s\n", 459 myErrorName(status)); 460 return; 461 } 462 source=(UChar*)malloc(sizeof(UChar) * 20); 463 u_uastrcpy(source, "abchdcba"); 464 iter=ucol_openElements(c1, source, u_strlen(source), &status); 465 if(U_FAILURE(status)){ 466 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 467 myErrorName(status)); 468 return; 469 } 470 backAndForth(iter); 471 ucol_closeElements(iter); 472 ucol_close(c1); 473 474 /* Test with an expanding character sequence */ 475 u_uastrcpy(rule, "&a < b < c/abd < d"); 476 c2 = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL, &status); 477 log_verbose("Expansion rule testing back and forth with no normalization\n"); 478 if (c2 == NULL || U_FAILURE(status)) 479 { 480 log_err("Couldn't create a RuleBasedCollator with a contracting sequence.\n %s\n", 481 myErrorName(status)); 482 return; 483 } 484 u_uastrcpy(source, "abcd"); 485 iter=ucol_openElements(c2, source, u_strlen(source), &status); 486 if(U_FAILURE(status)){ 487 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 488 myErrorName(status)); 489 return; 490 } 491 backAndForth(iter); 492 ucol_closeElements(iter); 493 ucol_close(c2); 494 /* Now try both */ 495 u_uastrcpy(rule, "&a < b < c/aba < d < z < ch"); 496 c3 = ucol_openRules(rule, u_strlen(rule), UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH,NULL, &status); 497 log_verbose("Expansion/contraction rule testing back and forth with no normalization\n"); 498 499 if (c3 == NULL || U_FAILURE(status)) 500 { 501 log_err("Couldn't create a RuleBasedCollator with a contracting sequence.\n %s\n", 502 myErrorName(status)); 503 return; 504 } 505 u_uastrcpy(source, "abcdbchdc"); 506 iter=ucol_openElements(c3, source, u_strlen(source), &status); 507 if(U_FAILURE(status)){ 508 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 509 myErrorName(status)); 510 return; 511 } 512 backAndForth(iter); 513 ucol_closeElements(iter); 514 ucol_close(c3); 515 source[0] = 0x0e41; 516 source[1] = 0x0e02; 517 source[2] = 0x0e41; 518 source[3] = 0x0e02; 519 source[4] = 0x0e27; 520 source[5] = 0x61; 521 source[6] = 0x62; 522 source[7] = 0x63; 523 source[8] = 0; 524 525 coll = ucol_open("th_TH", &status); 526 log_verbose("Thai locale testing back and forth with normalization\n"); 527 iter=ucol_openElements(coll, source, u_strlen(source), &status); 528 if(U_FAILURE(status)){ 529 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 530 myErrorName(status)); 531 return; 532 } 533 backAndForth(iter); 534 ucol_closeElements(iter); 535 ucol_close(coll); 536 537 /* prev test */ 538 source[0] = 0x0061; 539 source[1] = 0x30CF; 540 source[2] = 0x3099; 541 source[3] = 0x30FC; 542 source[4] = 0; 543 544 coll = ucol_open("ja_JP", &status); 545 log_verbose("Japanese locale testing back and forth with normalization\n"); 546 iter=ucol_openElements(coll, source, u_strlen(source), &status); 547 if(U_FAILURE(status)){ 548 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 549 myErrorName(status)); 550 return; 551 } 552 backAndForth(iter); 553 ucol_closeElements(iter); 554 ucol_close(coll); 555 556 free(source); 557 } 558 559 /** 560 * Test for getOffset() and setOffset() 561 */ 562 static void TestOffset() 563 { 564 UErrorCode status= U_ZERO_ERROR; 565 UCollator *en_us=NULL; 566 UCollationElements *iter, *pristine; 567 int32_t offset; 568 OrderAndOffset *orders; 569 int32_t orderLength=0; 570 int count = 0; 571 UChar test1[50]; 572 UChar test2[50]; 573 574 u_uastrcpy(test1, "What subset of all possible test cases?"); 575 u_uastrcpy(test2, "has the highest probability of detecting"); 576 en_us = ucol_open("en_US", &status); 577 log_verbose("Testing getOffset and setOffset for collations\n"); 578 iter = ucol_openElements(en_us, test1, u_strlen(test1), &status); 579 if(U_FAILURE(status)){ 580 log_err_status(status, "ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 581 myErrorName(status)); 582 ucol_close(en_us); 583 return; 584 } 585 586 /* testing boundaries */ 587 ucol_setOffset(iter, 0, &status); 588 if (U_FAILURE(status) || ucol_previous(iter, &status) != UCOL_NULLORDER) { 589 log_err("Error: After setting offset to 0, we should be at the end " 590 "of the backwards iteration"); 591 } 592 ucol_setOffset(iter, u_strlen(test1), &status); 593 if (U_FAILURE(status) || ucol_next(iter, &status) != UCOL_NULLORDER) { 594 log_err("Error: After setting offset to end of the string, we should " 595 "be at the end of the backwards iteration"); 596 } 597 598 /* Run all the way through the iterator, then get the offset */ 599 600 orders = getOrders(iter, &orderLength); 601 602 offset = ucol_getOffset(iter); 603 604 if (offset != u_strlen(test1)) 605 { 606 log_err("offset at end != length %d vs %d\n", offset, 607 u_strlen(test1) ); 608 } 609 610 /* Now set the offset back to the beginning and see if it works */ 611 pristine=ucol_openElements(en_us, test1, u_strlen(test1), &status); 612 if(U_FAILURE(status)){ 613 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 614 myErrorName(status)); 615 ucol_close(en_us); 616 return; 617 } 618 status = U_ZERO_ERROR; 619 620 ucol_setOffset(iter, 0, &status); 621 if (U_FAILURE(status)) 622 { 623 log_err("setOffset failed. %s\n", myErrorName(status)); 624 } 625 else 626 { 627 assertEqual(iter, pristine); 628 } 629 630 ucol_closeElements(pristine); 631 ucol_closeElements(iter); 632 free(orders); 633 634 /* testing offsets in normalization buffer */ 635 test1[0] = 0x61; 636 test1[1] = 0x300; 637 test1[2] = 0x316; 638 test1[3] = 0x62; 639 test1[4] = 0; 640 ucol_setAttribute(en_us, UCOL_NORMALIZATION_MODE, UCOL_ON, &status); 641 iter = ucol_openElements(en_us, test1, 4, &status); 642 if(U_FAILURE(status)){ 643 log_err("ERROR: in creation of collation element iterator using ucol_openElements()\n %s\n", 644 myErrorName(status)); 645 ucol_close(en_us); 646 return; 647 } 648 649 count = 0; 650 while (ucol_next(iter, &status) != UCOL_NULLORDER && 651 U_SUCCESS(status)) { 652 switch (count) { 653 case 0: 654 if (ucol_getOffset(iter) != 1) { 655 log_err("ERROR: Offset of iteration should be 1\n"); 656 } 657 break; 658 case 3: 659 if (ucol_getOffset(iter) != 4) { 660 log_err("ERROR: Offset of iteration should be 4\n"); 661 } 662 break; 663 default: 664 if (ucol_getOffset(iter) != 3) { 665 log_err("ERROR: Offset of iteration should be 3\n"); 666 } 667 } 668 count ++; 669 } 670 671 ucol_reset(iter); 672 count = 0; 673 while (ucol_previous(iter, &status) != UCOL_NULLORDER && 674 U_SUCCESS(status)) { 675 switch (count) { 676 case 0: 677 case 1: 678 if (ucol_getOffset(iter) != 3) { 679 log_err("ERROR: Offset of iteration should be 3\n"); 680 } 681 break; 682 case 2: 683 if (ucol_getOffset(iter) != 1) { 684 log_err("ERROR: Offset of iteration should be 1\n"); 685 } 686 break; 687 default: 688 if (ucol_getOffset(iter) != 0) { 689 log_err("ERROR: Offset of iteration should be 0\n"); 690 } 691 } 692 count ++; 693 } 694 695 if(U_FAILURE(status)){ 696 log_err("ERROR: in iterating collation elements %s\n", 697 myErrorName(status)); 698 } 699 700 ucol_closeElements(iter); 701 ucol_close(en_us); 702 } 703 704 /** 705 * Test for setText() 706 */ 707 static void TestSetText() 708 { 709 int32_t c,i; 710 UErrorCode status = U_ZERO_ERROR; 711 UCollator *en_us=NULL; 712 UCollationElements *iter1, *iter2; 713 UChar test1[50]; 714 UChar test2[50]; 715 716 u_uastrcpy(test1, "What subset of all possible test cases?"); 717 u_uastrcpy(test2, "has the highest probability of detecting"); 718 en_us = ucol_open("en_US", &status); 719 log_verbose("testing setText for Collation elements\n"); 720 iter1=ucol_openElements(en_us, test1, u_strlen(test1), &status); 721 if(U_FAILURE(status)){ 722 log_err_status(status, "ERROR: in creation of collation element iterator1 using ucol_openElements()\n %s\n", 723 myErrorName(status)); 724 ucol_close(en_us); 725 return; 726 } 727 iter2=ucol_openElements(en_us, test2, u_strlen(test2), &status); 728 if(U_FAILURE(status)){ 729 log_err("ERROR: in creation of collation element iterator2 using ucol_openElements()\n %s\n", 730 myErrorName(status)); 731 ucol_close(en_us); 732 return; 733 } 734 735 /* Run through the second iterator just to exercise it */ 736 c = ucol_next(iter2, &status); 737 i = 0; 738 739 while ( ++i < 10 && (c != UCOL_NULLORDER)) 740 { 741 if (U_FAILURE(status)) 742 { 743 log_err("iter2->next() returned an error. %s\n", myErrorName(status)); 744 ucol_closeElements(iter2); 745 ucol_closeElements(iter1); 746 ucol_close(en_us); 747 return; 748 } 749 750 c = ucol_next(iter2, &status); 751 } 752 753 /* Now set it to point to the same string as the first iterator */ 754 ucol_setText(iter2, test1, u_strlen(test1), &status); 755 if (U_FAILURE(status)) 756 { 757 log_err("call to iter2->setText(test1) failed. %s\n", myErrorName(status)); 758 } 759 else 760 { 761 assertEqual(iter1, iter2); 762 } 763 764 /* Now set it to point to a null string with fake length*/ 765 ucol_setText(iter2, NULL, 2, &status); 766 if (U_FAILURE(status)) 767 { 768 log_err("call to iter2->setText(null) failed. %s\n", myErrorName(status)); 769 } 770 else 771 { 772 if (ucol_next(iter2, &status) != UCOL_NULLORDER) { 773 log_err("iter2 with null text expected to return UCOL_NULLORDER\n"); 774 } 775 } 776 777 ucol_closeElements(iter2); 778 ucol_closeElements(iter1); 779 ucol_close(en_us); 780 } 781 782 /** @bug 4108762 783 * Test for getMaxExpansion() 784 */ 785 static void TestMaxExpansion() 786 { 787 UErrorCode status = U_ZERO_ERROR; 788 UCollator *coll ;/*= ucol_open("en_US", &status);*/ 789 UChar ch = 0; 790 UChar32 unassigned = 0xEFFFD; 791 UChar supplementary[2]; 792 uint32_t stringOffset = 0; 793 UBool isError = FALSE; 794 uint32_t sorder = 0; 795 UCollationElements *iter ;/*= ucol_openElements(coll, &ch, 1, &status);*/ 796 uint32_t temporder = 0; 797 798 UChar rule[256]; 799 u_uastrcpy(rule, "&a < ab < c/aba < d < z < ch"); 800 coll = ucol_openRules(rule, u_strlen(rule), UCOL_DEFAULT, 801 UCOL_DEFAULT_STRENGTH,NULL, &status); 802 if(U_SUCCESS(status) && coll) { 803 iter = ucol_openElements(coll, &ch, 1, &status); 804 805 while (ch < 0xFFFF && U_SUCCESS(status)) { 806 int count = 1; 807 uint32_t order; 808 int32_t size = 0; 809 810 ch ++; 811 812 ucol_setText(iter, &ch, 1, &status); 813 order = ucol_previous(iter, &status); 814 815 /* thai management */ 816 if (order == 0) 817 order = ucol_previous(iter, &status); 818 819 while (U_SUCCESS(status) && 820 ucol_previous(iter, &status) != UCOL_NULLORDER) { 821 count ++; 822 } 823 824 size = ucol_getMaxExpansion(iter, order); 825 if (U_FAILURE(status) || size < count) { 826 log_err("Failure at codepoint %d, maximum expansion count < %d\n", 827 ch, count); 828 } 829 } 830 831 /* testing for exact max expansion */ 832 ch = 0; 833 while (ch < 0x61) { 834 uint32_t order; 835 int32_t size; 836 ucol_setText(iter, &ch, 1, &status); 837 order = ucol_previous(iter, &status); 838 size = ucol_getMaxExpansion(iter, order); 839 if (U_FAILURE(status) || size != 1) { 840 log_err("Failure at codepoint %d, maximum expansion count < %d\n", 841 ch, 1); 842 } 843 ch ++; 844 } 845 846 ch = 0x63; 847 ucol_setText(iter, &ch, 1, &status); 848 temporder = ucol_previous(iter, &status); 849 850 if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) != 3) { 851 log_err("Failure at codepoint %d, maximum expansion count != %d\n", 852 ch, 3); 853 } 854 855 ch = 0x64; 856 ucol_setText(iter, &ch, 1, &status); 857 temporder = ucol_previous(iter, &status); 858 859 if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) != 1) { 860 log_err("Failure at codepoint %d, maximum expansion count != %d\n", 861 ch, 3); 862 } 863 864 U16_APPEND(supplementary, stringOffset, 2, unassigned, isError); 865 ucol_setText(iter, supplementary, 2, &status); 866 sorder = ucol_previous(iter, &status); 867 868 if (U_FAILURE(status) || ucol_getMaxExpansion(iter, sorder) != 2) { 869 log_err("Failure at codepoint %d, maximum expansion count < %d\n", 870 ch, 2); 871 } 872 873 /* testing jamo */ 874 ch = 0x1165; 875 876 ucol_setText(iter, &ch, 1, &status); 877 temporder = ucol_previous(iter, &status); 878 if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) > 3) { 879 log_err("Failure at codepoint %d, maximum expansion count > %d\n", 880 ch, 3); 881 } 882 883 ucol_closeElements(iter); 884 ucol_close(coll); 885 886 /* testing special jamo &a<\u1160 */ 887 rule[0] = 0x26; 888 rule[1] = 0x71; 889 rule[2] = 0x3c; 890 rule[3] = 0x1165; 891 rule[4] = 0x2f; 892 rule[5] = 0x71; 893 rule[6] = 0x71; 894 rule[7] = 0x71; 895 rule[8] = 0x71; 896 rule[9] = 0; 897 898 coll = ucol_openRules(rule, u_strlen(rule), UCOL_DEFAULT, 899 UCOL_DEFAULT_STRENGTH,NULL, &status); 900 iter = ucol_openElements(coll, &ch, 1, &status); 901 902 temporder = ucol_previous(iter, &status); 903 if (U_FAILURE(status) || ucol_getMaxExpansion(iter, temporder) != 6) { 904 log_err("Failure at codepoint %d, maximum expansion count > %d\n", 905 ch, 5); 906 } 907 908 ucol_closeElements(iter); 909 ucol_close(coll); 910 } else { 911 log_err_status(status, "Couldn't open collator -> %s\n", u_errorName(status)); 912 } 913 914 } 915 916 917 static void assertEqual(UCollationElements *i1, UCollationElements *i2) 918 { 919 int32_t c1, c2; 920 int32_t count = 0; 921 UErrorCode status = U_ZERO_ERROR; 922 923 do 924 { 925 c1 = ucol_next(i1, &status); 926 c2 = ucol_next(i2, &status); 927 928 if (c1 != c2) 929 { 930 log_err("Error in iteration %d assetEqual between\n %d and %d, they are not equal\n", count, c1, c2); 931 break; 932 } 933 934 count += 1; 935 } 936 while (c1 != UCOL_NULLORDER); 937 } 938 939 /** 940 * Testing iterators with extremely small buffers 941 */ 942 static void TestSmallBuffer() 943 { 944 UErrorCode status = U_ZERO_ERROR; 945 UCollator *coll; 946 UCollationElements *testiter, 947 *iter; 948 int32_t count = 0; 949 OrderAndOffset *testorders, 950 *orders; 951 952 UChar teststr[500]; 953 UChar str[] = {0x300, 0x31A, 0}; 954 /* 955 creating a long string of decomposable characters, 956 since by default the writable buffer is of size 256 957 */ 958 while (count < 500) { 959 if ((count & 1) == 0) { 960 teststr[count ++] = 0x300; 961 } 962 else { 963 teststr[count ++] = 0x31A; 964 } 965 } 966 967 coll = ucol_open("th_TH", &status); 968 if(U_SUCCESS(status) && coll) { 969 testiter = ucol_openElements(coll, teststr, 500, &status); 970 iter = ucol_openElements(coll, str, 2, &status); 971 972 orders = getOrders(iter, &count); 973 if (count != 2) { 974 log_err("Error collation elements size is not 2 for \\u0300\\u031A\n"); 975 } 976 977 /* 978 this will rearrange the string data to 250 characters of 0x300 first then 979 250 characters of 0x031A 980 */ 981 testorders = getOrders(testiter, &count); 982 983 if (count != 500) { 984 log_err("Error decomposition does not give the right sized collation elements\n"); 985 } 986 987 while (count != 0) { 988 /* UCA collation element for 0x0F76 */ 989 if ((count > 250 && testorders[-- count].order != orders[1].order) || 990 (count <= 250 && testorders[-- count].order != orders[0].order)) { 991 log_err("Error decomposition does not give the right collation element at %d count\n", count); 992 break; 993 } 994 } 995 996 free(testorders); 997 free(orders); 998 999 ucol_reset(testiter); 1000 1001 /* ensures closing of elements done properly to clear writable buffer */ 1002 ucol_next(testiter, &status); 1003 ucol_next(testiter, &status); 1004 ucol_closeElements(testiter); 1005 ucol_closeElements(iter); 1006 ucol_close(coll); 1007 } else { 1008 log_err_status(status, "Couldn't open collator -> %s\n", u_errorName(status)); 1009 } 1010 } 1011 1012 /** 1013 * Sniplets of code from genuca 1014 */ 1015 static int32_t hex2num(char hex) { 1016 if(hex>='0' && hex <='9') { 1017 return hex-'0'; 1018 } else if(hex>='a' && hex<='f') { 1019 return hex-'a'+10; 1020 } else if(hex>='A' && hex<='F') { 1021 return hex-'A'+10; 1022 } else { 1023 return 0; 1024 } 1025 } 1026 1027 /** 1028 * Getting codepoints from a string 1029 * @param str character string contain codepoints seperated by space and ended 1030 * by a semicolon 1031 * @param codepoints array for storage, assuming size > 5 1032 * @return position at the end of the codepoint section 1033 */ 1034 static char *getCodePoints(char *str, UChar *codepoints, UChar *contextCPs) { 1035 UErrorCode errorCode = U_ZERO_ERROR; 1036 char *semi = uprv_strchr(str, ';'); 1037 char *pipe = uprv_strchr(str, '|'); 1038 char *s; 1039 *codepoints = 0; 1040 *contextCPs = 0; 1041 if(semi == NULL) { 1042 log_err("expected semicolon after code point string in FractionalUCA.txt %s\n", str); 1043 return str; 1044 } 1045 if(pipe != NULL) { 1046 int32_t contextLength; 1047 *pipe = 0; 1048 contextLength = u_parseString(str, contextCPs, 99, NULL, &errorCode); 1049 *pipe = '|'; 1050 if(U_FAILURE(errorCode)) { 1051 log_err("error parsing precontext string from FractionalUCA.txt %s\n", str); 1052 return str; 1053 } 1054 /* prepend the precontext string to the codepoints */ 1055 u_memcpy(codepoints, contextCPs, contextLength); 1056 codepoints += contextLength; 1057 /* start of the code point string */ 1058 s = pipe + 1; 1059 } else { 1060 s = str; 1061 } 1062 u_parseString(s, codepoints, 99, NULL, &errorCode); 1063 if(U_FAILURE(errorCode)) { 1064 log_err("error parsing code point string from FractionalUCA.txt %s\n", str); 1065 return str; 1066 } 1067 return semi + 1; 1068 } 1069 1070 /** 1071 * Sniplets of code from genuca 1072 */ 1073 static int32_t 1074 readElement(char **from, char *to, char separator, UErrorCode *status) 1075 { 1076 if (U_SUCCESS(*status)) { 1077 char buffer[1024]; 1078 int32_t i = 0; 1079 while (**from != separator) { 1080 if (**from != ' ') { 1081 *(buffer+i++) = **from; 1082 } 1083 (*from)++; 1084 } 1085 (*from)++; 1086 *(buffer + i) = 0; 1087 strcpy(to, buffer); 1088 return i/2; 1089 } 1090 1091 return 0; 1092 } 1093 1094 /** 1095 * Sniplets of code from genuca 1096 */ 1097 static uint32_t 1098 getSingleCEValue(char *primary, char *secondary, char *tertiary, 1099 UErrorCode *status) 1100 { 1101 if (U_SUCCESS(*status)) { 1102 uint32_t value = 0; 1103 char primsave = '\0'; 1104 char secsave = '\0'; 1105 char tersave = '\0'; 1106 char *primend = primary+4; 1107 char *secend = secondary+2; 1108 char *terend = tertiary+2; 1109 uint32_t primvalue; 1110 uint32_t secvalue; 1111 uint32_t tervalue; 1112 1113 if (uprv_strlen(primary) > 4) { 1114 primsave = *primend; 1115 *primend = '\0'; 1116 } 1117 1118 if (uprv_strlen(secondary) > 2) { 1119 secsave = *secend; 1120 *secend = '\0'; 1121 } 1122 1123 if (uprv_strlen(tertiary) > 2) { 1124 tersave = *terend; 1125 *terend = '\0'; 1126 } 1127 1128 primvalue = (*primary!='\0')?uprv_strtoul(primary, &primend, 16):0; 1129 secvalue = (*secondary!='\0')?uprv_strtoul(secondary, &secend, 16):0; 1130 tervalue = (*tertiary!='\0')?uprv_strtoul(tertiary, &terend, 16):0; 1131 if(primvalue <= 0xFF) { 1132 primvalue <<= 8; 1133 } 1134 1135 value = ((primvalue << UCOL_PRIMARYORDERSHIFT) & UCOL_PRIMARYORDERMASK) 1136 | ((secvalue << UCOL_SECONDARYORDERSHIFT) & UCOL_SECONDARYORDERMASK) 1137 | (tervalue & UCOL_TERTIARYORDERMASK); 1138 1139 if(primsave!='\0') { 1140 *primend = primsave; 1141 } 1142 if(secsave!='\0') { 1143 *secend = secsave; 1144 } 1145 if(tersave!='\0') { 1146 *terend = tersave; 1147 } 1148 return value; 1149 } 1150 return 0; 1151 } 1152 1153 /** 1154 * Getting collation elements generated from a string 1155 * @param str character string contain collation elements contained in [] and 1156 * seperated by space 1157 * @param ce array for storage, assuming size > 20 1158 * @param status error status 1159 * @return position at the end of the codepoint section 1160 */ 1161 static char * getCEs(char *str, uint32_t *ces, UErrorCode *status) { 1162 char *pStartCP = uprv_strchr(str, '['); 1163 int count = 0; 1164 char *pEndCP; 1165 char primary[100]; 1166 char secondary[100]; 1167 char tertiary[100]; 1168 1169 while (*pStartCP == '[') { 1170 uint32_t primarycount = 0; 1171 uint32_t secondarycount = 0; 1172 uint32_t tertiarycount = 0; 1173 uint32_t CEi = 1; 1174 pEndCP = strchr(pStartCP, ']'); 1175 if(pEndCP == NULL) { 1176 break; 1177 } 1178 pStartCP ++; 1179 1180 primarycount = readElement(&pStartCP, primary, ',', status); 1181 secondarycount = readElement(&pStartCP, secondary, ',', status); 1182 tertiarycount = readElement(&pStartCP, tertiary, ']', status); 1183 1184 /* I want to get the CEs entered right here, including continuation */ 1185 ces[count ++] = getSingleCEValue(primary, secondary, tertiary, status); 1186 if (U_FAILURE(*status)) { 1187 break; 1188 } 1189 1190 while (2 * CEi < primarycount || CEi < secondarycount || 1191 CEi < tertiarycount) { 1192 uint32_t value = UCOL_CONTINUATION_MARKER; /* Continuation marker */ 1193 if (2 * CEi < primarycount) { 1194 value |= ((hex2num(*(primary + 4 * CEi)) & 0xF) << 28); 1195 value |= ((hex2num(*(primary + 4 * CEi + 1)) & 0xF) << 24); 1196 } 1197 1198 if (2 * CEi + 1 < primarycount) { 1199 value |= ((hex2num(*(primary + 4 * CEi + 2)) & 0xF) << 20); 1200 value |= ((hex2num(*(primary + 4 * CEi + 3)) &0xF) << 16); 1201 } 1202 1203 if (CEi < secondarycount) { 1204 value |= ((hex2num(*(secondary + 2 * CEi)) & 0xF) << 12); 1205 value |= ((hex2num(*(secondary + 2 * CEi + 1)) & 0xF) << 8); 1206 } 1207 1208 if (CEi < tertiarycount) { 1209 value |= ((hex2num(*(tertiary + 2 * CEi)) & 0x3) << 4); 1210 value |= (hex2num(*(tertiary + 2 * CEi + 1)) & 0xF); 1211 } 1212 1213 CEi ++; 1214 ces[count ++] = value; 1215 } 1216 1217 pStartCP = pEndCP + 1; 1218 } 1219 ces[count] = 0; 1220 return pStartCP; 1221 } 1222 1223 /** 1224 * Getting the FractionalUCA.txt file stream 1225 */ 1226 static FileStream * getFractionalUCA(void) 1227 { 1228 char newPath[256]; 1229 char backupPath[256]; 1230 FileStream *result = NULL; 1231 1232 /* Look inside ICU_DATA first */ 1233 uprv_strcpy(newPath, ctest_dataSrcDir()); 1234 uprv_strcat(newPath, "unidata" U_FILE_SEP_STRING ); 1235 uprv_strcat(newPath, "FractionalUCA.txt"); 1236 1237 /* As a fallback, try to guess where the source data was located 1238 * at the time ICU was built, and look there. 1239 */ 1240 #if defined (U_TOPSRCDIR) 1241 strcpy(backupPath, U_TOPSRCDIR U_FILE_SEP_STRING "data"); 1242 #else 1243 { 1244 UErrorCode errorCode = U_ZERO_ERROR; 1245 strcpy(backupPath, loadTestData(&errorCode)); 1246 strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data"); 1247 } 1248 #endif 1249 strcat(backupPath, U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING "FractionalUCA.txt"); 1250 1251 result = T_FileStream_open(newPath, "rb"); 1252 1253 if (result == NULL) { 1254 result = T_FileStream_open(backupPath, "rb"); 1255 if (result == NULL) { 1256 log_err("Failed to open either %s or %s\n", newPath, backupPath); 1257 } 1258 } 1259 return result; 1260 } 1261 1262 /** 1263 * Testing the CEs returned by the iterator 1264 */ 1265 static void TestCEs() { 1266 FileStream *file = NULL; 1267 char line[2048]; 1268 char *str; 1269 UChar codepoints[10]; 1270 uint32_t ces[20]; 1271 UErrorCode status = U_ZERO_ERROR; 1272 UCollator *coll = ucol_open("", &status); 1273 uint32_t lineNo = 0; 1274 UChar contextCPs[5]; 1275 1276 if (U_FAILURE(status)) { 1277 log_err_status(status, "Error in opening root collator -> %s\n", u_errorName(status)); 1278 return; 1279 } 1280 1281 file = getFractionalUCA(); 1282 1283 if (file == NULL) { 1284 log_err("*** unable to open input FractionalUCA.txt file ***\n"); 1285 return; 1286 } 1287 1288 1289 while (T_FileStream_readLine(file, line, sizeof(line)) != NULL) { 1290 int count = 0; 1291 UCollationElements *iter; 1292 int32_t preContextCeLen=0; 1293 lineNo++; 1294 /* skip this line if it is empty or a comment or is a return value 1295 or start of some variable section */ 1296 if(line[0] == 0 || line[0] == '#' || line[0] == '\n' || 1297 line[0] == 0x000D || line[0] == '[') { 1298 continue; 1299 } 1300 1301 str = getCodePoints(line, codepoints, contextCPs); 1302 1303 /* these are 'fake' codepoints in the fractional UCA, and are used just 1304 * for positioning of indirect values. They should not go through this 1305 * test. 1306 */ 1307 if(*codepoints == 0xFDD0) { 1308 continue; 1309 } 1310 if (*contextCPs != 0) { 1311 iter = ucol_openElements(coll, contextCPs, -1, &status); 1312 if (U_FAILURE(status)) { 1313 log_err("Error in opening collation elements\n"); 1314 break; 1315 } 1316 while((ces[preContextCeLen] = ucol_next(iter, &status)) != (uint32_t)UCOL_NULLORDER) { 1317 preContextCeLen++; 1318 } 1319 ucol_closeElements(iter); 1320 } 1321 1322 getCEs(str, ces+preContextCeLen, &status); 1323 if (U_FAILURE(status)) { 1324 log_err("Error in parsing collation elements in FractionalUCA.txt\n"); 1325 break; 1326 } 1327 iter = ucol_openElements(coll, codepoints, -1, &status); 1328 if (U_FAILURE(status)) { 1329 log_err("Error in opening collation elements\n"); 1330 break; 1331 } 1332 for (;;) { 1333 uint32_t ce = (uint32_t)ucol_next(iter, &status); 1334 if (ce == 0xFFFFFFFF) { 1335 ce = 0; 1336 } 1337 /* we now unconditionally reorder Thai/Lao prevowels, so this 1338 * test would fail if we don't skip here. 1339 */ 1340 if(UCOL_ISTHAIPREVOWEL(*codepoints) && ce == 0 && count == 0) { 1341 continue; 1342 } 1343 if (ce != ces[count] || U_FAILURE(status)) { 1344 log_err("Collation elements in FractionalUCA.txt and iterators do not match!\n"); 1345 break; 1346 } 1347 if (ces[count] == 0) { 1348 break; 1349 } 1350 count ++; 1351 } 1352 ucol_closeElements(iter); 1353 } 1354 1355 T_FileStream_close(file); 1356 ucol_close(coll); 1357 } 1358 1359 /** 1360 * Testing the discontigous contractions 1361 */ 1362 static void TestDiscontiguos() { 1363 const char *rulestr = 1364 "&z < AB < X\\u0300 < ABC < X\\u0300\\u0315"; 1365 UChar rule[50]; 1366 int rulelen = u_unescape(rulestr, rule, 50); 1367 const char *src[] = { 1368 "ADB", "ADBC", "A\\u0315B", "A\\u0315BC", 1369 /* base character blocked */ 1370 "XD\\u0300", "XD\\u0300\\u0315", 1371 /* non blocking combining character */ 1372 "X\\u0319\\u0300", "X\\u0319\\u0300\\u0315", 1373 /* blocking combining character */ 1374 "X\\u0314\\u0300", "X\\u0314\\u0300\\u0315", 1375 /* contraction prefix */ 1376 "ABDC", "AB\\u0315C","X\\u0300D\\u0315", "X\\u0300\\u0319\\u0315", 1377 "X\\u0300\\u031A\\u0315", 1378 /* ends not with a contraction character */ 1379 "X\\u0319\\u0300D", "X\\u0319\\u0300\\u0315D", "X\\u0300D\\u0315D", 1380 "X\\u0300\\u0319\\u0315D", "X\\u0300\\u031A\\u0315D" 1381 }; 1382 const char *tgt[] = { 1383 /* non blocking combining character */ 1384 "A D B", "A D BC", "A \\u0315 B", "A \\u0315 BC", 1385 /* base character blocked */ 1386 "X D \\u0300", "X D \\u0300\\u0315", 1387 /* non blocking combining character */ 1388 "X\\u0300 \\u0319", "X\\u0300\\u0315 \\u0319", 1389 /* blocking combining character */ 1390 "X \\u0314 \\u0300", "X \\u0314 \\u0300\\u0315", 1391 /* contraction prefix */ 1392 "AB DC", "AB \\u0315 C","X\\u0300 D \\u0315", "X\\u0300\\u0315 \\u0319", 1393 "X\\u0300 \\u031A \\u0315", 1394 /* ends not with a contraction character */ 1395 "X\\u0300 \\u0319D", "X\\u0300\\u0315 \\u0319D", "X\\u0300 D\\u0315D", 1396 "X\\u0300\\u0315 \\u0319D", "X\\u0300 \\u031A\\u0315D" 1397 }; 1398 int size = 20; 1399 UCollator *coll; 1400 UErrorCode status = U_ZERO_ERROR; 1401 int count = 0; 1402 UCollationElements *iter; 1403 UCollationElements *resultiter; 1404 1405 coll = ucol_openRules(rule, rulelen, UCOL_OFF, UCOL_DEFAULT_STRENGTH,NULL, &status); 1406 iter = ucol_openElements(coll, rule, 1, &status); 1407 resultiter = ucol_openElements(coll, rule, 1, &status); 1408 1409 if (U_FAILURE(status)) { 1410 log_err_status(status, "Error opening collation rules -> %s\n", u_errorName(status)); 1411 return; 1412 } 1413 1414 while (count < size) { 1415 UChar str[20]; 1416 UChar tstr[20]; 1417 int strLen = u_unescape(src[count], str, 20); 1418 UChar *s; 1419 1420 ucol_setText(iter, str, strLen, &status); 1421 if (U_FAILURE(status)) { 1422 log_err("Error opening collation iterator\n"); 1423 return; 1424 } 1425 1426 u_unescape(tgt[count], tstr, 20); 1427 s = tstr; 1428 1429 log_verbose("count %d\n", count); 1430 1431 for (;;) { 1432 uint32_t ce; 1433 UChar *e = u_strchr(s, 0x20); 1434 if (e == 0) { 1435 e = u_strchr(s, 0); 1436 } 1437 ucol_setText(resultiter, s, (int32_t)(e - s), &status); 1438 ce = ucol_next(resultiter, &status); 1439 if (U_FAILURE(status)) { 1440 log_err("Error manipulating collation iterator\n"); 1441 return; 1442 } 1443 while (ce != UCOL_NULLORDER) { 1444 if (ce != (uint32_t)ucol_next(iter, &status) || 1445 U_FAILURE(status)) { 1446 log_err("Discontiguos contraction test mismatch\n"); 1447 return; 1448 } 1449 ce = ucol_next(resultiter, &status); 1450 if (U_FAILURE(status)) { 1451 log_err("Error getting next collation element\n"); 1452 return; 1453 } 1454 } 1455 s = e + 1; 1456 if (*e == 0) { 1457 break; 1458 } 1459 } 1460 ucol_reset(iter); 1461 backAndForth(iter); 1462 count ++; 1463 } 1464 ucol_closeElements(resultiter); 1465 ucol_closeElements(iter); 1466 ucol_close(coll); 1467 } 1468 1469 static void TestCEBufferOverflow() 1470 { 1471 UChar str[UCOL_EXPAND_CE_BUFFER_SIZE + 1]; 1472 UErrorCode status = U_ZERO_ERROR; 1473 UChar rule[10]; 1474 UCollator *coll; 1475 UCollationElements *iter; 1476 1477 u_uastrcpy(rule, "&z < AB"); 1478 coll = ucol_openRules(rule, u_strlen(rule), UCOL_OFF, UCOL_DEFAULT_STRENGTH, NULL,&status); 1479 if (U_FAILURE(status)) { 1480 log_err_status(status, "Rule based collator not created for testing ce buffer overflow -> %s\n", u_errorName(status)); 1481 return; 1482 } 1483 1484 /* 0xDCDC is a trail surrogate hence deemed unsafe by the heuristic 1485 test. this will cause an overflow in getPrev */ 1486 str[0] = 0x0041; /* 'A' */ 1487 /*uprv_memset(str + 1, 0xE0, sizeof(UChar) * UCOL_EXPAND_CE_BUFFER_SIZE);*/ 1488 uprv_memset(str + 1, 0xDC, sizeof(UChar) * UCOL_EXPAND_CE_BUFFER_SIZE); 1489 str[UCOL_EXPAND_CE_BUFFER_SIZE] = 0x0042; /* 'B' */ 1490 iter = ucol_openElements(coll, str, UCOL_EXPAND_CE_BUFFER_SIZE + 1, 1491 &status); 1492 if (ucol_previous(iter, &status) == UCOL_NULLORDER || 1493 status == U_BUFFER_OVERFLOW_ERROR) { 1494 log_err("CE buffer should not overflow with long string of trail surrogates\n"); 1495 } 1496 ucol_closeElements(iter); 1497 ucol_close(coll); 1498 } 1499 1500 /** 1501 * Checking collation element validity. 1502 */ 1503 #define MAX_CODEPOINTS_TO_SHOW 10 1504 static void showCodepoints(const UChar *codepoints, int length, char * codepointText) { 1505 int i, lengthToUse = length; 1506 if (lengthToUse > MAX_CODEPOINTS_TO_SHOW) { 1507 lengthToUse = MAX_CODEPOINTS_TO_SHOW; 1508 } 1509 for (i = 0; i < lengthToUse; ++i) { 1510 int bytesWritten = sprintf(codepointText, " %04X", *codepoints++); 1511 if (bytesWritten <= 0) { 1512 break; 1513 } 1514 codepointText += bytesWritten; 1515 } 1516 if (i < length) { 1517 sprintf(codepointText, " ..."); 1518 } 1519 } 1520 1521 static UBool checkCEValidity(const UCollator *coll, const UChar *codepoints, 1522 int length) 1523 { 1524 UErrorCode status = U_ZERO_ERROR; 1525 UCollationElements *iter = ucol_openElements(coll, codepoints, length, 1526 &status); 1527 UBool result = FALSE; 1528 UBool primaryDone = FALSE, secondaryDone = FALSE, tertiaryDone = FALSE; 1529 const char * collLocale; 1530 1531 if (U_FAILURE(status)) { 1532 log_err("Error creating iterator for testing validity\n"); 1533 return FALSE; 1534 } 1535 collLocale = ucol_getLocale(coll, ULOC_VALID_LOCALE, &status); 1536 if (U_FAILURE(status) || collLocale==NULL) { 1537 status = U_ZERO_ERROR; 1538 collLocale = "?"; 1539 } 1540 1541 for (;;) { 1542 uint32_t ce = ucol_next(iter, &status); 1543 uint32_t primary, p1, p2, secondary, tertiary; 1544 if (ce == UCOL_NULLORDER) { 1545 result = TRUE; 1546 break; 1547 } 1548 if (ce == 0) { 1549 continue; 1550 } 1551 if (ce == 0x02000202) { 1552 /* special CE for merge-sort character */ 1553 if (*codepoints == 0xFFFE /* && length == 1 */) { 1554 /* 1555 * Note: We should check for length==1 but the token parser appears 1556 * to give us trailing NUL characters. 1557 * TODO: Ticket #8047: Change TestCEValidity to use ucol_getTailoredSet() 1558 * rather than the internal collation rule parser 1559 */ 1560 continue; 1561 } else { 1562 log_err("Special 02/02/02 weight for code point U+%04X [len %d] != U+FFFE\n", 1563 (int)*codepoints, (int)length); 1564 break; 1565 } 1566 } 1567 primary = UCOL_PRIMARYORDER(ce); 1568 p1 = primary >> 8; 1569 p2 = primary & 0xFF; 1570 secondary = UCOL_SECONDARYORDER(ce); 1571 tertiary = UCOL_TERTIARYORDER(ce) & UCOL_REMOVE_CONTINUATION; 1572 1573 if (!isContinuation(ce)) { 1574 if ((ce & UCOL_REMOVE_CONTINUATION) == 0) { 1575 log_err("Empty CE %08lX except for case bits\n", (long)ce); 1576 break; 1577 } 1578 if (p1 == 0) { 1579 if (p2 != 0) { 1580 log_err("Primary 00 xx in %08lX\n", (long)ce); 1581 break; 1582 } 1583 primaryDone = TRUE; 1584 } else { 1585 if (p1 <= 2 || p1 >= 0xF0) { 1586 /* Primary first bytes F0..FF are specials. */ 1587 log_err("Primary first byte of %08lX out of range\n", (long)ce); 1588 break; 1589 } 1590 if (p2 == 0) { 1591 primaryDone = TRUE; 1592 } else { 1593 if (p2 <= 3 || p2 >= 0xFF) { 1594 /* Primary second bytes 03 and FF are sort key compression terminators. */ 1595 log_err("Primary second byte of %08lX out of range\n", (long)ce); 1596 break; 1597 } 1598 primaryDone = FALSE; 1599 } 1600 } 1601 if (secondary == 0) { 1602 if (primary != 0) { 1603 log_err("Primary!=0 secondary==0 in %08lX\n", (long)ce); 1604 break; 1605 } 1606 secondaryDone = TRUE; 1607 } else { 1608 if (secondary <= 2 || 1609 (UCOL_BYTE_COMMON < secondary && secondary <= (UCOL_BYTE_COMMON + 0x80)) 1610 ) { 1611 /* Secondary first bytes common+1..+0x80 are used for sort key compression. */ 1612 log_err("Secondary byte of %08lX out of range\n", (long)ce); 1613 break; 1614 } 1615 secondaryDone = FALSE; 1616 } 1617 if (tertiary == 0) { 1618 /* We know that ce != 0. */ 1619 log_err("Primary!=0 or secondary!=0 but tertiary==0 in %08lX\n", (long)ce); 1620 break; 1621 } 1622 if (tertiary <= 2) { 1623 log_err("Tertiary byte of %08lX out of range\n", (long)ce); 1624 break; 1625 } 1626 tertiaryDone = FALSE; 1627 } else { 1628 if ((ce & UCOL_REMOVE_CONTINUATION) == 0) { 1629 log_err("Empty continuation %08lX\n", (long)ce); 1630 break; 1631 } 1632 if (primaryDone && primary != 0) { 1633 log_err("Primary was done but continues in %08lX\n", (long)ce); 1634 break; 1635 } 1636 if (p1 == 0) { 1637 if (p2 != 0) { 1638 log_err("Primary 00 xx in %08lX\n", (long)ce); 1639 break; 1640 } 1641 primaryDone = TRUE; 1642 } else { 1643 if (p1 <= 2) { 1644 log_err("Primary first byte of %08lX out of range\n", (long)ce); 1645 break; 1646 } 1647 if (p2 == 0) { 1648 primaryDone = TRUE; 1649 } else { 1650 if (p2 <= 3) { 1651 log_err("Primary second byte of %08lX out of range\n", (long)ce); 1652 break; 1653 } 1654 } 1655 } 1656 if (secondaryDone && secondary != 0) { 1657 log_err("Secondary was done but continues in %08lX\n", (long)ce); 1658 break; 1659 } 1660 if (secondary == 0) { 1661 secondaryDone = TRUE; 1662 } else { 1663 if (secondary <= 2) { 1664 log_err("Secondary byte of %08lX out of range\n", (long)ce); 1665 break; 1666 } 1667 } 1668 if (tertiaryDone && tertiary != 0) { 1669 log_err("Tertiary was done but continues in %08lX\n", (long)ce); 1670 break; 1671 } 1672 if (tertiary == 0) { 1673 tertiaryDone = TRUE; 1674 } else if (tertiary <= 2) { 1675 log_err("Tertiary byte of %08lX out of range\n", (long)ce); 1676 break; 1677 } 1678 } 1679 } 1680 if (!result) { 1681 char codepointText[5*MAX_CODEPOINTS_TO_SHOW + 5]; 1682 showCodepoints(codepoints, length, codepointText); 1683 log_err("Locale: %s Code point string: %s\n", collLocale, codepointText); 1684 } 1685 ucol_closeElements(iter); 1686 return result; 1687 } 1688 1689 static void TestCEValidity() 1690 { 1691 /* testing UCA collation elements */ 1692 UErrorCode status = U_ZERO_ERROR; 1693 /* en_US has no tailorings */ 1694 UCollator *coll = ucol_open("root", &status); 1695 /* tailored locales */ 1696 char locale[][11] = {"fr_FR", "ko_KR", "sh_YU", "th_TH", "zh_CN", "zh__PINYIN"}; 1697 const char *loc; 1698 FileStream *file = NULL; 1699 char line[2048]; 1700 UChar codepoints[11]; 1701 int count = 0; 1702 int maxCount = 0; 1703 UChar contextCPs[3]; 1704 UChar32 c; 1705 UParseError parseError; 1706 if (U_FAILURE(status)) { 1707 log_err_status(status, "en_US collator creation failed -> %s\n", u_errorName(status)); 1708 return; 1709 } 1710 log_verbose("Testing UCA elements\n"); 1711 file = getFractionalUCA(); 1712 if (file == NULL) { 1713 log_err("Fractional UCA data can not be opened\n"); 1714 return; 1715 } 1716 1717 while (T_FileStream_readLine(file, line, sizeof(line)) != NULL) { 1718 if(line[0] == 0 || line[0] == '#' || line[0] == '\n' || 1719 line[0] == 0x000D || line[0] == '[') { 1720 continue; 1721 } 1722 1723 getCodePoints(line, codepoints, contextCPs); 1724 checkCEValidity(coll, codepoints, u_strlen(codepoints)); 1725 } 1726 1727 log_verbose("Testing UCA elements for the whole range of unicode characters\n"); 1728 for (c = 0; c <= 0xffff; ++c) { 1729 if (u_isdefined(c)) { 1730 codepoints[0] = (UChar)c; 1731 checkCEValidity(coll, codepoints, 1); 1732 } 1733 } 1734 for (; c <= 0x10ffff; ++c) { 1735 if (u_isdefined(c)) { 1736 int32_t i = 0; 1737 U16_APPEND_UNSAFE(codepoints, i, c); 1738 checkCEValidity(coll, codepoints, i); 1739 } 1740 } 1741 1742 ucol_close(coll); 1743 1744 /* testing tailored collation elements */ 1745 log_verbose("Testing tailored elements\n"); 1746 if(getTestOption(QUICK_OPTION)) { 1747 maxCount = sizeof(locale)/sizeof(locale[0]); 1748 } else { 1749 maxCount = uloc_countAvailable(); 1750 } 1751 while (count < maxCount) { 1752 const UChar *rules = NULL, 1753 *current = NULL; 1754 UChar *rulesCopy = NULL; 1755 int32_t ruleLen = 0; 1756 1757 uint32_t chOffset = 0; 1758 uint32_t chLen = 0; 1759 uint32_t exOffset = 0; 1760 uint32_t exLen = 0; 1761 uint32_t prefixOffset = 0; 1762 uint32_t prefixLen = 0; 1763 UBool startOfRules = TRUE; 1764 UColOptionSet opts; 1765 1766 UColTokenParser src; 1767 uint32_t strength = 0; 1768 uint16_t specs = 0; 1769 if(getTestOption(QUICK_OPTION)) { 1770 loc = locale[count]; 1771 } else { 1772 loc = uloc_getAvailable(count); 1773 if(!hasCollationElements(loc)) { 1774 count++; 1775 continue; 1776 } 1777 } 1778 1779 uprv_memset(&src, 0, sizeof(UColTokenParser)); 1780 1781 log_verbose("Testing CEs for %s\n", loc); 1782 1783 coll = ucol_open(loc, &status); 1784 if (U_FAILURE(status)) { 1785 log_err("%s collator creation failed\n", loc); 1786 return; 1787 } 1788 1789 src.opts = &opts; 1790 rules = ucol_getRules(coll, &ruleLen); 1791 1792 if (ruleLen > 0) { 1793 rulesCopy = (UChar *)uprv_malloc((ruleLen + 1794 UCOL_TOK_EXTRA_RULE_SPACE_SIZE) * sizeof(UChar)); 1795 uprv_memcpy(rulesCopy, rules, ruleLen * sizeof(UChar)); 1796 src.current = src.source = rulesCopy; 1797 src.end = rulesCopy + ruleLen; 1798 src.extraCurrent = src.end; 1799 src.extraEnd = src.end + UCOL_TOK_EXTRA_RULE_SPACE_SIZE; 1800 1801 /* Note that as a result of tickets 7015 or 6912, ucol_tok_parseNextToken can cause the pointer to 1802 the rules copy in src.source to get reallocated, freeing the original pointer in rulesCopy */ 1803 while ((current = ucol_tok_parseNextToken(&src, startOfRules, &parseError,&status)) != NULL) { 1804 strength = src.parsedToken.strength; 1805 chOffset = src.parsedToken.charsOffset; 1806 chLen = src.parsedToken.charsLen; 1807 exOffset = src.parsedToken.extensionOffset; 1808 exLen = src.parsedToken.extensionLen; 1809 prefixOffset = src.parsedToken.prefixOffset; 1810 prefixLen = src.parsedToken.prefixLen; 1811 specs = src.parsedToken.flags; 1812 1813 startOfRules = FALSE; 1814 uprv_memcpy(codepoints, src.source + chOffset, 1815 chLen * sizeof(UChar)); 1816 codepoints[chLen] = 0; 1817 checkCEValidity(coll, codepoints, chLen); 1818 } 1819 uprv_free(src.source); 1820 } 1821 1822 ucol_close(coll); 1823 count ++; 1824 } 1825 T_FileStream_close(file); 1826 } 1827 1828 static void printSortKeyError(const UChar *codepoints, int length, 1829 uint8_t *sortkey, int sklen) 1830 { 1831 int count = 0; 1832 log_err("Sortkey not valid for "); 1833 while (length > 0) { 1834 log_err("0x%04x ", *codepoints); 1835 length --; 1836 codepoints ++; 1837 } 1838 log_err("\nSortkey : "); 1839 while (count < sklen) { 1840 log_err("0x%02x ", sortkey[count]); 1841 count ++; 1842 } 1843 log_err("\n"); 1844 } 1845 1846 /** 1847 * Checking sort key validity for all levels 1848 */ 1849 static UBool checkSortKeyValidity(UCollator *coll, 1850 const UChar *codepoints, 1851 int length) 1852 { 1853 UErrorCode status = U_ZERO_ERROR; 1854 UCollationStrength strength[5] = {UCOL_PRIMARY, UCOL_SECONDARY, 1855 UCOL_TERTIARY, UCOL_QUATERNARY, 1856 UCOL_IDENTICAL}; 1857 int strengthlen = 5; 1858 int strengthIndex = 0; 1859 int caselevel = 0; 1860 1861 while (caselevel < 1) { 1862 if (caselevel == 0) { 1863 ucol_setAttribute(coll, UCOL_CASE_LEVEL, UCOL_OFF, &status); 1864 } 1865 else { 1866 ucol_setAttribute(coll, UCOL_CASE_LEVEL, UCOL_ON, &status); 1867 } 1868 1869 while (strengthIndex < strengthlen) { 1870 int count01 = 0; 1871 uint32_t count = 0; 1872 uint8_t sortkey[128]; 1873 uint32_t sklen; 1874 1875 ucol_setStrength(coll, strength[strengthIndex]); 1876 sklen = ucol_getSortKey(coll, codepoints, length, sortkey, 128); 1877 while (sortkey[count] != 0) { 1878 if (sortkey[count] == 2 || (sortkey[count] == 3 && count01 > 0 && strengthIndex != 4)) { 1879 printSortKeyError(codepoints, length, sortkey, sklen); 1880 return FALSE; 1881 } 1882 if (sortkey[count] == 1) { 1883 count01 ++; 1884 } 1885 count ++; 1886 } 1887 1888 if (count + 1 != sklen || (count01 != strengthIndex + caselevel)) { 1889 printSortKeyError(codepoints, length, sortkey, sklen); 1890 return FALSE; 1891 } 1892 strengthIndex ++; 1893 } 1894 caselevel ++; 1895 } 1896 return TRUE; 1897 } 1898 1899 static void TestSortKeyValidity(void) 1900 { 1901 /* testing UCA collation elements */ 1902 UErrorCode status = U_ZERO_ERROR; 1903 /* en_US has no tailorings */ 1904 UCollator *coll = ucol_open("en_US", &status); 1905 /* tailored locales */ 1906 char locale[][6] = {"fr_FR", "ko_KR", "sh_YU", "th_TH", "zh_CN"}; 1907 FileStream *file = NULL; 1908 char line[2048]; 1909 UChar codepoints[10]; 1910 int count = 0; 1911 UChar contextCPs[5]; 1912 UParseError parseError; 1913 if (U_FAILURE(status)) { 1914 log_err_status(status, "en_US collator creation failed -> %s\n", u_errorName(status)); 1915 return; 1916 } 1917 log_verbose("Testing UCA elements\n"); 1918 file = getFractionalUCA(); 1919 if (file == NULL) { 1920 log_err("Fractional UCA data can not be opened\n"); 1921 return; 1922 } 1923 1924 while (T_FileStream_readLine(file, line, sizeof(line)) != NULL) { 1925 if(line[0] == 0 || line[0] == '#' || line[0] == '\n' || 1926 line[0] == 0x000D || line[0] == '[') { 1927 continue; 1928 } 1929 1930 getCodePoints(line, codepoints, contextCPs); 1931 if(codepoints[0] == 0xFFFE) { 1932 /* Skip special merge-sort character U+FFFE which has otherwise illegal 02 weight bytes. */ 1933 continue; 1934 } 1935 checkSortKeyValidity(coll, codepoints, u_strlen(codepoints)); 1936 } 1937 1938 log_verbose("Testing UCA elements for the whole range of unicode characters\n"); 1939 codepoints[0] = 0; 1940 1941 while (codepoints[0] < 0xFFFF) { 1942 if (u_isdefined((UChar32)codepoints[0])) { 1943 checkSortKeyValidity(coll, codepoints, 1); 1944 } 1945 codepoints[0] ++; 1946 } 1947 1948 ucol_close(coll); 1949 1950 /* testing tailored collation elements */ 1951 log_verbose("Testing tailored elements\n"); 1952 while (count < 5) { 1953 const UChar *rules = NULL, 1954 *current = NULL; 1955 UChar *rulesCopy = NULL; 1956 int32_t ruleLen = 0; 1957 1958 uint32_t chOffset = 0; 1959 uint32_t chLen = 0; 1960 uint32_t exOffset = 0; 1961 uint32_t exLen = 0; 1962 uint32_t prefixOffset = 0; 1963 uint32_t prefixLen = 0; 1964 UBool startOfRules = TRUE; 1965 UColOptionSet opts; 1966 1967 UColTokenParser src; 1968 uint32_t strength = 0; 1969 uint16_t specs = 0; 1970 1971 uprv_memset(&src, 0, sizeof(UColTokenParser)); 1972 1973 coll = ucol_open(locale[count], &status); 1974 if (U_FAILURE(status)) { 1975 log_err("%s collator creation failed\n", locale[count]); 1976 return; 1977 } 1978 1979 src.opts = &opts; 1980 rules = ucol_getRules(coll, &ruleLen); 1981 1982 if (ruleLen > 0) { 1983 rulesCopy = (UChar *)uprv_malloc((ruleLen + 1984 UCOL_TOK_EXTRA_RULE_SPACE_SIZE) * sizeof(UChar)); 1985 uprv_memcpy(rulesCopy, rules, ruleLen * sizeof(UChar)); 1986 src.current = src.source = rulesCopy; 1987 src.end = rulesCopy + ruleLen; 1988 src.extraCurrent = src.end; 1989 src.extraEnd = src.end + UCOL_TOK_EXTRA_RULE_SPACE_SIZE; 1990 1991 /* Note that as a result of tickets 7015 or 6912, ucol_tok_parseNextToken can cause the pointer to 1992 the rules copy in src.source to get reallocated, freeing the original pointer in rulesCopy */ 1993 while ((current = ucol_tok_parseNextToken(&src, startOfRules,&parseError, &status)) != NULL) { 1994 strength = src.parsedToken.strength; 1995 chOffset = src.parsedToken.charsOffset; 1996 chLen = src.parsedToken.charsLen; 1997 exOffset = src.parsedToken.extensionOffset; 1998 exLen = src.parsedToken.extensionLen; 1999 prefixOffset = src.parsedToken.prefixOffset; 2000 prefixLen = src.parsedToken.prefixLen; 2001 specs = src.parsedToken.flags; 2002 2003 startOfRules = FALSE; 2004 uprv_memcpy(codepoints, src.source + chOffset, 2005 chLen * sizeof(UChar)); 2006 codepoints[chLen] = 0; 2007 if(codepoints[0] == 0xFFFE) { 2008 /* Skip special merge-sort character U+FFFE which has otherwise illegal 02 weight bytes. */ 2009 continue; 2010 } 2011 checkSortKeyValidity(coll, codepoints, chLen); 2012 } 2013 uprv_free(src.source); 2014 } 2015 2016 ucol_close(coll); 2017 count ++; 2018 } 2019 T_FileStream_close(file); 2020 } 2021 2022 /** 2023 * TestSearchCollatorElements tests iterator behavior (forwards and backwards) with 2024 * normalization on AND jamo tailoring, among other things. 2025 */ 2026 static const UChar tsceText[] = { /* Nothing in here should be ignorable */ 2027 0x0020, 0xAC00, /* simple LV Hangul */ 2028 0x0020, 0xAC01, /* simple LVT Hangul */ 2029 0x0020, 0xAC0F, /* LVTT, last jamo expands for search */ 2030 0x0020, 0xAFFF, /* LLVVVTT, every jamo expands for search */ 2031 0x0020, 0x1100, 0x1161, 0x11A8, /* 0xAC01 as conjoining jamo */ 2032 0x0020, 0x3131, 0x314F, 0x3131, /* 0xAC01 as compatibility jamo */ 2033 0x0020, 0x1100, 0x1161, 0x11B6, /* 0xAC0F as conjoining jamo; last expands for search */ 2034 0x0020, 0x1101, 0x1170, 0x11B6, /* 0xAFFF as conjoining jamo; all expand for search */ 2035 0x0020, 0x00E6, /* small letter ae, expands */ 2036 0x0020, 0x1E4D, /* small letter o with tilde and acute, decomposes */ 2037 0x0020 2038 }; 2039 enum { kLen_tsceText = sizeof(tsceText)/sizeof(tsceText[0]) }; 2040 2041 static const int32_t rootStandardOffsets[] = { 2042 0, 1,2, 2043 2, 3,4,4, 2044 4, 5,6,6, 2045 6, 7,8,8, 2046 8, 9,10,11, 2047 12, 13,14,15, 2048 16, 17,18,19, 2049 20, 21,22,23, 2050 24, 25,26,26,26, 2051 26, 27,28,28, 2052 28, 2053 29 2054 }; 2055 enum { kLen_rootStandardOffsets = sizeof(rootStandardOffsets)/sizeof(rootStandardOffsets[0]) }; 2056 2057 static const int32_t rootSearchOffsets[] = { 2058 0, 1,2, 2059 2, 3,4,4, 2060 4, 5,6,6,6, 2061 6, 7,8,8,8,8,8,8, 2062 8, 9,10,11, 2063 12, 13,14,15, 2064 16, 17,18,19,20, 2065 20, 21,22,22,23,23,23,24, 2066 24, 25,26,26,26, 2067 26, 27,28,28, 2068 28, 2069 29 2070 }; 2071 enum { kLen_rootSearchOffsets = sizeof(rootSearchOffsets)/sizeof(rootSearchOffsets[0]) }; 2072 2073 typedef struct { 2074 const char * locale; 2075 const int32_t * offsets; 2076 int32_t offsetsLen; 2077 } TSCEItem; 2078 2079 static const TSCEItem tsceItems[] = { 2080 { "root", rootStandardOffsets, kLen_rootStandardOffsets }, 2081 { "root@collation=search", rootSearchOffsets, kLen_rootSearchOffsets }, 2082 { NULL, NULL, 0 } 2083 }; 2084 2085 static void TestSearchCollatorElements(void) 2086 { 2087 const TSCEItem * tsceItemPtr; 2088 for (tsceItemPtr = tsceItems; tsceItemPtr->locale != NULL; tsceItemPtr++) { 2089 UErrorCode status = U_ZERO_ERROR; 2090 UCollator* ucol = ucol_open(tsceItemPtr->locale, &status); 2091 if ( U_SUCCESS(status) ) { 2092 UCollationElements * uce = ucol_openElements(ucol, tsceText, kLen_tsceText, &status); 2093 if ( U_SUCCESS(status) ) { 2094 int32_t offset, element; 2095 const int32_t * nextOffsetPtr; 2096 const int32_t * limitOffsetPtr; 2097 2098 nextOffsetPtr = tsceItemPtr->offsets; 2099 limitOffsetPtr = tsceItemPtr->offsets + tsceItemPtr->offsetsLen; 2100 do { 2101 offset = ucol_getOffset(uce); 2102 element = ucol_next(uce, &status); 2103 if ( element == 0 ) { 2104 log_err("error, locale %s, ucol_next returned element 0\n", tsceItemPtr->locale ); 2105 } 2106 if ( nextOffsetPtr < limitOffsetPtr ) { 2107 if (offset != *nextOffsetPtr) { 2108 log_err("error, locale %s, expected ucol_next -> ucol_getOffset %d, got %d\n", 2109 tsceItemPtr->locale, *nextOffsetPtr, offset ); 2110 nextOffsetPtr = limitOffsetPtr; 2111 break; 2112 } 2113 nextOffsetPtr++; 2114 } else { 2115 log_err("error, locale %s, ucol_next returned more elements than expected\n", tsceItemPtr->locale ); 2116 } 2117 } while ( U_SUCCESS(status) && element != UCOL_NULLORDER ); 2118 if ( nextOffsetPtr < limitOffsetPtr ) { 2119 log_err("error, locale %s, ucol_next returned fewer elements than expected\n", tsceItemPtr->locale ); 2120 } 2121 2122 ucol_setOffset(uce, kLen_tsceText, &status); 2123 status = U_ZERO_ERROR; 2124 nextOffsetPtr = tsceItemPtr->offsets + tsceItemPtr->offsetsLen; 2125 limitOffsetPtr = tsceItemPtr->offsets; 2126 do { 2127 offset = ucol_getOffset(uce); 2128 element = ucol_previous(uce, &status); 2129 if ( element == 0 ) { 2130 log_err("error, locale %s, ucol_previous returned element 0\n", tsceItemPtr->locale ); 2131 } 2132 if ( nextOffsetPtr > limitOffsetPtr ) { 2133 nextOffsetPtr--; 2134 if (offset != *nextOffsetPtr) { 2135 log_err("error, locale %s, expected ucol_previous -> ucol_getOffset %d, got %d\n", 2136 tsceItemPtr->locale, *nextOffsetPtr, offset ); 2137 nextOffsetPtr = limitOffsetPtr; 2138 break; 2139 } 2140 } else { 2141 log_err("error, locale %s, ucol_previous returned more elements than expected\n", tsceItemPtr->locale ); 2142 } 2143 } while ( U_SUCCESS(status) && element != UCOL_NULLORDER ); 2144 if ( nextOffsetPtr > limitOffsetPtr ) { 2145 log_err("error, locale %s, ucol_previous returned fewer elements than expected\n", tsceItemPtr->locale ); 2146 } 2147 2148 ucol_closeElements(uce); 2149 } else { 2150 log_err("error, locale %s, ucol_openElements failed: %s\n", tsceItemPtr->locale, u_errorName(status) ); 2151 } 2152 ucol_close(ucol); 2153 } else { 2154 log_err("error, locale %s, ucol_open failed: %s\n", tsceItemPtr->locale, u_errorName(status) ); 2155 } 2156 } 2157 } 2158 2159 #endif /* #if !UCONFIG_NO_COLLATION */ 2160
