1 /* 2 ********************************************************************** 3 * Copyright (C) 2005-2010, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 */ 7 8 9 #include "unicode/utypes.h" 10 11 #if !UCONFIG_NO_COLLATION 12 13 #include "unicode/unistr.h" 14 #include "unicode/putil.h" 15 #include "unicode/usearch.h" 16 17 #include "cmemory.h" 18 #include "unicode/coll.h" 19 #include "unicode/tblcoll.h" 20 #include "unicode/coleitr.h" 21 #include "unicode/ucoleitr.h" 22 23 #include "unicode/regex.h" // TODO: make conditional on regexp being built. 24 25 #include "unicode/uniset.h" 26 #include "unicode/uset.h" 27 #include "unicode/ustring.h" 28 #include "hash.h" 29 #include "uhash.h" 30 #include "ucol_imp.h" 31 32 #include "intltest.h" 33 #include "ssearch.h" 34 35 #include "unicode/colldata.h" 36 #include "unicode/bmsearch.h" 37 #include "unicode/bms.h" 38 39 #include "xmlparser.h" 40 #include "ucbuf.h" 41 42 #include <stdlib.h> 43 #include <string.h> 44 #include <stdio.h> 45 46 char testId[100]; 47 48 #define TEST_ASSERT(x) {if (!(x)) { \ 49 errln("Failure in file %s, line %d, test ID = \"%s\"", __FILE__, __LINE__, testId);}} 50 51 #define TEST_ASSERT_M(x, m) {if (!(x)) { \ 52 errln("Failure in file %s, line %d. \"%s\"", __FILE__, __LINE__, m);return;}} 53 54 #define TEST_ASSERT_SUCCESS(errcode) {if (U_FAILURE(errcode)) { \ 55 dataerrln("Failure in file %s, line %d, test ID \"%s\", status = \"%s\"", \ 56 __FILE__, __LINE__, testId, u_errorName(errcode));}} 57 58 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0]) 59 #define NEW_ARRAY(type, count) (type *) uprv_malloc((count) * sizeof(type)) 60 #define DELETE_ARRAY(array) uprv_free((void *) (array)) 61 62 //--------------------------------------------------------------------------- 63 // 64 // Test class boilerplate 65 // 66 //--------------------------------------------------------------------------- 67 SSearchTest::SSearchTest() 68 { 69 } 70 71 SSearchTest::~SSearchTest() 72 { 73 } 74 75 void SSearchTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char *params ) 76 { 77 if (exec) logln("TestSuite SSearchTest: "); 78 switch (index) { 79 #if !UCONFIG_NO_BREAK_ITERATION 80 case 0: name = "searchTest"; 81 if (exec) searchTest(); 82 break; 83 84 case 1: name = "offsetTest"; 85 if (exec) offsetTest(); 86 break; 87 88 case 2: name = "monkeyTest"; 89 if (exec) monkeyTest(params); 90 break; 91 92 case 3: name = "bmMonkeyTest"; 93 if (exec) bmMonkeyTest(params); 94 break; 95 96 case 4: name = "boyerMooreTest"; 97 if (exec) boyerMooreTest(); 98 break; 99 100 case 5: name = "goodSuffixTest"; 101 if (exec) goodSuffixTest(); 102 break; 103 104 case 6: name = "searchTime"; 105 if (exec) searchTime(); 106 break; 107 108 case 7: name = "bmsTest"; 109 if (exec) bmsTest(); 110 break; 111 112 case 8: name = "bmSearchTest"; 113 if (exec) bmSearchTest(); 114 break; 115 116 case 9: name = "udhrTest"; 117 if (exec) udhrTest(); 118 break; 119 case 10: name = "stringListTest"; 120 if (exec) stringListTest(); 121 break; 122 #endif 123 default: name = ""; 124 break; //needed to end loop 125 } 126 } 127 128 129 #if !UCONFIG_NO_BREAK_ITERATION 130 131 #define PATH_BUFFER_SIZE 2048 132 const char *SSearchTest::getPath(char buffer[2048], const char *filename) { 133 UErrorCode status = U_ZERO_ERROR; 134 const char *testDataDirectory = IntlTest::getSourceTestData(status); 135 136 if (U_FAILURE(status) || strlen(testDataDirectory) + strlen(filename) + 1 >= PATH_BUFFER_SIZE) { 137 errln("ERROR: getPath() failed - %s", u_errorName(status)); 138 return NULL; 139 } 140 141 strcpy(buffer, testDataDirectory); 142 strcat(buffer, filename); 143 return buffer; 144 } 145 146 147 void SSearchTest::searchTest() 148 { 149 #if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_FILE_IO 150 UErrorCode status = U_ZERO_ERROR; 151 char path[PATH_BUFFER_SIZE]; 152 const char *testFilePath = getPath(path, "ssearch.xml"); 153 154 if (testFilePath == NULL) { 155 return; /* Couldn't get path: error message already output. */ 156 } 157 158 LocalPointer<UXMLParser> parser(UXMLParser::createParser(status)); 159 TEST_ASSERT_SUCCESS(status); 160 LocalPointer<UXMLElement> root(parser->parseFile(testFilePath, status)); 161 TEST_ASSERT_SUCCESS(status); 162 if (U_FAILURE(status)) { 163 return; 164 } 165 166 const UnicodeString *debugTestCase = root->getAttribute("debug"); 167 if (debugTestCase != NULL) { 168 // setenv("USEARCH_DEBUG", "1", 1); 169 } 170 171 172 const UXMLElement *testCase; 173 int32_t tc = 0; 174 175 while((testCase = root->nextChildElement(tc)) != NULL) { 176 177 if (testCase->getTagName().compare("test-case") != 0) { 178 errln("ssearch, unrecognized XML Element in test file"); 179 continue; 180 } 181 const UnicodeString *id = testCase->getAttribute("id"); 182 *testId = 0; 183 if (id != NULL) { 184 id->extract(0, id->length(), testId, sizeof(testId), US_INV); 185 } 186 187 // If debugging test case has been specified and this is not it, skip to next. 188 if (id!=NULL && debugTestCase!=NULL && *id != *debugTestCase) { 189 continue; 190 } 191 // 192 // Get the requested collation strength. 193 // Default is tertiary if the XML attribute is missing from the test case. 194 // 195 const UnicodeString *strength = testCase->getAttribute("strength"); 196 UColAttributeValue collatorStrength = UCOL_PRIMARY; 197 if (strength==NULL) { collatorStrength = UCOL_TERTIARY;} 198 else if (*strength=="PRIMARY") { collatorStrength = UCOL_PRIMARY;} 199 else if (*strength=="SECONDARY") { collatorStrength = UCOL_SECONDARY;} 200 else if (*strength=="TERTIARY") { collatorStrength = UCOL_TERTIARY;} 201 else if (*strength=="QUATERNARY") { collatorStrength = UCOL_QUATERNARY;} 202 else if (*strength=="IDENTICAL") { collatorStrength = UCOL_IDENTICAL;} 203 else { 204 // Bogus value supplied for strength. Shouldn't happen, even from 205 // typos, if the XML source has been validated. 206 // This assert is a little deceiving in that strength can be 207 // any of the allowed values, not just TERTIARY, but it will 208 // do the job of getting the error output. 209 TEST_ASSERT(*strength=="TERTIARY") 210 } 211 212 // 213 // Get the collator normalization flag. Default is UCOL_OFF. 214 // 215 UColAttributeValue normalize = UCOL_OFF; 216 const UnicodeString *norm = testCase->getAttribute("norm"); 217 TEST_ASSERT (norm==NULL || *norm=="ON" || *norm=="OFF"); 218 if (norm!=NULL && *norm=="ON") { 219 normalize = UCOL_ON; 220 } 221 222 // 223 // Get the alternate_handling flag. Default is UCOL_NON_IGNORABLE. 224 // 225 UColAttributeValue alternateHandling = UCOL_NON_IGNORABLE; 226 const UnicodeString *alt = testCase->getAttribute("alternate_handling"); 227 TEST_ASSERT (alt == NULL || *alt == "SHIFTED" || *alt == "NON_IGNORABLE"); 228 if (alt != NULL && *alt == "SHIFTED") { 229 alternateHandling = UCOL_SHIFTED; 230 } 231 232 const UnicodeString defLocale("en"); 233 char clocale[100]; 234 const UnicodeString *locale = testCase->getAttribute("locale"); 235 if (locale == NULL || locale->length()==0) { 236 locale = &defLocale; 237 }; 238 locale->extract(0, locale->length(), clocale, sizeof(clocale), NULL); 239 240 241 UnicodeString text; 242 UnicodeString target; 243 UnicodeString pattern; 244 int32_t expectedMatchStart = -1; 245 int32_t expectedMatchLimit = -1; 246 const UXMLElement *n; 247 int32_t nodeCount = 0; 248 249 n = testCase->getChildElement("pattern"); 250 TEST_ASSERT(n != NULL); 251 if (n==NULL) { 252 continue; 253 } 254 text = n->getText(FALSE); 255 text = text.unescape(); 256 pattern.append(text); 257 nodeCount++; 258 259 n = testCase->getChildElement("pre"); 260 if (n!=NULL) { 261 text = n->getText(FALSE); 262 text = text.unescape(); 263 target.append(text); 264 nodeCount++; 265 } 266 267 n = testCase->getChildElement("m"); 268 if (n!=NULL) { 269 expectedMatchStart = target.length(); 270 text = n->getText(FALSE); 271 text = text.unescape(); 272 target.append(text); 273 expectedMatchLimit = target.length(); 274 nodeCount++; 275 } 276 277 n = testCase->getChildElement("post"); 278 if (n!=NULL) { 279 text = n->getText(FALSE); 280 text = text.unescape(); 281 target.append(text); 282 nodeCount++; 283 } 284 285 // Check that there weren't extra things in the XML 286 TEST_ASSERT(nodeCount == testCase->countChildren()); 287 288 // Open a collator and StringSearch based on the parameters 289 // obtained from the XML. 290 // 291 status = U_ZERO_ERROR; 292 LocalUCollatorPointer collator(ucol_open(clocale, &status)); 293 ucol_setStrength(collator.getAlias(), collatorStrength); 294 ucol_setAttribute(collator.getAlias(), UCOL_NORMALIZATION_MODE, normalize, &status); 295 ucol_setAttribute(collator.getAlias(), UCOL_ALTERNATE_HANDLING, alternateHandling, &status); 296 LocalUStringSearchPointer uss(usearch_openFromCollator(pattern.getBuffer(), pattern.length(), 297 target.getBuffer(), target.length(), 298 collator.getAlias(), 299 NULL, // the break iterator 300 &status)); 301 302 TEST_ASSERT_SUCCESS(status); 303 if (U_FAILURE(status)) { 304 continue; 305 } 306 307 int32_t foundStart = 0; 308 int32_t foundLimit = 0; 309 UBool foundMatch; 310 311 // 312 // Do the search, check the match result against the expected results. 313 // 314 foundMatch= usearch_search(uss.getAlias(), 0, &foundStart, &foundLimit, &status); 315 TEST_ASSERT_SUCCESS(status); 316 if ((foundMatch && expectedMatchStart<0) || 317 (foundStart != expectedMatchStart) || 318 (foundLimit != expectedMatchLimit)) { 319 TEST_ASSERT(FALSE); // ouput generic error position 320 infoln("Found, expected match start = %d, %d \n" 321 "Found, expected match limit = %d, %d", 322 foundStart, expectedMatchStart, foundLimit, expectedMatchLimit); 323 } 324 325 // In case there are other matches... 326 // (should we only do this if the test case passed?) 327 while (foundMatch) { 328 expectedMatchStart = foundStart; 329 expectedMatchLimit = foundLimit; 330 331 foundMatch = usearch_search(uss.getAlias(), foundLimit, &foundStart, &foundLimit, &status); 332 } 333 334 uss.adoptInstead(usearch_openFromCollator(pattern.getBuffer(), pattern.length(), 335 target.getBuffer(), target.length(), 336 collator.getAlias(), 337 NULL, 338 &status)); 339 340 // 341 // Do the backwards search, check the match result against the expected results. 342 // 343 foundMatch= usearch_searchBackwards(uss.getAlias(), target.length(), &foundStart, &foundLimit, &status); 344 TEST_ASSERT_SUCCESS(status); 345 if ((foundMatch && expectedMatchStart<0) || 346 (foundStart != expectedMatchStart) || 347 (foundLimit != expectedMatchLimit)) { 348 TEST_ASSERT(FALSE); // ouput generic error position 349 infoln("Found, expected backwards match start = %d, %d \n" 350 "Found, expected backwards match limit = %d, %d", 351 foundStart, expectedMatchStart, foundLimit, expectedMatchLimit); 352 } 353 } 354 #endif 355 } 356 357 struct UdhrTestCase 358 { 359 const char *locale; 360 const char *file; 361 }; 362 363 void SSearchTest::udhrTest() 364 { 365 UErrorCode status = U_ZERO_ERROR; 366 char path[PATH_BUFFER_SIZE]; 367 const char *udhrPath = getPath(path, "udhr"); 368 369 if (udhrPath == NULL) { 370 // couldn't get path: error message already output... 371 return; 372 } 373 374 UdhrTestCase testCases[] = { 375 {"en", "udhr_eng.txt"}, 376 {"de", "udhr_deu_1996.txt"}, 377 {"fr", "udhr_fra.txt"}, 378 {"ru", "udhr_rus.txt"}, 379 {"th", "udhr_tha.txt"}, 380 {"ja", "udhr_jpn.txt"}, 381 {"ko", "udhr_kor.txt"}, 382 {"zh", "udhr_cmn_hans.txt"}, 383 {"zh_Hant", "udhr_cmn_hant.txt"} 384 }; 385 386 int32_t testCount = ARRAY_SIZE(testCases); 387 388 for (int32_t t = 0; t < testCount; t += 1) { 389 int32_t len = 0; 390 char *resolvedFileName = NULL; 391 const char *encoding = NULL; 392 UCHARBUF *ucharBuf = NULL; 393 394 ucbuf_resolveFileName(udhrPath, testCases[t].file, NULL, &len, &status); 395 resolvedFileName = NEW_ARRAY(char, len); 396 397 if(resolvedFileName == NULL){ 398 continue; 399 } 400 401 if(status == U_BUFFER_OVERFLOW_ERROR){ 402 status = U_ZERO_ERROR; 403 } 404 405 ucbuf_resolveFileName(udhrPath, testCases[t].file, resolvedFileName, &len, &status); 406 ucharBuf = ucbuf_open(resolvedFileName, &encoding, TRUE, FALSE, &status); 407 408 DELETE_ARRAY(resolvedFileName); 409 410 if(U_FAILURE(status)){ 411 infoln("Could not open the input file %s. Test skipped\n", testCases[t].file); 412 continue; 413 } 414 415 int32_t targetLen = 0; 416 const UChar *target = ucbuf_getBuffer(ucharBuf, &targetLen, &status); 417 418 /* The first line of the file contains the pattern */ 419 int32_t start = 0, end = 0, plen = 0; 420 421 for(end = start; ; end += 1) { 422 UChar ch = target[end]; 423 424 if (ch == 0x000A || ch == 0x000D || ch == 0x2028) { 425 break; 426 } 427 } 428 429 plen = end - start; 430 431 UChar *pattern = NEW_ARRAY(UChar, plen); 432 for (int32_t i = 0; i < plen; i += 1) { 433 pattern[i] = target[start++]; 434 } 435 436 int32_t offset = 0; 437 UCollator *coll = ucol_open(testCases[t].locale, &status); 438 UCD *ucd = NULL; 439 BMS *bms = NULL; 440 441 if (U_FAILURE(status)) { 442 errln("Could not open collator for %s", testCases[t].locale); 443 goto delete_collator; 444 } 445 446 ucd = ucd_open(coll, &status); 447 448 if (U_FAILURE(status)) { 449 errln("Could not open CollData object for %s", testCases[t].locale); 450 goto delete_ucd; 451 } 452 453 bms = bms_open(ucd, pattern, plen, target, targetLen, &status); 454 455 if (U_FAILURE(status)) { 456 errln("Could not open search object for %s", testCases[t].locale); 457 goto delete_bms; 458 } 459 460 start = end = -1; 461 while (bms_search(bms, offset, &start, &end)) { 462 offset = end; 463 } 464 465 if (offset == 0) { 466 errln("Could not find pattern - locale: %s, file: %s ", testCases[t].locale, testCases[t].file); 467 } 468 469 delete_bms: 470 bms_close(bms); 471 472 delete_ucd: 473 ucd_close(ucd); 474 475 delete_collator: 476 ucol_close(coll); 477 478 DELETE_ARRAY(pattern); 479 ucbuf_close(ucharBuf); 480 } 481 482 ucd_flushCache(); 483 } 484 485 void SSearchTest::bmSearchTest() 486 { 487 #if !UCONFIG_NO_REGULAR_EXPRESSIONS 488 UErrorCode status = U_ZERO_ERROR; 489 char path[PATH_BUFFER_SIZE]; 490 const char *testFilePath = getPath(path, "ssearch.xml"); 491 492 if (testFilePath == NULL) { 493 return; /* Couldn't get path: error message already output. */ 494 } 495 496 UXMLParser *parser = UXMLParser::createParser(status); 497 TEST_ASSERT_SUCCESS(status); 498 UXMLElement *root = parser->parseFile(testFilePath, status); 499 TEST_ASSERT_SUCCESS(status); 500 if (U_FAILURE(status)) { 501 return; 502 } 503 504 const UnicodeString *debugTestCase = root->getAttribute("debug"); 505 if (debugTestCase != NULL) { 506 // setenv("USEARCH_DEBUG", "1", 1); 507 } 508 509 510 const UXMLElement *testCase; 511 int32_t tc = 0; 512 513 while((testCase = root->nextChildElement(tc)) != NULL) { 514 515 if (testCase->getTagName().compare("test-case") != 0) { 516 errln("ssearch, unrecognized XML Element in test file"); 517 continue; 518 } 519 const UnicodeString *id = testCase->getAttribute("id"); 520 *testId = 0; 521 if (id != NULL) { 522 id->extract(0, id->length(), testId, sizeof(testId), US_INV); 523 } 524 525 // If debugging test case has been specified and this is not it, skip to next. 526 if (id!=NULL && debugTestCase!=NULL && *id != *debugTestCase) { 527 continue; 528 } 529 // 530 // Get the requested collation strength. 531 // Default is tertiary if the XML attribute is missing from the test case. 532 // 533 const UnicodeString *strength = testCase->getAttribute("strength"); 534 UColAttributeValue collatorStrength = UCOL_PRIMARY; 535 if (strength==NULL) { collatorStrength = UCOL_TERTIARY;} 536 else if (*strength=="PRIMARY") { collatorStrength = UCOL_PRIMARY;} 537 else if (*strength=="SECONDARY") { collatorStrength = UCOL_SECONDARY;} 538 else if (*strength=="TERTIARY") { collatorStrength = UCOL_TERTIARY;} 539 else if (*strength=="QUATERNARY") { collatorStrength = UCOL_QUATERNARY;} 540 else if (*strength=="IDENTICAL") { collatorStrength = UCOL_IDENTICAL;} 541 else { 542 // Bogus value supplied for strength. Shouldn't happen, even from 543 // typos, if the XML source has been validated. 544 // This assert is a little deceiving in that strength can be 545 // any of the allowed values, not just TERTIARY, but it will 546 // do the job of getting the error output. 547 TEST_ASSERT(*strength=="TERTIARY") 548 } 549 550 // 551 // Get the collator normalization flag. Default is UCOL_OFF. 552 // 553 UColAttributeValue normalize = UCOL_OFF; 554 const UnicodeString *norm = testCase->getAttribute("norm"); 555 TEST_ASSERT (norm==NULL || *norm=="ON" || *norm=="OFF"); 556 if (norm!=NULL && *norm=="ON") { 557 normalize = UCOL_ON; 558 } 559 560 // 561 // Get the alternate_handling flag. Default is UCOL_NON_IGNORABLE. 562 // 563 UColAttributeValue alternateHandling = UCOL_NON_IGNORABLE; 564 const UnicodeString *alt = testCase->getAttribute("alternate_handling"); 565 TEST_ASSERT (alt == NULL || *alt == "SHIFTED" || *alt == "NON_IGNORABLE"); 566 if (alt != NULL && *alt == "SHIFTED") { 567 alternateHandling = UCOL_SHIFTED; 568 } 569 570 const UnicodeString defLocale("en"); 571 char clocale[100]; 572 const UnicodeString *locale = testCase->getAttribute("locale"); 573 if (locale == NULL || locale->length()==0) { 574 locale = &defLocale; 575 }; 576 locale->extract(0, locale->length(), clocale, sizeof(clocale), NULL); 577 578 579 UnicodeString text; 580 UnicodeString target; 581 UnicodeString pattern; 582 int32_t expectedMatchStart = -1; 583 int32_t expectedMatchLimit = -1; 584 const UXMLElement *n; 585 int32_t nodeCount = 0; 586 587 n = testCase->getChildElement("pattern"); 588 TEST_ASSERT(n != NULL); 589 if (n==NULL) { 590 continue; 591 } 592 text = n->getText(FALSE); 593 text = text.unescape(); 594 pattern.append(text); 595 nodeCount++; 596 597 n = testCase->getChildElement("pre"); 598 if (n!=NULL) { 599 text = n->getText(FALSE); 600 text = text.unescape(); 601 target.append(text); 602 nodeCount++; 603 } 604 605 n = testCase->getChildElement("m"); 606 if (n!=NULL) { 607 expectedMatchStart = target.length(); 608 text = n->getText(FALSE); 609 text = text.unescape(); 610 target.append(text); 611 expectedMatchLimit = target.length(); 612 nodeCount++; 613 } 614 615 n = testCase->getChildElement("post"); 616 if (n!=NULL) { 617 text = n->getText(FALSE); 618 text = text.unescape(); 619 target.append(text); 620 nodeCount++; 621 } 622 623 // Check that there weren't extra things in the XML 624 TEST_ASSERT(nodeCount == testCase->countChildren()); 625 626 // Open a collator and StringSearch based on the parameters 627 // obtained from the XML. 628 // 629 status = U_ZERO_ERROR; 630 UCollator *collator = ucol_open(clocale, &status); 631 ucol_setStrength(collator, collatorStrength); 632 ucol_setAttribute(collator, UCOL_NORMALIZATION_MODE, normalize, &status); 633 ucol_setAttribute(collator, UCOL_ALTERNATE_HANDLING, alternateHandling, &status); 634 UCD *ucd = ucd_open(collator, &status); 635 BMS *bms = bms_open(ucd, pattern.getBuffer(), pattern.length(), target.getBuffer(), target.length(), &status); 636 637 TEST_ASSERT_SUCCESS(status); 638 if (U_FAILURE(status)) { 639 bms_close(bms); 640 ucd_close(ucd); 641 ucol_close(collator); 642 continue; 643 } 644 645 int32_t foundStart = 0; 646 int32_t foundLimit = 0; 647 UBool foundMatch; 648 649 // 650 // Do the search, check the match result against the expected results. 651 // 652 foundMatch = bms_search(bms, 0, &foundStart, &foundLimit); 653 //TEST_ASSERT_SUCCESS(status); 654 if ((foundMatch && expectedMatchStart < 0) || 655 (foundStart != expectedMatchStart) || 656 (foundLimit != expectedMatchLimit)) { 657 TEST_ASSERT(FALSE); // ouput generic error position 658 infoln("Found, expected match start = %d, %d \n" 659 "Found, expected match limit = %d, %d", 660 foundStart, expectedMatchStart, foundLimit, expectedMatchLimit); 661 } 662 663 bms_close(bms); 664 ucd_close(ucd); 665 ucol_close(collator); 666 } 667 668 ucd_flushCache(); 669 delete root; 670 delete parser; 671 #endif 672 } 673 674 struct Order 675 { 676 int32_t order; 677 int32_t lowOffset; 678 int32_t highOffset; 679 }; 680 681 class OrderList 682 { 683 public: 684 OrderList(); 685 OrderList(UCollator *coll, const UnicodeString &string, int32_t stringOffset = 0); 686 ~OrderList(); 687 688 int32_t size(void) const; 689 void add(int32_t order, int32_t low, int32_t high); 690 const Order *get(int32_t index) const; 691 int32_t getLowOffset(int32_t index) const; 692 int32_t getHighOffset(int32_t index) const; 693 int32_t getOrder(int32_t index) const; 694 void reverse(void); 695 UBool compare(const OrderList &other) const; 696 UBool matchesAt(int32_t offset, const OrderList &other) const; 697 698 private: 699 Order *list; 700 int32_t listMax; 701 int32_t listSize; 702 }; 703 704 OrderList::OrderList() 705 : list(NULL), listMax(16), listSize(0) 706 { 707 list = new Order[listMax]; 708 } 709 710 OrderList::OrderList(UCollator *coll, const UnicodeString &string, int32_t stringOffset) 711 : list(NULL), listMax(16), listSize(0) 712 { 713 UErrorCode status = U_ZERO_ERROR; 714 UCollationElements *elems = ucol_openElements(coll, string.getBuffer(), string.length(), &status); 715 uint32_t strengthMask = 0; 716 int32_t order, low, high; 717 718 switch (ucol_getStrength(coll)) 719 { 720 default: 721 strengthMask |= UCOL_TERTIARYORDERMASK; 722 /* fall through */ 723 724 case UCOL_SECONDARY: 725 strengthMask |= UCOL_SECONDARYORDERMASK; 726 /* fall through */ 727 728 case UCOL_PRIMARY: 729 strengthMask |= UCOL_PRIMARYORDERMASK; 730 } 731 732 list = new Order[listMax]; 733 734 ucol_setOffset(elems, stringOffset, &status); 735 736 do { 737 low = ucol_getOffset(elems); 738 order = ucol_next(elems, &status); 739 high = ucol_getOffset(elems); 740 741 if (order != UCOL_NULLORDER) { 742 order &= strengthMask; 743 } 744 745 if (order != UCOL_IGNORABLE) { 746 add(order, low, high); 747 } 748 } while (order != UCOL_NULLORDER); 749 750 ucol_closeElements(elems); 751 } 752 753 OrderList::~OrderList() 754 { 755 delete[] list; 756 } 757 758 void OrderList::add(int32_t order, int32_t low, int32_t high) 759 { 760 if (listSize >= listMax) { 761 listMax *= 2; 762 763 Order *newList = new Order[listMax]; 764 765 uprv_memcpy(newList, list, listSize * sizeof(Order)); 766 delete[] list; 767 list = newList; 768 } 769 770 list[listSize].order = order; 771 list[listSize].lowOffset = low; 772 list[listSize].highOffset = high; 773 774 listSize += 1; 775 } 776 777 const Order *OrderList::get(int32_t index) const 778 { 779 if (index >= listSize) { 780 return NULL; 781 } 782 783 return &list[index]; 784 } 785 786 int32_t OrderList::getLowOffset(int32_t index) const 787 { 788 const Order *order = get(index); 789 790 if (order != NULL) { 791 return order->lowOffset; 792 } 793 794 return -1; 795 } 796 797 int32_t OrderList::getHighOffset(int32_t index) const 798 { 799 const Order *order = get(index); 800 801 if (order != NULL) { 802 return order->highOffset; 803 } 804 805 return -1; 806 } 807 808 int32_t OrderList::getOrder(int32_t index) const 809 { 810 const Order *order = get(index); 811 812 if (order != NULL) { 813 return order->order; 814 } 815 816 return UCOL_NULLORDER; 817 } 818 819 int32_t OrderList::size() const 820 { 821 return listSize; 822 } 823 824 void OrderList::reverse() 825 { 826 for(int32_t f = 0, b = listSize - 1; f < b; f += 1, b -= 1) { 827 Order swap = list[b]; 828 829 list[b] = list[f]; 830 list[f] = swap; 831 } 832 } 833 834 UBool OrderList::compare(const OrderList &other) const 835 { 836 if (listSize != other.listSize) { 837 return FALSE; 838 } 839 840 for(int32_t i = 0; i < listSize; i += 1) { 841 if (list[i].order != other.list[i].order || 842 list[i].lowOffset != other.list[i].lowOffset || 843 list[i].highOffset != other.list[i].highOffset) { 844 return FALSE; 845 } 846 } 847 848 return TRUE; 849 } 850 851 UBool OrderList::matchesAt(int32_t offset, const OrderList &other) const 852 { 853 // NOTE: sizes include the NULLORDER, which we don't want to compare. 854 int32_t otherSize = other.size() - 1; 855 856 if (listSize - 1 - offset < otherSize) { 857 return FALSE; 858 } 859 860 for (int32_t i = offset, j = 0; j < otherSize; i += 1, j += 1) { 861 if (getOrder(i) != other.getOrder(j)) { 862 return FALSE; 863 } 864 } 865 866 return TRUE; 867 } 868 869 static char *printOffsets(char *buffer, OrderList &list) 870 { 871 int32_t size = list.size(); 872 char *s = buffer; 873 874 for(int32_t i = 0; i < size; i += 1) { 875 const Order *order = list.get(i); 876 877 if (i != 0) { 878 s += sprintf(s, ", "); 879 } 880 881 s += sprintf(s, "(%d, %d)", order->lowOffset, order->highOffset); 882 } 883 884 return buffer; 885 } 886 887 static char *printOrders(char *buffer, OrderList &list) 888 { 889 int32_t size = list.size(); 890 char *s = buffer; 891 892 for(int32_t i = 0; i < size; i += 1) { 893 const Order *order = list.get(i); 894 895 if (i != 0) { 896 s += sprintf(s, ", "); 897 } 898 899 s += sprintf(s, "%8.8X", order->order); 900 } 901 902 return buffer; 903 } 904 905 void SSearchTest::offsetTest() 906 { 907 static const UVersionInfo icu47 = { 4, 7, 0, 0 }; 908 const char *test[] = { 909 // The sequence \u0FB3\u0F71\u0F71\u0F80 contains a discontiguous 910 // contraction (\u0FB3\u0F71\u0F80) logically followed by \u0F71. 911 "\\u1E33\\u0FB3\\u0F71\\u0F71\\u0F80\\uD835\\uDF6C\\u01B0", 912 913 "\\ua191\\u16ef\\u2036\\u017a", 914 915 #if 0 916 // This results in a complex interaction between contraction, 917 // expansion and normalization that confuses the backwards offset fixups. 918 "\\u0F7F\\u0F80\\u0F81\\u0F82\\u0F83\\u0F84\\u0F85", 919 #endif 920 921 "\\u0F80\\u0F81\\u0F82\\u0F83\\u0F84\\u0F85", 922 "\\u07E9\\u07EA\\u07F1\\u07F2\\u07F3", 923 924 "\\u02FE\\u02FF" 925 "\\u0300\\u0301\\u0302\\u0303\\u0304\\u0305\\u0306\\u0307\\u0308\\u0309\\u030A\\u030B\\u030C\\u030D\\u030E\\u030F" 926 "\\u0310\\u0311\\u0312\\u0313\\u0314\\u0315\\u0316\\u0317\\u0318\\u0319\\u031A\\u031B\\u031C\\u031D\\u031E\\u031F" 927 "\\u0320\\u0321\\u0322\\u0323\\u0324\\u0325\\u0326\\u0327\\u0328\\u0329\\u032A\\u032B\\u032C\\u032D\\u032E\\u032F" 928 "\\u0330\\u0331\\u0332\\u0333\\u0334\\u0335\\u0336\\u0337\\u0338\\u0339\\u033A\\u033B\\u033C\\u033D\\u033E\\u033F" 929 "\\u0340\\u0341\\u0342\\u0343\\u0344\\u0345\\u0346\\u0347\\u0348\\u0349\\u034A\\u034B\\u034C\\u034D\\u034E", // currently not working, see #8081 930 931 "\\u02FE\\u02FF\\u0300\\u0301\\u0302\\u0303\\u0316\\u0317\\u0318", // currently not working, see #8081 932 "a\\u02FF\\u0301\\u0316", // currently not working, see #8081 933 "a\\u02FF\\u0316\\u0301", 934 "a\\u0430\\u0301\\u0316", 935 "a\\u0430\\u0316\\u0301", 936 "abc\\u0E41\\u0301\\u0316", 937 "abc\\u0E41\\u0316\\u0301", 938 "\\u0E41\\u0301\\u0316", 939 "\\u0E41\\u0316\\u0301", 940 "a\\u0301\\u0316", 941 "a\\u0316\\u0301", 942 "\\uAC52\\uAC53", 943 "\\u34CA\\u34CB", 944 "\\u11ED\\u11EE", 945 "\\u30C3\\u30D0", 946 "p\\u00E9ch\\u00E9", 947 "a\\u0301\\u0325", 948 "a\\u0300\\u0325", 949 "a\\u0325\\u0300", 950 "A\\u0323\\u0300B", 951 "A\\u0300\\u0323B", 952 "A\\u0301\\u0323B", 953 "A\\u0302\\u0301\\u0323B", 954 "abc", 955 "ab\\u0300c", 956 "ab\\u0300\\u0323c", 957 " \\uD800\\uDC00\\uDC00", 958 "a\\uD800\\uDC00\\uDC00", 959 "A\\u0301\\u0301", 960 "A\\u0301\\u0323", 961 "A\\u0301\\u0323B", 962 "B\\u0301\\u0323C", 963 "A\\u0300\\u0323B", 964 "\\u0301A\\u0301\\u0301", 965 "abcd\\r\\u0301", 966 "p\\u00EAche", 967 "pe\\u0302che", 968 }; 969 970 int32_t testCount = ARRAY_SIZE(test); 971 UErrorCode status = U_ZERO_ERROR; 972 RuleBasedCollator *col = (RuleBasedCollator *) Collator::createInstance(Locale::getEnglish(), status); 973 if (U_FAILURE(status)) { 974 errcheckln(status, "Failed to create collator in offsetTest! - %s", u_errorName(status)); 975 return; 976 } 977 char buffer[4096]; // A bit of a hack... just happens to be long enough for all the test cases... 978 // We could allocate one that's the right size by (CE_count * 10) + 2 979 // 10 chars is enough room for 8 hex digits plus ", ". 2 extra chars for "[" and "]" 980 981 col->setAttribute(UCOL_NORMALIZATION_MODE, UCOL_ON, status); 982 983 for(int32_t i = 0; i < testCount; i += 1) { 984 if (!isICUVersionAtLeast(icu47) && i>=4 && i<=6) { 985 continue; // timebomb until ticket #8080 is resolved 986 } 987 UnicodeString ts = CharsToUnicodeString(test[i]); 988 CollationElementIterator *iter = col->createCollationElementIterator(ts); 989 OrderList forwardList; 990 OrderList backwardList; 991 int32_t order, low, high; 992 993 do { 994 low = iter->getOffset(); 995 order = iter->next(status); 996 high = iter->getOffset(); 997 998 forwardList.add(order, low, high); 999 } while (order != CollationElementIterator::NULLORDER); 1000 1001 iter->reset(); 1002 iter->setOffset(ts.length(), status); 1003 1004 backwardList.add(CollationElementIterator::NULLORDER, iter->getOffset(), iter->getOffset()); 1005 1006 do { 1007 high = iter->getOffset(); 1008 order = iter->previous(status); 1009 low = iter->getOffset(); 1010 1011 if (order == CollationElementIterator::NULLORDER) { 1012 break; 1013 } 1014 1015 backwardList.add(order, low, high); 1016 } while (TRUE); 1017 1018 backwardList.reverse(); 1019 1020 if (forwardList.compare(backwardList)) { 1021 logln("Works with \"%s\"", test[i]); 1022 logln("Forward offsets: [%s]", printOffsets(buffer, forwardList)); 1023 // logln("Backward offsets: [%s]", printOffsets(buffer, backwardList)); 1024 1025 logln("Forward CEs: [%s]", printOrders(buffer, forwardList)); 1026 // logln("Backward CEs: [%s]", printOrders(buffer, backwardList)); 1027 1028 logln(); 1029 } else { 1030 errln("Fails with \"%s\"", test[i]); 1031 infoln("Forward offsets: [%s]", printOffsets(buffer, forwardList)); 1032 infoln("Backward offsets: [%s]", printOffsets(buffer, backwardList)); 1033 1034 infoln("Forward CEs: [%s]", printOrders(buffer, forwardList)); 1035 infoln("Backward CEs: [%s]", printOrders(buffer, backwardList)); 1036 1037 infoln(); 1038 } 1039 delete iter; 1040 } 1041 delete col; 1042 } 1043 1044 #if 0 1045 static UnicodeString &escape(const UnicodeString &string, UnicodeString &buffer) 1046 { 1047 for(int32_t i = 0; i < string.length(); i += 1) { 1048 UChar32 ch = string.char32At(i); 1049 1050 if (ch >= 0x0020 && ch <= 0x007F) { 1051 if (ch == 0x005C) { 1052 buffer.append("\\\\"); 1053 } else { 1054 buffer.append(ch); 1055 } 1056 } else { 1057 char cbuffer[12]; 1058 1059 if (ch <= 0xFFFFL) { 1060 sprintf(cbuffer, "\\u%4.4X", ch); 1061 } else { 1062 sprintf(cbuffer, "\\U%8.8X", ch); 1063 } 1064 1065 buffer.append(cbuffer); 1066 } 1067 1068 if (ch >= 0x10000L) { 1069 i += 1; 1070 } 1071 } 1072 1073 return buffer; 1074 } 1075 #endif 1076 1077 #if 1 1078 1079 struct PCE 1080 { 1081 uint64_t ce; 1082 int32_t lowOffset; 1083 int32_t highOffset; 1084 }; 1085 1086 class PCEList 1087 { 1088 public: 1089 PCEList(UCollator *coll, const UnicodeString &string); 1090 ~PCEList(); 1091 1092 int32_t size() const; 1093 1094 const PCE *get(int32_t index) const; 1095 1096 int32_t getLowOffset(int32_t index) const; 1097 int32_t getHighOffset(int32_t index) const; 1098 uint64_t getOrder(int32_t index) const; 1099 1100 UBool matchesAt(int32_t offset, const PCEList &other) const; 1101 1102 uint64_t operator[](int32_t index) const; 1103 1104 private: 1105 void add(uint64_t ce, int32_t low, int32_t high); 1106 1107 PCE *list; 1108 int32_t listMax; 1109 int32_t listSize; 1110 }; 1111 1112 PCEList::PCEList(UCollator *coll, const UnicodeString &string) 1113 { 1114 UErrorCode status = U_ZERO_ERROR; 1115 UCollationElements *elems = ucol_openElements(coll, string.getBuffer(), string.length(), &status); 1116 uint64_t order; 1117 int32_t low, high; 1118 1119 list = new PCE[listMax]; 1120 1121 ucol_setOffset(elems, 0, &status); 1122 1123 do { 1124 order = ucol_nextProcessed(elems, &low, &high, &status); 1125 add(order, low, high); 1126 } while (order != UCOL_PROCESSED_NULLORDER); 1127 1128 ucol_closeElements(elems); 1129 } 1130 1131 PCEList::~PCEList() 1132 { 1133 delete[] list; 1134 } 1135 1136 void PCEList::add(uint64_t order, int32_t low, int32_t high) 1137 { 1138 if (listSize >= listMax) { 1139 listMax *= 2; 1140 1141 PCE *newList = new PCE[listMax]; 1142 1143 uprv_memcpy(newList, list, listSize * sizeof(Order)); 1144 delete[] list; 1145 list = newList; 1146 } 1147 1148 list[listSize].ce = order; 1149 list[listSize].lowOffset = low; 1150 list[listSize].highOffset = high; 1151 1152 listSize += 1; 1153 } 1154 1155 const PCE *PCEList::get(int32_t index) const 1156 { 1157 if (index >= listSize) { 1158 return NULL; 1159 } 1160 1161 return &list[index]; 1162 } 1163 1164 int32_t PCEList::getLowOffset(int32_t index) const 1165 { 1166 const PCE *pce = get(index); 1167 1168 if (pce != NULL) { 1169 return pce->lowOffset; 1170 } 1171 1172 return -1; 1173 } 1174 1175 int32_t PCEList::getHighOffset(int32_t index) const 1176 { 1177 const PCE *pce = get(index); 1178 1179 if (pce != NULL) { 1180 return pce->highOffset; 1181 } 1182 1183 return -1; 1184 } 1185 1186 uint64_t PCEList::getOrder(int32_t index) const 1187 { 1188 const PCE *pce = get(index); 1189 1190 if (pce != NULL) { 1191 return pce->ce; 1192 } 1193 1194 return UCOL_PROCESSED_NULLORDER; 1195 } 1196 1197 int32_t PCEList::size() const 1198 { 1199 return listSize; 1200 } 1201 1202 UBool PCEList::matchesAt(int32_t offset, const PCEList &other) const 1203 { 1204 // NOTE: sizes include the NULLORDER, which we don't want to compare. 1205 int32_t otherSize = other.size() - 1; 1206 1207 if (listSize - 1 - offset < otherSize) { 1208 return FALSE; 1209 } 1210 1211 for (int32_t i = offset, j = 0; j < otherSize; i += 1, j += 1) { 1212 if (getOrder(i) != other.getOrder(j)) { 1213 return FALSE; 1214 } 1215 } 1216 1217 return TRUE; 1218 } 1219 1220 uint64_t PCEList::operator[](int32_t index) const 1221 { 1222 return getOrder(index); 1223 } 1224 1225 void SSearchTest::boyerMooreTest() 1226 { 1227 UErrorCode status = U_ZERO_ERROR; 1228 UCollator *coll = NULL; 1229 CollData *data = NULL; 1230 const CEList* ce = NULL; 1231 const CEList* ce1 = NULL; 1232 UnicodeString lp = "fuss"; 1233 UnicodeString sp = "fu\\u00DF"; 1234 BoyerMooreSearch *longPattern = NULL; 1235 BoyerMooreSearch *shortPattern = NULL; 1236 UnicodeString targets[] = {"fu\\u00DF", "fu\\u00DFball", "1fu\\u00DFball", "12fu\\u00DFball", "123fu\\u00DFball", "1234fu\\u00DFball", 1237 "ffu\\u00DF", "fufu\\u00DF", "fusfu\\u00DF", 1238 "fuss", "ffuss", "fufuss", "fusfuss", "1fuss", "12fuss", "123fuss", "1234fuss", "fu\\u00DF", "1fu\\u00DF", "12fu\\u00DF", "123fu\\u00DF", "1234fu\\u00DF"}; 1239 int32_t start = -1, end = -1; 1240 1241 coll = ucol_openFromShortString("LEN_S1", FALSE, NULL, &status); 1242 if (U_FAILURE(status)) { 1243 errcheckln(status, "Could not open collator. - %s", u_errorName(status)); 1244 return; 1245 } 1246 1247 data = CollData::open(coll, status); 1248 if (U_FAILURE(status)) { 1249 errln("Could not open CollData object."); 1250 goto close_data; 1251 } 1252 1253 data->getDynamicClassID(); 1254 if (U_FAILURE(status)) { 1255 errln("Could not get dynamic class ID of CollData."); 1256 goto close_patterns; 1257 } 1258 1259 data->getStaticClassID(); 1260 if (U_FAILURE(status)) { 1261 errln("Could not get static class ID of CollData."); 1262 goto close_patterns; 1263 } 1264 1265 longPattern = new BoyerMooreSearch(data, lp.unescape(), NULL, status); 1266 shortPattern = new BoyerMooreSearch(data, sp.unescape(), NULL, status); 1267 if (U_FAILURE(status)) { 1268 errln("Could not create pattern objects."); 1269 goto close_patterns; 1270 } 1271 1272 longPattern->getBadCharacterTable(); 1273 shortPattern->getBadCharacterTable(); 1274 if (U_FAILURE(status)) { 1275 errln("Could not get bad character table."); 1276 goto close_patterns; 1277 } 1278 1279 longPattern->getGoodSuffixTable(); 1280 shortPattern->getGoodSuffixTable(); 1281 if (U_FAILURE(status)) { 1282 errln("Could not get good suffix table."); 1283 goto close_patterns; 1284 } 1285 1286 longPattern->getDynamicClassID(); 1287 shortPattern->getDynamicClassID(); 1288 if (U_FAILURE(status)) { 1289 errln("Could not get dynamic class ID of BoyerMooreSearch."); 1290 goto close_patterns; 1291 } 1292 1293 longPattern->getStaticClassID(); 1294 shortPattern->getStaticClassID(); 1295 if (U_FAILURE(status)) { 1296 errln("Could not get static class ID of BoyerMooreSearch."); 1297 goto close_patterns; 1298 } 1299 1300 longPattern->getData(); 1301 shortPattern->getData(); 1302 if (U_FAILURE(status)) { 1303 errln("Could not get collate data."); 1304 goto close_patterns; 1305 } 1306 1307 ce = longPattern->getPatternCEs(); 1308 ce1 = shortPattern->getPatternCEs(); 1309 if (U_FAILURE(status)) { 1310 errln("Could not get pattern CEs."); 1311 goto close_patterns; 1312 } 1313 1314 ce->getDynamicClassID(); 1315 ce1->getDynamicClassID(); 1316 if (U_FAILURE(status)) { 1317 errln("Could not get dynamic class ID of CEList."); 1318 goto close_patterns; 1319 } 1320 1321 ce->getStaticClassID(); 1322 ce1->getStaticClassID(); 1323 if (U_FAILURE(status)) { 1324 errln("Could not get static class ID of CEList."); 1325 goto close_patterns; 1326 } 1327 1328 if(data->minLengthInChars(ce,0) != 3){ 1329 errln("Minimal Length in Characters for 'data' with 'ce' was suppose to give 3."); 1330 goto close_patterns; 1331 } 1332 1333 if(data->minLengthInChars(ce1,0) != 3){ 1334 errln("Minimal Length in Characters for 'data' with 'ce1' was suppose to give 3."); 1335 goto close_patterns; 1336 } 1337 1338 for (uint32_t t = 0; t < (sizeof(targets)/sizeof(targets[0])); t += 1) { 1339 UnicodeString target = targets[t].unescape(); 1340 1341 longPattern->setTargetString(&target, status); 1342 if (longPattern->search(0, start, end)) { 1343 logln("Test %d: found long pattern at [%d, %d].", t, start, end); 1344 } else { 1345 errln("Test %d: did not find long pattern.", t); 1346 } 1347 1348 shortPattern->setTargetString(&target, status); 1349 if (shortPattern->search(0, start, end)) { 1350 logln("Test %d: found short pattern at [%d, %d].", t, start, end); 1351 } else { 1352 errln("Test %d: did not find short pattern.", t); 1353 } 1354 1355 if(longPattern->empty()){ 1356 errln("Test %d: Long pattern should not have been empty."); 1357 } 1358 1359 if(shortPattern->empty()){ 1360 errln("Test %d: Short pattern should not have been empty."); 1361 } 1362 } 1363 1364 close_patterns: 1365 delete shortPattern; 1366 delete longPattern; 1367 1368 close_data: 1369 CollData::close(data); 1370 ucol_close(coll); 1371 } 1372 1373 void SSearchTest::bmsTest() 1374 { 1375 UErrorCode status = U_ZERO_ERROR; 1376 UCollator *coll = NULL; 1377 UCD *data = NULL; 1378 UnicodeString lp = "fuss"; 1379 UnicodeString lpu = lp.unescape(); 1380 UnicodeString sp = "fu\\u00DF"; 1381 UnicodeString spu = sp.unescape(); 1382 BMS *longPattern = NULL; 1383 BMS *shortPattern = NULL; 1384 UnicodeString targets[] = {"fu\\u00DF", "fu\\u00DFball", "1fu\\u00DFball", "12fu\\u00DFball", "123fu\\u00DFball", "1234fu\\u00DFball", 1385 "ffu\\u00DF", "fufu\\u00DF", "fusfu\\u00DF", 1386 "fuss", "ffuss", "fufuss", "fusfuss", "1fuss", "12fuss", "123fuss", "1234fuss", "fu\\u00DF", "1fu\\u00DF", "12fu\\u00DF", "123fu\\u00DF", "1234fu\\u00DF"}; 1387 int32_t start = -1, end = -1; 1388 1389 coll = ucol_openFromShortString("LEN_S1", FALSE, NULL, &status); 1390 if (U_FAILURE(status)) { 1391 errcheckln(status, "Could not open collator. - %s", u_errorName(status)); 1392 return; 1393 } 1394 1395 data = ucd_open(coll, &status); 1396 if (U_FAILURE(status)) { 1397 errln("Could not open CollData object."); 1398 goto close_data; 1399 } 1400 1401 longPattern = bms_open(data, lpu.getBuffer(), lpu.length(), NULL, 0, &status); 1402 shortPattern = bms_open(data, spu.getBuffer(), spu.length(), NULL, 0, &status); 1403 if (U_FAILURE(status)) { 1404 errln("Couldn't open pattern objects."); 1405 goto close_patterns; 1406 } 1407 1408 for (uint32_t t = 0; t < (sizeof(targets)/sizeof(targets[0])); t += 1) { 1409 UnicodeString target = targets[t].unescape(); 1410 1411 bms_setTargetString(longPattern, target.getBuffer(), target.length(), &status); 1412 if (bms_search(longPattern, 0, &start, &end)) { 1413 logln("Test %d: found long pattern at [%d, %d].", t, start, end); 1414 } else { 1415 errln("Test %d: did not find long pattern.", t); 1416 } 1417 1418 bms_setTargetString(shortPattern, target.getBuffer(), target.length(), &status); 1419 if (bms_search(shortPattern, 0, &start, &end)) { 1420 logln("Test %d: found short pattern at [%d, %d].", t, start, end); 1421 } else { 1422 errln("Test %d: did not find short pattern.", t); 1423 } 1424 } 1425 1426 /* Add better coverage for bms code. */ 1427 if(bms_empty(longPattern)) { 1428 errln("FAIL: longgPattern is empty."); 1429 } 1430 1431 if (!bms_getData(longPattern)) { 1432 errln("FAIL: bms_getData returned NULL."); 1433 } 1434 1435 if (!ucd_getCollator(data)) { 1436 errln("FAIL: ucd_getCollator returned NULL."); 1437 } 1438 1439 close_patterns: 1440 bms_close(shortPattern); 1441 bms_close(longPattern); 1442 1443 close_data: 1444 ucd_close(data); 1445 ucd_freeCache(); 1446 ucol_close(coll); 1447 } 1448 1449 void SSearchTest::goodSuffixTest() 1450 { 1451 UErrorCode status = U_ZERO_ERROR; 1452 UCollator *coll = NULL; 1453 CollData *data = NULL; 1454 UnicodeString pat = /*"gcagagag"*/ "fxeld"; 1455 UnicodeString target = /*"gcatcgcagagagtatacagtacg"*/ "cloveldfxeld"; 1456 BoyerMooreSearch *pattern = NULL; 1457 int32_t start = -1, end = -1; 1458 1459 coll = ucol_open(NULL, &status); 1460 if (U_FAILURE(status)) { 1461 errcheckln(status, "Couldn't open collator. - %s", u_errorName(status)); 1462 return; 1463 } 1464 1465 data = CollData::open(coll, status); 1466 if (U_FAILURE(status)) { 1467 errln("Couldn't open CollData object."); 1468 goto close_data; 1469 } 1470 1471 pattern = new BoyerMooreSearch(data, pat, &target, status); 1472 if (U_FAILURE(status)) { 1473 errln("Couldn't open pattern object."); 1474 goto close_pattern; 1475 } 1476 1477 if (pattern->search(0, start, end)) { 1478 logln("Found pattern at [%d, %d].", start, end); 1479 } else { 1480 errln("Did not find pattern."); 1481 } 1482 1483 close_pattern: 1484 delete pattern; 1485 1486 close_data: 1487 CollData::close(data); 1488 ucol_close(coll); 1489 } 1490 1491 // 1492 // searchTime() A quick and dirty performance test for string search. 1493 // Probably doesn't really belong as part of intltest, but it 1494 // does check that the search succeeds, and gets the right result, 1495 // so it serves as a functionality test also. 1496 // 1497 // To run as a perf test, up the loop count, select by commenting 1498 // and uncommenting in the code the operation to be measured, 1499 // rebuild, and measure the running time of this test alone. 1500 // 1501 // time LD_LIBRARY_PATH=whatever ./intltest collate/SSearchTest/searchTime 1502 // 1503 void SSearchTest::searchTime() { 1504 static const char *longishText = 1505 "Whylom, as olde stories tellen us,\n" 1506 "Ther was a duk that highte Theseus:\n" 1507 "Of Athenes he was lord and governour,\n" 1508 "And in his tyme swich a conquerour,\n" 1509 "That gretter was ther noon under the sonne.\n" 1510 "Ful many a riche contree hadde he wonne;\n" 1511 "What with his wisdom and his chivalrye,\n" 1512 "He conquered al the regne of Femenye,\n" 1513 "That whylom was y-cleped Scithia;\n" 1514 "And weddede the quene Ipolita,\n" 1515 "And broghte hir hoom with him in his contree\n" 1516 "With muchel glorie and greet solempnitee,\n" 1517 "And eek hir yonge suster Emelye.\n" 1518 "And thus with victorie and with melodye\n" 1519 "Lete I this noble duk to Athenes ryde,\n" 1520 "And al his hoost, in armes, him bisyde.\n" 1521 "And certes, if it nere to long to here,\n" 1522 "I wolde han told yow fully the manere,\n" 1523 "How wonnen was the regne of Femenye\n" 1524 "By Theseus, and by his chivalrye;\n" 1525 "And of the grete bataille for the nones\n" 1526 "Bitwixen Athen's and Amazones;\n" 1527 "And how asseged was Ipolita,\n" 1528 "The faire hardy quene of Scithia;\n" 1529 "And of the feste that was at hir weddinge,\n" 1530 "And of the tempest at hir hoom-cominge;\n" 1531 "But al that thing I moot as now forbere.\n" 1532 "I have, God woot, a large feeld to ere,\n" 1533 "And wayke been the oxen in my plough.\n" 1534 "The remenant of the tale is long y-nough.\n" 1535 "I wol nat letten eek noon of this route;\n" 1536 "Lat every felawe telle his tale aboute,\n" 1537 "And lat see now who shal the soper winne;\n" 1538 "And ther I lefte, I wol ageyn biginne.\n" 1539 "This duk, of whom I make mencioun,\n" 1540 "When he was come almost unto the toun,\n" 1541 "In al his wele and in his moste pryde,\n" 1542 "He was war, as he caste his eye asyde,\n" 1543 "Wher that ther kneled in the hye weye\n" 1544 "A companye of ladies, tweye and tweye,\n" 1545 "Ech after other, clad in clothes blake; \n" 1546 "But swich a cry and swich a wo they make,\n" 1547 "That in this world nis creature livinge,\n" 1548 "That herde swich another weymentinge;\n" 1549 "And of this cry they nolde never stenten,\n" 1550 "Til they the reynes of his brydel henten.\n" 1551 "'What folk ben ye, that at myn hoomcominge\n" 1552 "Perturben so my feste with cryinge'?\n" 1553 "Quod Theseus, 'have ye so greet envye\n" 1554 "Of myn honour, that thus compleyne and crye? \n" 1555 "Or who hath yow misboden, or offended?\n" 1556 "And telleth me if it may been amended;\n" 1557 "And why that ye ben clothed thus in blak'?\n" 1558 "The eldest lady of hem alle spak,\n" 1559 "When she hadde swowned with a deedly chere,\n" 1560 "That it was routhe for to seen and here,\n" 1561 "And seyde: 'Lord, to whom Fortune hath yiven\n" 1562 "Victorie, and as a conquerour to liven,\n" 1563 "Noght greveth us your glorie and your honour;\n" 1564 "But we biseken mercy and socour.\n" 1565 "Have mercy on our wo and our distresse.\n" 1566 "Som drope of pitee, thurgh thy gentilesse,\n" 1567 "Up-on us wrecched wommen lat thou falle.\n" 1568 "For certes, lord, ther nis noon of us alle,\n" 1569 "That she nath been a duchesse or a quene;\n" 1570 "Now be we caitifs, as it is wel sene:\n" 1571 "Thanked be Fortune, and hir false wheel,\n" 1572 "That noon estat assureth to be weel.\n" 1573 "And certes, lord, t'abyden your presence,\n" 1574 "Here in the temple of the goddesse Clemence\n" 1575 "We han ben waytinge al this fourtenight;\n" 1576 "Now help us, lord, sith it is in thy might.\n" 1577 "I wrecche, which that wepe and waille thus,\n" 1578 "Was whylom wyf to king Capaneus,\n" 1579 "That starf at Thebes, cursed be that day!\n" 1580 "And alle we, that been in this array,\n" 1581 "And maken al this lamentacioun,\n" 1582 "We losten alle our housbondes at that toun,\n" 1583 "Whyl that the sege ther-aboute lay.\n" 1584 "And yet now th'olde Creon, weylaway!\n" 1585 "The lord is now of Thebes the citee, \n" 1586 "Fulfild of ire and of iniquitee,\n" 1587 "He, for despyt, and for his tirannye,\n" 1588 "To do the dede bodyes vileinye,\n" 1589 "Of alle our lordes, whiche that ben slawe,\n" 1590 "Hath alle the bodyes on an heep y-drawe,\n" 1591 "And wol nat suffren hem, by noon assent,\n" 1592 "Neither to been y-buried nor y-brent,\n" 1593 "But maketh houndes ete hem in despyt. zet'\n"; 1594 1595 #define TEST_BOYER_MOORE 1 1596 const char *cPattern = "maketh houndes ete hem"; 1597 //const char *cPattern = "Whylom"; 1598 //const char *cPattern = "zet"; 1599 const char *testId = "searchTime()"; // for error macros. 1600 UnicodeString target = longishText; 1601 UErrorCode status = U_ZERO_ERROR; 1602 1603 1604 LocalUCollatorPointer collator(ucol_open("en", &status)); 1605 CollData *data = CollData::open(collator.getAlias(), status); 1606 if (U_FAILURE(status) || collator.isNull() || data == NULL) { 1607 errcheckln(status, "Unable to open UCollator or CollData. - %s", u_errorName(status)); 1608 return; 1609 } 1610 //ucol_setStrength(collator.getAlias(), collatorStrength); 1611 //ucol_setAttribute(collator.getAlias(), UCOL_NORMALIZATION_MODE, normalize, &status); 1612 UnicodeString uPattern = cPattern; 1613 #ifndef TEST_BOYER_MOORE 1614 LocalUStringSearchPointer uss(usearch_openFromCollator(uPattern.getBuffer(), uPattern.length(), 1615 target.getBuffer(), target.length(), 1616 collator.getAlias(), 1617 NULL, // the break iterator 1618 &status)); 1619 TEST_ASSERT_SUCCESS(status); 1620 #else 1621 BoyerMooreSearch bms(data, uPattern, &target, status); 1622 TEST_ASSERT_SUCCESS(status); 1623 #endif 1624 1625 // int32_t foundStart; 1626 // int32_t foundEnd; 1627 UBool found; 1628 1629 // Find the match position usgin strstr 1630 const char *pm = strstr(longishText, cPattern); 1631 TEST_ASSERT_M(pm!=NULL, "No pattern match with strstr"); 1632 int32_t refMatchPos = (int32_t)(pm - longishText); 1633 int32_t icuMatchPos; 1634 int32_t icuMatchEnd; 1635 #ifndef TEST_BOYER_MOORE 1636 usearch_search(uss.getAlias(), 0, &icuMatchPos, &icuMatchEnd, &status); 1637 TEST_ASSERT_SUCCESS(status); 1638 #else 1639 found = bms.search(0, icuMatchPos, icuMatchEnd); 1640 #endif 1641 TEST_ASSERT_M(refMatchPos == icuMatchPos, "strstr and icu give different match positions."); 1642 1643 int32_t i; 1644 int32_t j=0; 1645 1646 // Try loopcounts around 100000 to some millions, depending on the operation, 1647 // to get runtimes of at least several seconds. 1648 for (i=0; i<10000; i++) { 1649 #ifndef TEST_BOYER_MOORE 1650 found = usearch_search(uss.getAlias(), 0, &icuMatchPos, &icuMatchEnd, &status); 1651 #else 1652 found = bms.search(0, icuMatchPos, icuMatchEnd); 1653 #endif 1654 //TEST_ASSERT_SUCCESS(status); 1655 //TEST_ASSERT(found); 1656 1657 // usearch_setOffset(uss.getAlias(), 0, &status); 1658 // icuMatchPos = usearch_next(uss.getAlias(), &status); 1659 1660 // The i+j stuff is to confuse the optimizer and get it to actually leave the 1661 // call to strstr in place. 1662 //pm = strstr(longishText+j, cPattern); 1663 //j = (j + i)%5; 1664 } 1665 1666 printf("%ld, %d\n", pm-longishText, j); 1667 #ifdef TEST_BOYER_MOORE 1668 CollData::close(data); 1669 #endif 1670 } 1671 #endif 1672 1673 //---------------------------------------------------------------------------------------- 1674 // 1675 // Random Numbers. Similar to standard lib rand() and srand() 1676 // Not using library to 1677 // 1. Get same results on all platforms. 1678 // 2. Get access to current seed, to more easily reproduce failures. 1679 // 1680 //--------------------------------------------------------------------------------------- 1681 static uint32_t m_seed = 1; 1682 1683 static uint32_t m_rand() 1684 { 1685 m_seed = m_seed * 1103515245 + 12345; 1686 return (uint32_t)(m_seed/65536) % 32768; 1687 } 1688 1689 class Monkey 1690 { 1691 public: 1692 virtual void append(UnicodeString &test, UnicodeString &alternate) = 0; 1693 1694 protected: 1695 Monkey(); 1696 virtual ~Monkey(); 1697 }; 1698 1699 Monkey::Monkey() 1700 { 1701 // ook? 1702 } 1703 1704 Monkey::~Monkey() 1705 { 1706 // ook? 1707 } 1708 1709 class SetMonkey : public Monkey 1710 { 1711 public: 1712 SetMonkey(const USet *theSet); 1713 ~SetMonkey(); 1714 1715 virtual void append(UnicodeString &test, UnicodeString &alternate); 1716 1717 private: 1718 const USet *set; 1719 }; 1720 1721 SetMonkey::SetMonkey(const USet *theSet) 1722 : Monkey(), set(theSet) 1723 { 1724 // ook? 1725 } 1726 1727 SetMonkey::~SetMonkey() 1728 { 1729 //ook... 1730 } 1731 1732 void SetMonkey::append(UnicodeString &test, UnicodeString &alternate) 1733 { 1734 int32_t size = uset_size(set); 1735 int32_t index = m_rand() % size; 1736 UChar32 ch = uset_charAt(set, index); 1737 UnicodeString str(ch); 1738 1739 test.append(str); 1740 alternate.append(str); // flip case, or some junk? 1741 } 1742 1743 class StringSetMonkey : public Monkey 1744 { 1745 public: 1746 StringSetMonkey(const USet *theSet, UCollator *theCollator, CollData *theCollData); 1747 ~StringSetMonkey(); 1748 1749 void append(UnicodeString &testCase, UnicodeString &alternate); 1750 1751 private: 1752 UnicodeString &generateAlternative(const UnicodeString &testCase, UnicodeString &alternate); 1753 1754 const USet *set; 1755 UCollator *coll; 1756 CollData *collData; 1757 }; 1758 1759 StringSetMonkey::StringSetMonkey(const USet *theSet, UCollator *theCollator, CollData *theCollData) 1760 : Monkey(), set(theSet), coll(theCollator), collData(theCollData) 1761 { 1762 // ook. 1763 } 1764 1765 StringSetMonkey::~StringSetMonkey() 1766 { 1767 // ook? 1768 } 1769 1770 void StringSetMonkey::append(UnicodeString &testCase, UnicodeString &alternate) 1771 { 1772 int32_t itemCount = uset_getItemCount(set), len = 0; 1773 int32_t index = m_rand() % itemCount; 1774 UChar32 rangeStart = 0, rangeEnd = 0; 1775 UChar buffer[16]; 1776 UErrorCode err = U_ZERO_ERROR; 1777 1778 len = uset_getItem(set, index, &rangeStart, &rangeEnd, buffer, 16, &err); 1779 1780 if (len == 0) { 1781 int32_t offset = m_rand() % (rangeEnd - rangeStart + 1); 1782 UChar32 ch = rangeStart + offset; 1783 UnicodeString str(ch); 1784 1785 testCase.append(str); 1786 generateAlternative(str, alternate); 1787 } else if (len > 0) { 1788 // should check that len < 16... 1789 UnicodeString str(buffer, len); 1790 1791 testCase.append(str); 1792 generateAlternative(str, alternate); 1793 } else { 1794 // shouldn't happen... 1795 } 1796 } 1797 1798 UnicodeString &StringSetMonkey::generateAlternative(const UnicodeString &testCase, UnicodeString &alternate) 1799 { 1800 // find out shortest string for the longest sequence of ces. 1801 // needs to be refined to use dynamic programming, but will be roughly right 1802 UErrorCode status = U_ZERO_ERROR; 1803 CEList ceList(coll, testCase, status); 1804 UnicodeString alt; 1805 int32_t offset = 0; 1806 1807 if (ceList.size() == 0) { 1808 return alternate.append(testCase); 1809 } 1810 1811 while (offset < ceList.size()) { 1812 int32_t ce = ceList.get(offset); 1813 const StringList *strings = collData->getStringList(ce); 1814 1815 if (strings == NULL) { 1816 return alternate.append(testCase); 1817 } 1818 1819 int32_t stringCount = strings->size(); 1820 int32_t tries = 0; 1821 1822 // find random string that generates the same CEList 1823 const CEList *ceList2 = NULL; 1824 const UnicodeString *string = NULL; 1825 UBool matches = FALSE; 1826 1827 do { 1828 int32_t s = m_rand() % stringCount; 1829 1830 if (tries++ > stringCount) { 1831 alternate.append(testCase); 1832 return alternate; 1833 } 1834 1835 string = strings->get(s); 1836 ceList2 = collData->getCEList(string); 1837 matches = ceList.matchesAt(offset, ceList2); 1838 1839 if (! matches) { 1840 collData->freeCEList((CEList *) ceList2); 1841 } 1842 } while (! matches); 1843 1844 alt.append(*string); 1845 offset += ceList2->size(); 1846 collData->freeCEList(ceList2); 1847 } 1848 1849 const CEList altCEs(coll, alt, status); 1850 1851 if (ceList.matchesAt(0, &altCEs)) { 1852 return alternate.append(alt); 1853 } 1854 1855 return alternate.append(testCase); 1856 } 1857 1858 static void generateTestCase(UCollator *coll, Monkey *monkeys[], int32_t monkeyCount, UnicodeString &testCase, UnicodeString &alternate) 1859 { 1860 int32_t pieces = (m_rand() % 4) + 1; 1861 UErrorCode status = U_ZERO_ERROR; 1862 UBool matches; 1863 1864 do { 1865 testCase.remove(); 1866 alternate.remove(); 1867 monkeys[0]->append(testCase, alternate); 1868 1869 for(int32_t piece = 0; piece < pieces; piece += 1) { 1870 int32_t monkey = m_rand() % monkeyCount; 1871 1872 monkeys[monkey]->append(testCase, alternate); 1873 } 1874 1875 const CEList ceTest(coll, testCase, status); 1876 const CEList ceAlt(coll, alternate, status); 1877 1878 matches = ceTest.matchesAt(0, &ceAlt); 1879 } while (! matches); 1880 } 1881 1882 // 1883 // Find the next acceptable boundary following the specified starting index 1884 // in the target text being searched. 1885 // TODO: refine what is an acceptable boundary. For the moment, 1886 // choose the next position not within a combining sequence. 1887 // 1888 #if 0 1889 static int32_t nextBoundaryAfter(const UnicodeString &string, int32_t startIndex) { 1890 const UChar *text = string.getBuffer(); 1891 int32_t textLen = string.length(); 1892 1893 if (startIndex >= textLen) { 1894 return startIndex; 1895 } 1896 1897 UChar32 c; 1898 int32_t i = startIndex; 1899 1900 U16_NEXT(text, i, textLen, c); 1901 1902 // If we are on a control character, stop without looking for combining marks. 1903 // Control characters do not combine. 1904 int32_t gcProperty = u_getIntPropertyValue(c, UCHAR_GRAPHEME_CLUSTER_BREAK); 1905 if (gcProperty==U_GCB_CONTROL || gcProperty==U_GCB_LF || gcProperty==U_GCB_CR) { 1906 return i; 1907 } 1908 1909 // The initial character was not a control, and can thus accept trailing 1910 // combining characters. Advance over however many of them there are. 1911 int32_t indexOfLastCharChecked; 1912 1913 for (;;) { 1914 indexOfLastCharChecked = i; 1915 1916 if (i>=textLen) { 1917 break; 1918 } 1919 1920 U16_NEXT(text, i, textLen, c); 1921 gcProperty = u_getIntPropertyValue(c, UCHAR_GRAPHEME_CLUSTER_BREAK); 1922 1923 if (gcProperty != U_GCB_EXTEND && gcProperty != U_GCB_SPACING_MARK) { 1924 break; 1925 } 1926 } 1927 1928 return indexOfLastCharChecked; 1929 } 1930 #endif 1931 1932 #if 0 1933 static UBool isInCombiningSequence(const UnicodeString &string, int32_t index) { 1934 const UChar *text = string.getBuffer(); 1935 int32_t textLen = string.length(); 1936 1937 if (index>=textLen || index<=0) { 1938 return FALSE; 1939 } 1940 1941 // If the character at the current index is not a GRAPHEME_EXTEND 1942 // then we can not be within a combining sequence. 1943 UChar32 c; 1944 U16_GET(text, 0, index, textLen, c); 1945 int32_t gcProperty = u_getIntPropertyValue(c, UCHAR_GRAPHEME_CLUSTER_BREAK); 1946 if (gcProperty != U_GCB_EXTEND && gcProperty != U_GCB_SPACING_MARK) { 1947 return FALSE; 1948 } 1949 1950 // We are at a combining mark. If the preceding character is anything 1951 // except a CONTROL, CR or LF, we are in a combining sequence. 1952 U16_PREV(text, 0, index, c); 1953 gcProperty = u_getIntPropertyValue(c, UCHAR_GRAPHEME_CLUSTER_BREAK); 1954 1955 return !(gcProperty==U_GCB_CONTROL || gcProperty==U_GCB_LF || gcProperty==U_GCB_CR); 1956 } 1957 #endif 1958 1959 static UBool simpleSearch(UCollator *coll, const UnicodeString &target, int32_t offset, const UnicodeString &pattern, int32_t &matchStart, int32_t &matchEnd) 1960 { 1961 UErrorCode status = U_ZERO_ERROR; 1962 OrderList targetOrders(coll, target, offset); 1963 OrderList patternOrders(coll, pattern); 1964 int32_t targetSize = targetOrders.size() - 1; 1965 int32_t patternSize = patternOrders.size() - 1; 1966 UBreakIterator *charBreakIterator = ubrk_open(UBRK_CHARACTER, ucol_getLocaleByType(coll, ULOC_VALID_LOCALE, &status), 1967 target.getBuffer(), target.length(), &status); 1968 1969 if (patternSize == 0) { 1970 // Searching for an empty pattern always fails 1971 matchStart = matchEnd = -1; 1972 ubrk_close(charBreakIterator); 1973 return FALSE; 1974 } 1975 1976 matchStart = matchEnd = -1; 1977 1978 for(int32_t i = 0; i < targetSize; i += 1) { 1979 if (targetOrders.matchesAt(i, patternOrders)) { 1980 int32_t start = targetOrders.getLowOffset(i); 1981 int32_t maxLimit = targetOrders.getLowOffset(i + patternSize); 1982 int32_t minLimit = targetOrders.getLowOffset(i + patternSize - 1); 1983 1984 // if the low and high offsets of the first CE in 1985 // the match are the same, it means that the match 1986 // starts in the middle of an expansion - all but 1987 // the first CE of the expansion will have the offset 1988 // of the following character. 1989 if (start == targetOrders.getHighOffset(i)) { 1990 continue; 1991 } 1992 1993 // Make sure match starts on a grapheme boundary 1994 if (! ubrk_isBoundary(charBreakIterator, start)) { 1995 continue; 1996 } 1997 1998 // If the low and high offsets of the CE after the match 1999 // are the same, it means that the match ends in the middle 2000 // of an expansion sequence. 2001 if (maxLimit == targetOrders.getHighOffset(i + patternSize) && 2002 targetOrders.getOrder(i + patternSize) != UCOL_NULLORDER) { 2003 continue; 2004 } 2005 2006 int32_t mend = maxLimit; 2007 2008 // Find the first grapheme break after the character index 2009 // of the last CE in the match. If it's after character index 2010 // that's after the last CE in the match, use that index 2011 // as the end of the match. 2012 if (minLimit < maxLimit) { 2013 int32_t nba = ubrk_following(charBreakIterator, minLimit); 2014 2015 if (nba >= targetOrders.getHighOffset(i + patternSize - 1)) { 2016 mend = nba; 2017 } 2018 } 2019 2020 if (mend > maxLimit) { 2021 continue; 2022 } 2023 2024 if (! ubrk_isBoundary(charBreakIterator, mend)) { 2025 continue; 2026 } 2027 2028 matchStart = start; 2029 matchEnd = mend; 2030 2031 ubrk_close(charBreakIterator); 2032 return TRUE; 2033 } 2034 } 2035 2036 ubrk_close(charBreakIterator); 2037 return FALSE; 2038 } 2039 2040 #if !UCONFIG_NO_REGULAR_EXPRESSIONS 2041 static int32_t getIntParam(UnicodeString name, UnicodeString ¶ms, int32_t defaultVal) { 2042 int32_t val = defaultVal; 2043 2044 name.append(" *= *(-?\\d+)"); 2045 2046 UErrorCode status = U_ZERO_ERROR; 2047 RegexMatcher m(name, params, 0, status); 2048 2049 if (m.find()) { 2050 // The param exists. Convert the string to an int. 2051 char valString[100]; 2052 int32_t paramLength = m.end(1, status) - m.start(1, status); 2053 2054 if (paramLength >= (int32_t)(sizeof(valString)-1)) { 2055 paramLength = (int32_t)(sizeof(valString)-2); 2056 } 2057 2058 params.extract(m.start(1, status), paramLength, valString, sizeof(valString)); 2059 val = strtol(valString, NULL, 10); 2060 2061 // Delete this parameter from the params string. 2062 m.reset(); 2063 params = m.replaceFirst("", status); 2064 } 2065 2066 //U_ASSERT(U_SUCCESS(status)); 2067 if (! U_SUCCESS(status)) { 2068 val = defaultVal; 2069 } 2070 2071 return val; 2072 } 2073 #endif 2074 2075 #if !UCONFIG_NO_COLLATION 2076 int32_t SSearchTest::monkeyTestCase(UCollator *coll, const UnicodeString &testCase, const UnicodeString &pattern, const UnicodeString &altPattern, 2077 const char *name, const char *strength, uint32_t seed) 2078 { 2079 UErrorCode status = U_ZERO_ERROR; 2080 int32_t actualStart = -1, actualEnd = -1; 2081 //int32_t expectedStart = prefix.length(), expectedEnd = prefix.length() + altPattern.length(); 2082 int32_t expectedStart = -1, expectedEnd = -1; 2083 int32_t notFoundCount = 0; 2084 LocalUStringSearchPointer uss(usearch_openFromCollator(pattern.getBuffer(), pattern.length(), 2085 testCase.getBuffer(), testCase.length(), 2086 coll, 2087 NULL, // the break iterator 2088 &status)); 2089 2090 // **** TODO: find *all* matches, not just first one **** 2091 simpleSearch(coll, testCase, 0, pattern, expectedStart, expectedEnd); 2092 2093 usearch_search(uss.getAlias(), 0, &actualStart, &actualEnd, &status); 2094 2095 if (expectedStart >= 0 && (actualStart != expectedStart || actualEnd != expectedEnd)) { 2096 errln("Search for <pattern> in <%s> failed: expected [%d, %d], got [%d, %d]\n" 2097 " strength=%s seed=%d", 2098 name, expectedStart, expectedEnd, actualStart, actualEnd, strength, seed); 2099 } 2100 2101 if (expectedStart == -1 && actualStart == -1) { 2102 notFoundCount += 1; 2103 } 2104 2105 // **** TODO: find *all* matches, not just first one **** 2106 simpleSearch(coll, testCase, 0, altPattern, expectedStart, expectedEnd); 2107 2108 usearch_setPattern(uss.getAlias(), altPattern.getBuffer(), altPattern.length(), &status); 2109 2110 usearch_search(uss.getAlias(), 0, &actualStart, &actualEnd, &status); 2111 2112 if (expectedStart >= 0 && (actualStart != expectedStart || actualEnd != expectedEnd)) { 2113 errln("Search for <alt_pattern> in <%s> failed: expected [%d, %d], got [%d, %d]\n" 2114 " strength=%s seed=%d", 2115 name, expectedStart, expectedEnd, actualStart, actualEnd, strength, seed); 2116 } 2117 2118 if (expectedStart == -1 && actualStart == -1) { 2119 notFoundCount += 1; 2120 } 2121 2122 return notFoundCount; 2123 } 2124 2125 static void hexForUnicodeString(const UnicodeString &ustr, char * cbuf, int32_t cbuflen) 2126 { 2127 int32_t ustri, ustrlen = ustr.length(); 2128 2129 for (ustri = 0; ustri < ustrlen; ++ustri) { 2130 if (cbuflen >= 9 /* format width for single code unit(5) + terminating ellipsis(3) + null(1) */) { 2131 int len = sprintf(cbuf, " %04X", ustr.charAt(ustri)); 2132 cbuflen -= len; 2133 cbuf += len; 2134 } else { 2135 if (cbuflen >= 4 /* terminating ellipsis(3) + null(1) */) { 2136 sprintf(cbuf, "..."); 2137 } else if (cbuflen >= 1) { 2138 cbuf = 0; 2139 } 2140 break; 2141 } 2142 } 2143 } 2144 2145 int32_t SSearchTest::bmMonkeyTestCase(UCollator *coll, const UnicodeString &testCase, const UnicodeString &pattern, const UnicodeString &altPattern, 2146 BoyerMooreSearch *bms, BoyerMooreSearch *abms, 2147 const char *name, const char *strength, uint32_t seed) 2148 { 2149 UErrorCode status = U_ZERO_ERROR; 2150 int32_t actualStart = -1, actualEnd = -1; 2151 //int32_t expectedStart = prefix.length(), expectedEnd = prefix.length() + altPattern.length(); 2152 int32_t expectedStart = -1, expectedEnd = -1; 2153 int32_t notFoundCount = 0; 2154 char hexbuf[128]; 2155 2156 // **** TODO: find *all* matches, not just first one **** 2157 simpleSearch(coll, testCase, 0, pattern, expectedStart, expectedEnd); 2158 2159 bms->setTargetString(&testCase, status); 2160 bms->search(0, actualStart, actualEnd); 2161 2162 if (expectedStart >= 0 && (actualStart != expectedStart || actualEnd != expectedEnd)) { 2163 hexForUnicodeString(pattern, hexbuf, sizeof(hexbuf)); 2164 errln("Boyer-Moore Search for <pattern> in <%s> failed: expected [%d, %d], got [%d, %d]\n" 2165 " strength=%s seed=%d <pattern>: %s", 2166 name, expectedStart, expectedEnd, actualStart, actualEnd, strength, seed, hexbuf); 2167 } 2168 2169 if (expectedStart == -1 && actualStart == -1) { 2170 notFoundCount += 1; 2171 } 2172 2173 // **** TODO: find *all* matches, not just first one **** 2174 simpleSearch(coll, testCase, 0, altPattern, expectedStart, expectedEnd); 2175 2176 abms->setTargetString(&testCase, status); 2177 abms->search(0, actualStart, actualEnd); 2178 2179 if (expectedStart >= 0 && (actualStart != expectedStart || actualEnd != expectedEnd)) { 2180 hexForUnicodeString(altPattern, hexbuf, sizeof(hexbuf)); 2181 errln("Boyer-Moore Search for <alt_pattern> in <%s> failed: expected [%d, %d], got [%d, %d]\n" 2182 " strength=%s seed=%d <pattern>: %s", 2183 name, expectedStart, expectedEnd, actualStart, actualEnd, strength, seed, hexbuf); 2184 } 2185 2186 if (expectedStart == -1 && actualStart == -1) { 2187 notFoundCount += 1; 2188 } 2189 2190 2191 return notFoundCount; 2192 } 2193 #endif 2194 2195 void SSearchTest::monkeyTest(char *params) 2196 { 2197 // ook! 2198 UErrorCode status = U_ZERO_ERROR; 2199 //UCollator *coll = ucol_open(NULL, &status); 2200 UCollator *coll = ucol_openFromShortString("S1", FALSE, NULL, &status); 2201 2202 if (U_FAILURE(status)) { 2203 errcheckln(status, "Failed to create collator in MonkeyTest! - %s", u_errorName(status)); 2204 return; 2205 } 2206 2207 CollData *monkeyData = CollData::open(coll, status); 2208 2209 USet *expansions = uset_openEmpty(); 2210 USet *contractions = uset_openEmpty(); 2211 2212 ucol_getContractionsAndExpansions(coll, contractions, expansions, FALSE, &status); 2213 2214 U_STRING_DECL(letter_pattern, "[[:letter:]-[:ideographic:]-[:hangul:]]", 39); 2215 U_STRING_INIT(letter_pattern, "[[:letter:]-[:ideographic:]-[:hangul:]]", 39); 2216 USet *letters = uset_openPattern(letter_pattern, 39, &status); 2217 SetMonkey letterMonkey(letters); 2218 StringSetMonkey contractionMonkey(contractions, coll, monkeyData); 2219 StringSetMonkey expansionMonkey(expansions, coll, monkeyData); 2220 UnicodeString testCase; 2221 UnicodeString alternate; 2222 UnicodeString pattern, altPattern; 2223 UnicodeString prefix, altPrefix; 2224 UnicodeString suffix, altSuffix; 2225 2226 Monkey *monkeys[] = { 2227 &letterMonkey, 2228 &contractionMonkey, 2229 &expansionMonkey, 2230 &contractionMonkey, 2231 &expansionMonkey, 2232 &contractionMonkey, 2233 &expansionMonkey, 2234 &contractionMonkey, 2235 &expansionMonkey}; 2236 int32_t monkeyCount = sizeof(monkeys) / sizeof(monkeys[0]); 2237 // int32_t nonMatchCount = 0; 2238 2239 UCollationStrength strengths[] = {UCOL_PRIMARY, UCOL_SECONDARY, UCOL_TERTIARY}; 2240 const char *strengthNames[] = {"primary", "secondary", "tertiary"}; 2241 int32_t strengthCount = sizeof(strengths) / sizeof(strengths[0]); 2242 int32_t loopCount = quick? 1000 : 10000; 2243 int32_t firstStrength = 0; 2244 int32_t lastStrength = strengthCount - 1; //*/ 0; 2245 2246 if (params != NULL) { 2247 #if !UCONFIG_NO_REGULAR_EXPRESSIONS 2248 UnicodeString p(params); 2249 2250 loopCount = getIntParam("loop", p, loopCount); 2251 m_seed = getIntParam("seed", p, m_seed); 2252 2253 RegexMatcher m(" *strength *= *(primary|secondary|tertiary) *", p, 0, status); 2254 if (m.find()) { 2255 UnicodeString breakType = m.group(1, status); 2256 2257 for (int32_t s = 0; s < strengthCount; s += 1) { 2258 if (breakType == strengthNames[s]) { 2259 firstStrength = lastStrength = s; 2260 break; 2261 } 2262 } 2263 2264 m.reset(); 2265 p = m.replaceFirst("", status); 2266 } 2267 2268 if (RegexMatcher("\\S", p, 0, status).find()) { 2269 // Each option is stripped out of the option string as it is processed. 2270 // All options have been checked. The option string should have been completely emptied.. 2271 char buf[100]; 2272 p.extract(buf, sizeof(buf), NULL, status); 2273 buf[sizeof(buf)-1] = 0; 2274 errln("Unrecognized or extra parameter: %s\n", buf); 2275 return; 2276 } 2277 #else 2278 infoln("SSearchTest built with UCONFIG_NO_REGULAR_EXPRESSIONS: ignoring parameters."); 2279 #endif 2280 } 2281 2282 for(int32_t s = firstStrength; s <= lastStrength; s += 1) { 2283 int32_t notFoundCount = 0; 2284 2285 logln("Setting strength to %s.", strengthNames[s]); 2286 ucol_setStrength(coll, strengths[s]); 2287 2288 // TODO: try alternate prefix and suffix too? 2289 // TODO: alterntaes are only equal at primary strength. Is this OK? 2290 for(int32_t t = 0; t < loopCount; t += 1) { 2291 uint32_t seed = m_seed; 2292 // int32_t nmc = 0; 2293 2294 generateTestCase(coll, monkeys, monkeyCount, pattern, altPattern); 2295 generateTestCase(coll, monkeys, monkeyCount, prefix, altPrefix); 2296 generateTestCase(coll, monkeys, monkeyCount, suffix, altSuffix); 2297 2298 // pattern 2299 notFoundCount += monkeyTestCase(coll, pattern, pattern, altPattern, "pattern", strengthNames[s], seed); 2300 2301 testCase.remove(); 2302 testCase.append(prefix); 2303 testCase.append(/*alt*/pattern); 2304 2305 // prefix + pattern 2306 notFoundCount += monkeyTestCase(coll, testCase, pattern, altPattern, "prefix + pattern", strengthNames[s], seed); 2307 2308 testCase.append(suffix); 2309 2310 // prefix + pattern + suffix 2311 notFoundCount += monkeyTestCase(coll, testCase, pattern, altPattern, "prefix + pattern + suffix", strengthNames[s], seed); 2312 2313 testCase.remove(); 2314 testCase.append(pattern); 2315 testCase.append(suffix); 2316 2317 // pattern + suffix 2318 notFoundCount += monkeyTestCase(coll, testCase, pattern, altPattern, "pattern + suffix", strengthNames[s], seed); 2319 } 2320 2321 logln("For strength %s the not found count is %d.", strengthNames[s], notFoundCount); 2322 } 2323 2324 uset_close(contractions); 2325 uset_close(expansions); 2326 uset_close(letters); 2327 2328 CollData::close(monkeyData); 2329 2330 ucol_close(coll); 2331 } 2332 2333 void SSearchTest::bmMonkeyTest(char *params) 2334 { 2335 static const UVersionInfo icu47 = { 4, 7, 0, 0 }; // for timebomb 2336 static const UChar skipChars[] = { 0x0E40, 0x0E41, 0x0E42, 0x0E43, 0x0E44, 0xAAB5, 0xAAB6, 0xAAB9, 0xAABB, 0xAABC, 0 }; // for timebomb 2337 // ook! 2338 UErrorCode status = U_ZERO_ERROR; 2339 UCollator *coll = ucol_openFromShortString("LEN_S1", FALSE, NULL, &status); 2340 2341 if (U_FAILURE(status)) { 2342 errcheckln(status, "Failed to create collator in MonkeyTest! - %s", u_errorName(status)); 2343 return; 2344 } 2345 2346 CollData *monkeyData = CollData::open(coll, status); 2347 2348 USet *expansions = uset_openEmpty(); 2349 USet *contractions = uset_openEmpty(); 2350 2351 ucol_getContractionsAndExpansions(coll, contractions, expansions, FALSE, &status); 2352 2353 U_STRING_DECL(letter_pattern, "[[:letter:]-[:ideographic:]-[:hangul:]]", 39); 2354 U_STRING_INIT(letter_pattern, "[[:letter:]-[:ideographic:]-[:hangul:]]", 39); 2355 USet *letters = uset_openPattern(letter_pattern, 39, &status); 2356 SetMonkey letterMonkey(letters); 2357 StringSetMonkey contractionMonkey(contractions, coll, monkeyData); 2358 StringSetMonkey expansionMonkey(expansions, coll, monkeyData); 2359 UnicodeString testCase; 2360 UnicodeString alternate; 2361 UnicodeString pattern, altPattern; 2362 UnicodeString prefix, altPrefix; 2363 UnicodeString suffix, altSuffix; 2364 2365 Monkey *monkeys[] = { 2366 &letterMonkey, 2367 &contractionMonkey, 2368 &expansionMonkey, 2369 &contractionMonkey, 2370 &expansionMonkey, 2371 &contractionMonkey, 2372 &expansionMonkey, 2373 &contractionMonkey, 2374 &expansionMonkey}; 2375 int32_t monkeyCount = sizeof(monkeys) / sizeof(monkeys[0]); 2376 // int32_t nonMatchCount = 0; 2377 2378 UCollationStrength strengths[] = {UCOL_PRIMARY, UCOL_SECONDARY, UCOL_TERTIARY}; 2379 const char *strengthNames[] = {"primary", "secondary", "tertiary"}; 2380 int32_t strengthCount = sizeof(strengths) / sizeof(strengths[0]); 2381 int32_t loopCount = quick? 1000 : 10000; 2382 int32_t firstStrength = 0; 2383 int32_t lastStrength = strengthCount - 1; //*/ 0; 2384 2385 if (params != NULL) { 2386 #if !UCONFIG_NO_REGULAR_EXPRESSIONS 2387 UnicodeString p(params); 2388 2389 loopCount = getIntParam("loop", p, loopCount); 2390 m_seed = getIntParam("seed", p, m_seed); 2391 2392 RegexMatcher m(" *strength *= *(primary|secondary|tertiary) *", p, 0, status); 2393 if (m.find()) { 2394 UnicodeString breakType = m.group(1, status); 2395 2396 for (int32_t s = 0; s < strengthCount; s += 1) { 2397 if (breakType == strengthNames[s]) { 2398 firstStrength = lastStrength = s; 2399 break; 2400 } 2401 } 2402 2403 m.reset(); 2404 p = m.replaceFirst("", status); 2405 } 2406 2407 if (RegexMatcher("\\S", p, 0, status).find()) { 2408 // Each option is stripped out of the option string as it is processed. 2409 // All options have been checked. The option string should have been completely emptied.. 2410 char buf[100]; 2411 p.extract(buf, sizeof(buf), NULL, status); 2412 buf[sizeof(buf)-1] = 0; 2413 errln("Unrecognized or extra parameter: %s\n", buf); 2414 return; 2415 } 2416 #else 2417 infoln("SSearchTest built with UCONFIG_NO_REGULAR_EXPRESSIONS: ignoring parameters."); 2418 #endif 2419 } 2420 2421 for(int32_t s = firstStrength; s <= lastStrength; s += 1) { 2422 int32_t notFoundCount = 0; 2423 2424 logln("Setting strength to %s.", strengthNames[s]); 2425 ucol_setStrength(coll, strengths[s]); 2426 2427 CollData *data = CollData::open(coll, status); 2428 2429 UnicodeString skipString(skipChars); // for timebomb 2430 UnicodeSet* skipSet = UnicodeSet::createFromAll(skipString); // for timebomb 2431 // TODO: try alternate prefix and suffix too? 2432 // TODO: alterntaes are only equal at primary strength. Is this OK? 2433 for(int32_t t = 0; t < loopCount; t += 1) { 2434 uint32_t seed = m_seed; 2435 // int32_t nmc = 0; 2436 2437 generateTestCase(coll, monkeys, monkeyCount, pattern, altPattern); 2438 generateTestCase(coll, monkeys, monkeyCount, prefix, altPrefix); 2439 generateTestCase(coll, monkeys, monkeyCount, suffix, altSuffix); 2440 2441 if (!isICUVersionAtLeast(icu47) && skipSet->containsSome(pattern)) { 2442 continue; // timebomb until ticket #8080 is resolved 2443 } 2444 2445 BoyerMooreSearch pat(data, pattern, NULL, status); 2446 BoyerMooreSearch alt(data, altPattern, NULL, status); 2447 2448 // **** need a better way to deal with this **** 2449 #if 0 2450 if (pat.empty() || 2451 alt.empty()) { 2452 continue; 2453 } 2454 #endif 2455 2456 // pattern 2457 notFoundCount += bmMonkeyTestCase(coll, pattern, pattern, altPattern, &pat, &alt, "pattern", strengthNames[s], seed); 2458 2459 testCase.remove(); 2460 testCase.append(prefix); 2461 testCase.append(/*alt*/pattern); 2462 2463 // prefix + pattern 2464 notFoundCount += bmMonkeyTestCase(coll, testCase, pattern, altPattern, &pat, &alt, "prefix + pattern", strengthNames[s], seed); 2465 2466 testCase.append(suffix); 2467 2468 // prefix + pattern + suffix 2469 notFoundCount += bmMonkeyTestCase(coll, testCase, pattern, altPattern, &pat, &alt, "prefix + pattern + suffix", strengthNames[s], seed); 2470 2471 testCase.remove(); 2472 testCase.append(pattern); 2473 testCase.append(suffix); 2474 2475 // pattern + suffix 2476 notFoundCount += bmMonkeyTestCase(coll, testCase, pattern, altPattern, &pat, &alt, "pattern + suffix", strengthNames[s], seed); 2477 } 2478 delete skipSet; // for timebomb 2479 2480 CollData::close(data); 2481 2482 logln("For strength %s the not found count is %d.", strengthNames[s], notFoundCount); 2483 } 2484 2485 uset_close(contractions); 2486 uset_close(expansions); 2487 uset_close(letters); 2488 2489 CollData::close(monkeyData); 2490 2491 ucol_close(coll); 2492 } 2493 2494 void SSearchTest::stringListTest(){ 2495 UErrorCode status = U_ZERO_ERROR; 2496 StringList *sl = new StringList(status); 2497 if(U_FAILURE(status)){ 2498 errln("ERROR: stringListTest: Could not start StringList"); 2499 } 2500 2501 const UChar chars[] = { 2502 0x0000 2503 }; 2504 sl->add(chars, (int32_t) 0, status); 2505 if(U_FAILURE(status)){ 2506 errln("ERROR: stringListTest: StringList::add"); 2507 } 2508 2509 if(sl->getDynamicClassID() != StringList::getStaticClassID()){ 2510 errln("ERROR: stringListTest: getDynamicClassID and getStaticClassID does not match"); 2511 } 2512 delete sl; 2513 } 2514 2515 #endif 2516 2517 #endif 2518
