Home | History | Annotate | Download | only in intltest 
      1 /********************************************************************
      2  * COPYRIGHT:
      3  * Copyright (c) 1997-2014, International Business Machines Corporation and
      4  * others. All Rights Reserved.
      5  ********************************************************************/
      6 
      7 /**
      8  * IntlTestCollator is the medium level test class for everything in the directory "collate".
      9  */
     10 
     11 /***********************************************************************
     12 * Modification history
     13 * Date        Name        Description
     14 * 02/14/2001  synwee      Compare with cintltst and commented away tests
     15 *                         that are not run.
     16 ***********************************************************************/
     17 
     18 #include "unicode/utypes.h"
     19 
     20 #if !UCONFIG_NO_COLLATION
     21 
     22 #include "unicode/localpointer.h"
     23 #include "unicode/sortkey.h"
     24 #include "unicode/uchar.h"
     25 #include "unicode/ustring.h"
     26 
     27 #include "encoll.h"
     28 #include "frcoll.h"
     29 #include "decoll.h"
     30 #include "escoll.h"
     31 #include "ficoll.h"
     32 #include "jacoll.h"
     33 #include "trcoll.h"
     34 #include "allcoll.h"
     35 #include "g7coll.h"
     36 #include "mnkytst.h"
     37 #include "apicoll.h"
     38 #include "regcoll.h"
     39 #include "currcoll.h"
     40 #include "itercoll.h"
     41 #include "tstnorm.h"
     42 #include "normconf.h"
     43 #include "thcoll.h"
     44 #include "srchtest.h"
     45 #include "ssearch.h"
     46 #include "lcukocol.h"
     47 #include "ucaconf.h"
     48 #include "svccoll.h"
     49 #include "cmemory.h"
     50 #include "alphaindextst.h"
     51 
     52 // Set to 1 to test offsets in backAndForth()
     53 #define TEST_OFFSETS 0
     54 
     55 extern IntlTest *createCollationTest();
     56 
     57 void IntlTestCollator::runIndexedTest( int32_t index, UBool exec, const char* &name, char* par )
     58 {
     59     if (exec) {
     60         logln("TestSuite Collator: ");
     61     }
     62 
     63     TESTCASE_AUTO_BEGIN;
     64     TESTCASE_AUTO_CLASS(CollationEnglishTest);
     65     TESTCASE_AUTO_CLASS(CollationFrenchTest);
     66     TESTCASE_AUTO_CLASS(CollationGermanTest);
     67     TESTCASE_AUTO_CLASS(CollationSpanishTest);
     68     TESTCASE_AUTO_CLASS(CollationKanaTest);
     69     TESTCASE_AUTO_CLASS(CollationTurkishTest);
     70     TESTCASE_AUTO_CLASS(CollationDummyTest);
     71     TESTCASE_AUTO_CLASS(G7CollationTest);
     72     TESTCASE_AUTO_CLASS(CollationMonkeyTest);
     73     TESTCASE_AUTO_CLASS(CollationAPITest);
     74     TESTCASE_AUTO_CLASS(CollationRegressionTest);
     75     TESTCASE_AUTO_CLASS(CollationCurrencyTest);
     76     TESTCASE_AUTO_CLASS(CollationIteratorTest);
     77     TESTCASE_AUTO_CLASS(CollationThaiTest);
     78     TESTCASE_AUTO_CLASS(LotusCollationKoreanTest);
     79     TESTCASE_AUTO_CLASS(StringSearchTest);
     80     TESTCASE_AUTO_CLASS(UCAConformanceTest);
     81     TESTCASE_AUTO_CLASS(CollationServiceTest);
     82     TESTCASE_AUTO_CLASS(CollationFinnishTest); // removed by weiv - we have changed Finnish collation
     83     TESTCASE_AUTO_CLASS(SSearchTest);
     84 #if !UCONFIG_NO_NORMALIZATION
     85     TESTCASE_AUTO_CLASS(AlphabeticIndexTest);
     86 #endif
     87     TESTCASE_AUTO_CREATE_CLASS(CollationTest);
     88     TESTCASE_AUTO_END;
     89 }
     90 
     91 UCollationResult
     92 IntlTestCollator::compareUsingPartials(UCollator *coll, const UChar source[], int32_t sLen, const UChar target[], int32_t tLen, int32_t pieceSize, UErrorCode &status) {
     93   int32_t partialSKResult = 0;
     94   uint8_t sBuf[512], tBuf[512];
     95   UCharIterator sIter, tIter;
     96   uint32_t sState[2], tState[2];
     97   int32_t sSize = pieceSize, tSize = pieceSize;
     98   int32_t i = 0;
     99   status = U_ZERO_ERROR;
    100   sState[0] = 0; sState[1] = 0;
    101   tState[0] = 0; tState[1] = 0;
    102   while(sSize == pieceSize && tSize == pieceSize && partialSKResult == 0) {
    103     uiter_setString(&sIter, source, sLen);
    104     uiter_setString(&tIter, target, tLen);
    105     sSize = ucol_nextSortKeyPart(coll, &sIter, sState, sBuf, pieceSize, &status);
    106     tSize = ucol_nextSortKeyPart(coll, &tIter, tState, tBuf, pieceSize, &status);
    107 
    108     if(sState[0] != 0 || tState[0] != 0) {
    109       log("State != 0 : %08X %08X\n", sState[0], tState[0]);
    110     }
    111     log("%i ", i++);
    112 
    113     partialSKResult = memcmp(sBuf, tBuf, pieceSize);
    114   }
    115 
    116   if(partialSKResult < 0) {
    117       return UCOL_LESS;
    118   } else if(partialSKResult > 0) {
    119     return UCOL_GREATER;
    120   } else {
    121     return UCOL_EQUAL;
    122   }
    123 }
    124 
    125 void
    126 IntlTestCollator::doTestVariant(Collator* col, const UnicodeString &source, const UnicodeString &target, Collator::EComparisonResult result)
    127 {
    128   UErrorCode status = U_ZERO_ERROR;
    129 
    130   UCollator *myCollation = col->toUCollator();
    131 
    132   Collator::EComparisonResult compareResult = col->compare(source, target);
    133 
    134   CollationKey srckey, tgtkey;
    135   col->getCollationKey(source, srckey, status);
    136   col->getCollationKey(target, tgtkey, status);
    137   if (U_FAILURE(status)){
    138     errln("Creation of collation keys failed\n");
    139   }
    140   Collator::EComparisonResult keyResult = srckey.compareTo(tgtkey);
    141 
    142   reportCResult(source, target, srckey, tgtkey, compareResult, keyResult, result, result);
    143 
    144     UColAttributeValue norm = ucol_getAttribute(myCollation, UCOL_NORMALIZATION_MODE, &status);
    145 
    146     int32_t sLen = source.length(), tLen = target.length();
    147     const UChar* src = source.getBuffer();
    148     const UChar* trg = target.getBuffer();
    149     UCollationResult compareResultIter = (UCollationResult)result;
    150 
    151     {
    152       UCharIterator sIter, tIter;
    153       uiter_setString(&sIter, src, sLen);
    154       uiter_setString(&tIter, trg, tLen);
    155       compareResultIter = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
    156       if(compareResultIter != (UCollationResult)result) {
    157         errln("Different result for iterative comparison "+source+" "+target);
    158       }
    159     }
    160     /* convert the strings to UTF-8 and do try comparing with char iterator */
    161     if(!quick) { /*!QUICK*/
    162       char utf8Source[256], utf8Target[256];
    163       int32_t utf8SourceLen = 0, utf8TargetLen = 0;
    164       u_strToUTF8(utf8Source, 256, &utf8SourceLen, src, sLen, &status);
    165       if(U_FAILURE(status)) { /* probably buffer is not big enough */
    166         log("Src UTF-8 buffer too small! Will not compare!\n");
    167       } else {
    168         u_strToUTF8(utf8Target, 256, &utf8TargetLen, trg, tLen, &status);
    169         if(U_SUCCESS(status)) { /* probably buffer is not big enough */
    170           UCollationResult compareResultUTF8 = (UCollationResult)result, compareResultUTF8Norm = (UCollationResult)result;
    171           UCharIterator sIter, tIter;
    172           /*log_verbose("Strings converted to UTF-8:%s, %s\n", aescstrdup(source,-1), aescstrdup(target,-1));*/
    173           uiter_setUTF8(&sIter, utf8Source, utf8SourceLen);
    174           uiter_setUTF8(&tIter, utf8Target, utf8TargetLen);
    175        /*uiter_setString(&sIter, source, sLen);
    176       uiter_setString(&tIter, target, tLen);*/
    177           compareResultUTF8 = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
    178           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);
    179           sIter.move(&sIter, 0, UITER_START);
    180           tIter.move(&tIter, 0, UITER_START);
    181           compareResultUTF8Norm = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
    182           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, norm, &status);
    183           if(compareResultUTF8 != compareResultIter) {
    184             errln("different results in iterative comparison for UTF-16 and UTF-8 encoded strings. "+source+", "+target);
    185           }
    186           if(compareResultUTF8 != compareResultUTF8Norm) {
    187             errln("different results in iterative when normalization is turned on with UTF-8 strings. "+source+", "+target);
    188           }
    189         } else {
    190           log("Target UTF-8 buffer too small! Did not compare!\n");
    191         }
    192         if(U_FAILURE(status)) {
    193           log("UTF-8 strcoll failed! Ignoring result\n");
    194         }
    195       }
    196     }
    197 
    198     /* testing the partial sortkeys */
    199     { /*!QUICK*/
    200       int32_t partialSizes[] = { 3, 1, 2, 4, 8, 20, 80 }; /* just size 3 in the quick mode */
    201       int32_t partialSizesSize = 1;
    202       if(!quick) {
    203         partialSizesSize = 7;
    204       }
    205       int32_t i = 0;
    206       log("partial sortkey test piecesize=");
    207       for(i = 0; i < partialSizesSize; i++) {
    208         UCollationResult partialSKResult = (UCollationResult)result, partialNormalizedSKResult = (UCollationResult)result;
    209         log("%i ", partialSizes[i]);
    210 
    211         partialSKResult = compareUsingPartials(myCollation, src, sLen, trg, tLen, partialSizes[i], status);
    212         if(partialSKResult != (UCollationResult)result) {
    213           errln("Partial sortkey comparison returned wrong result: "+source+", "+target+" (size "+partialSizes[i]+")");
    214         }
    215 
    216         if(norm != UCOL_ON && !quick) {
    217           log("N ");
    218           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, UCOL_ON, &status);
    219           partialNormalizedSKResult = compareUsingPartials(myCollation, src, sLen, trg, tLen, partialSizes[i], status);
    220           ucol_setAttribute(myCollation, UCOL_NORMALIZATION_MODE, norm, &status);
    221           if(partialSKResult != partialNormalizedSKResult) {
    222             errln("Partial sortkey comparison gets different result when normalization is on: "+source+", "+target+" (size "+partialSizes[i]+")");
    223           }
    224         }
    225       }
    226       log("\n");
    227     }
    228 /*
    229   if (compareResult != result) {
    230     errln("String comparison failed in variant test\n");
    231   }
    232   if (keyResult != result) {
    233     errln("Collation key comparison failed in variant test\n");
    234   }
    235 */
    236 }
    237 
    238 void
    239 IntlTestCollator::doTest(Collator* col, const UChar *source, const UChar *target, Collator::EComparisonResult result) {
    240   doTest(col, UnicodeString(source), UnicodeString(target), result);
    241 }
    242 
    243 void
    244 IntlTestCollator::doTest(Collator* col, const UnicodeString &source, const UnicodeString &target, Collator::EComparisonResult result)
    245 {
    246   if(col) {
    247     doTestVariant(col, source, target, result);
    248     if(result == Collator::LESS) {
    249       doTestVariant(col, target, source, Collator::GREATER);
    250     } else if (result == Collator::GREATER) {
    251       doTestVariant(col, target, source, Collator::LESS);
    252     }
    253 
    254     UErrorCode status = U_ZERO_ERROR;
    255     LocalPointer<CollationElementIterator> c(((RuleBasedCollator *)col)->createCollationElementIterator(source));
    256     logln("Testing iterating source: "+source);
    257     backAndForth(*c);
    258     c->setText(target, status);
    259     logln("Testing iterating target: "+target);
    260     backAndForth(*c);
    261   }
    262 }
    263 
    264 
    265 // used for collation result reporting, defined here for convenience
    266 // (maybe moved later)
    267 void
    268 IntlTestCollator::reportCResult( const UnicodeString &source, const UnicodeString &target,
    269              CollationKey &sourceKey, CollationKey &targetKey,
    270              Collator::EComparisonResult compareResult,
    271              Collator::EComparisonResult keyResult,
    272                                 Collator::EComparisonResult incResult,
    273                          Collator::EComparisonResult expectedResult )
    274 {
    275     if (expectedResult < -1 || expectedResult > 1)
    276     {
    277         errln("***** invalid call to reportCResult ****");
    278         return;
    279     }
    280 
    281     UBool ok1 = (compareResult == expectedResult);
    282     UBool ok2 = (keyResult == expectedResult);
    283     UBool ok3 = (incResult == expectedResult);
    284 
    285 
    286     if (ok1 && ok2 && ok3 && !verbose) {
    287         // Keep non-verbose, passing tests fast
    288         return;
    289     } else {
    290         UnicodeString msg1(ok1 ? "Ok: compare(" : "FAIL: compare(");
    291         UnicodeString msg2(", "), msg3(") returned "), msg4("; expected ");
    292         UnicodeString prettySource, prettyTarget, sExpect, sResult;
    293 
    294         IntlTest::prettify(source, prettySource);
    295         IntlTest::prettify(target, prettyTarget);
    296         appendCompareResult(compareResult, sResult);
    297         appendCompareResult(expectedResult, sExpect);
    298 
    299         if (ok1) {
    300             logln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult);
    301         } else {
    302             errln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult + msg4 + sExpect);
    303         }
    304 
    305         msg1 = UnicodeString(ok2 ? "Ok: key(" : "FAIL: key(");
    306         msg2 = ").compareTo(key(";
    307         msg3 = ")) returned ";
    308 
    309         appendCompareResult(keyResult, sResult);
    310 
    311         if (ok2) {
    312             logln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult);
    313         } else {
    314             errln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult + msg4 + sExpect);
    315 
    316             msg1 = "  ";
    317             msg2 = " vs. ";
    318 
    319             prettify(sourceKey, prettySource);
    320             prettify(targetKey, prettyTarget);
    321 
    322             errln(msg1 + prettySource + msg2 + prettyTarget);
    323         }
    324         msg1 = UnicodeString (ok3 ? "Ok: incCompare(" : "FAIL: incCompare(");
    325         msg2 = ", ";
    326         msg3 = ") returned ";
    327 
    328         appendCompareResult(incResult, sResult);
    329 
    330         if (ok3) {
    331             logln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult);
    332         } else {
    333             errln(msg1 + prettySource + msg2 + prettyTarget + msg3 + sResult + msg4 + sExpect);
    334         }
    335     }
    336 }
    337 
    338 UnicodeString&
    339 IntlTestCollator::appendCompareResult(Collator::EComparisonResult result,
    340                   UnicodeString& target)
    341 {
    342     if (result == Collator::LESS)
    343     {
    344         target += "LESS";
    345     }
    346     else if (result == Collator::EQUAL)
    347     {
    348         target += "EQUAL";
    349     }
    350     else if (result == Collator::GREATER)
    351     {
    352         target += "GREATER";
    353     }
    354     else
    355     {
    356         UnicodeString huh = "?";
    357 
    358         target += (huh + (int32_t)result);
    359     }
    360 
    361     return target;
    362 }
    363 
    364 // Produce a printable representation of a CollationKey
    365 UnicodeString &IntlTestCollator::prettify(const CollationKey &source, UnicodeString &target)
    366 {
    367     int32_t i, byteCount;
    368     const uint8_t *bytes = source.getByteArray(byteCount);
    369 
    370     target.remove();
    371     target += "[";
    372 
    373     for (i = 0; i < byteCount; i += 1)
    374     {
    375         if (i != 0) {
    376             target += " ";
    377         }
    378         appendHex(bytes[i], 2, target);
    379     }
    380 
    381     target += "]";
    382 
    383     return target;
    384 }
    385 
    386 void IntlTestCollator::backAndForth(CollationElementIterator &iter)
    387 {
    388     // Run through the iterator forwards and stick it into an array
    389     int32_t orderLength = 0;
    390     LocalArray<Order> orders(getOrders(iter, orderLength));
    391     UErrorCode status = U_ZERO_ERROR;
    392 
    393     // Now go through it backwards and make sure we get the same values
    394     int32_t index = orderLength;
    395     int32_t o;
    396 
    397     // reset the iterator
    398     iter.reset();
    399 
    400     while ((o = iter.previous(status)) != CollationElementIterator::NULLORDER)
    401     {
    402         /*int32_t offset = */iter.getOffset();
    403 
    404         if (index == 0) {
    405           if(o == 0) {
    406             continue;
    407           } else { // this is an error, orders exhausted but there are non-ignorable CEs from
    408             // going backwards
    409             errln("Backward iteration returned a non ignorable after orders are exhausted");
    410             break;
    411           }
    412         }
    413 
    414         index -= 1;
    415         if (o != orders[index].order) {
    416             if (o == 0)
    417                 index += 1;
    418             else {
    419                 while (index > 0 && orders[--index].order == 0) {
    420                   // nothing...
    421                 }
    422 
    423                 if (o != orders[index].order) {
    424                     errln("Mismatched order at index %d: 0x%0:8X vs. 0x%0:8X", index,
    425                     orders[index].order, o);
    426                 //break;
    427                   return;
    428                 }
    429             }
    430         }
    431 
    432 #if TEST_OFFSETS
    433         if (offset != orders[index].offset) {
    434           errln("Mismatched offset at index %d: %d vs. %d", index,
    435             orders[index].offset, offset);
    436        //break;
    437          return;
    438         }
    439 #endif
    440 
    441     }
    442 
    443     while (index != 0 && orders[index - 1].order == 0)
    444     {
    445       index --;
    446     }
    447 
    448     if (index != 0)
    449     {
    450         UnicodeString msg("Didn't get back to beginning - index is ");
    451         errln(msg + index);
    452 
    453         iter.reset();
    454         err("next: ");
    455         while ((o = iter.next(status)) != CollationElementIterator::NULLORDER)
    456         {
    457             UnicodeString hexString("0x");
    458 
    459             appendHex(o, 8, hexString);
    460             hexString += " ";
    461             err(hexString);
    462         }
    463         errln("");
    464 
    465         err("prev: ");
    466         while ((o = iter.previous(status)) != CollationElementIterator::NULLORDER)
    467         {
    468             UnicodeString hexString("0x");
    469 
    470             appendHex(o, 8, hexString);
    471             hexString += " ";
    472              err(hexString);
    473         }
    474         errln("");
    475     }
    476 }
    477 
    478 
    479 /**
    480  * Return an integer array containing all of the collation orders
    481  * returned by calls to next on the specified iterator
    482  */
    483 IntlTestCollator::Order *IntlTestCollator::getOrders(CollationElementIterator &iter, int32_t &orderLength)
    484 {
    485     int32_t maxSize = 100;
    486     int32_t size = 0;
    487     LocalArray<Order> orders(new Order[maxSize]);
    488     UErrorCode status = U_ZERO_ERROR;
    489     int32_t offset = iter.getOffset();
    490 
    491     int32_t order;
    492     while ((order = iter.next(status)) != CollationElementIterator::NULLORDER)
    493     {
    494         if (size == maxSize)
    495         {
    496             maxSize *= 2;
    497             Order *temp = new Order[maxSize];
    498 
    499             uprv_memcpy(temp, orders.getAlias(), size * sizeof(Order));
    500             orders.adoptInstead(temp);
    501         }
    502 
    503         orders[size].order  = order;
    504         orders[size].offset = offset;
    505 
    506         offset = iter.getOffset();
    507         size += 1;
    508     }
    509     if (U_FAILURE(status)) {
    510         errln("CollationElementIterator.next() failed - %s",
    511               u_errorName(status));
    512     }
    513 
    514     if (maxSize > size)
    515     {
    516         Order *temp = new Order[size];
    517 
    518         uprv_memcpy(temp, orders.getAlias(), size * sizeof(Order));
    519         orders.adoptInstead(temp);
    520     }
    521 
    522     orderLength = size;
    523     return orders.orphan();
    524 }
    525 
    526 #endif /* #if !UCONFIG_NO_COLLATION */
    527