Home | History | Annotate | Download | only in intltest 
      1 /********************************************************************
      2  * COPYRIGHT:
      3  * Copyright (c) 1997-2010, International Business Machines Corporation and
      4  * others. All Rights Reserved.
      5  ********************************************************************/
      6 
      7 #include "unicode/ustring.h"
      8 #include "unicode/uchar.h"
      9 #include "unicode/uniset.h"
     10 #include "unicode/putil.h"
     11 #include "cstring.h"
     12 #include "hash.h"
     13 #include "uparse.h"
     14 #include "ucdtest.h"
     15 
     16 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
     17 
     18 static const char *ignorePropNames[]={
     19     "FC_NFKC",
     20     "NFD_QC",
     21     "NFC_QC",
     22     "NFKD_QC",
     23     "NFKC_QC",
     24     "Expands_On_NFD",
     25     "Expands_On_NFC",
     26     "Expands_On_NFKD",
     27     "Expands_On_NFKC",
     28     "NFKC_CF"
     29 };
     30 
     31 UnicodeTest::UnicodeTest()
     32 {
     33     UErrorCode errorCode=U_ZERO_ERROR;
     34     unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
     35     if(U_FAILURE(errorCode)) {
     36         delete unknownPropertyNames;
     37         unknownPropertyNames=NULL;
     38     }
     39     // Ignore some property names altogether.
     40     for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
     41         unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
     42     }
     43 }
     44 
     45 UnicodeTest::~UnicodeTest()
     46 {
     47     delete unknownPropertyNames;
     48 }
     49 
     50 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
     51 {
     52     if (exec) logln("TestSuite UnicodeTest: ");
     53     switch (index) {
     54         case 0: name = "TestAdditionalProperties"; if(exec) TestAdditionalProperties(); break;
     55         case 1: name = "TestBinaryValues"; if(exec) TestBinaryValues(); break;
     56         default: name = ""; break; //needed to end loop
     57     }
     58 }
     59 
     60 //====================================================
     61 // private data used by the tests
     62 //====================================================
     63 
     64 // test DerivedCoreProperties.txt -------------------------------------------
     65 
     66 // copied from genprops.c
     67 static int32_t
     68 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
     69     const char *t, *z;
     70     int32_t i, j;
     71 
     72     s=u_skipWhitespace(s);
     73     for(i=0; i<countTokens; ++i) {
     74         t=tokens[i];
     75         if(t!=NULL) {
     76             for(j=0;; ++j) {
     77                 if(t[j]!=0) {
     78                     if(s[j]!=t[j]) {
     79                         break;
     80                     }
     81                 } else {
     82                     z=u_skipWhitespace(s+j);
     83                     if(*z==';' || *z==0) {
     84                         return i;
     85                     } else {
     86                         break;
     87                     }
     88                 }
     89             }
     90         }
     91     }
     92     return -1;
     93 }
     94 
     95 static const char *const
     96 derivedPropsNames[]={
     97     "Math",
     98     "Alphabetic",
     99     "Lowercase",
    100     "Uppercase",
    101     "ID_Start",
    102     "ID_Continue",
    103     "XID_Start",
    104     "XID_Continue",
    105     "Default_Ignorable_Code_Point",
    106     "Full_Composition_Exclusion",
    107     "Grapheme_Extend",
    108     "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
    109     "Grapheme_Base",
    110     "Cased",
    111     "Case_Ignorable",
    112     "Changes_When_Lowercased",
    113     "Changes_When_Uppercased",
    114     "Changes_When_Titlecased",
    115     "Changes_When_Casefolded",
    116     "Changes_When_Casemapped",
    117     "Changes_When_NFKC_Casefolded"
    118 };
    119 
    120 static const UProperty
    121 derivedPropsIndex[]={
    122     UCHAR_MATH,
    123     UCHAR_ALPHABETIC,
    124     UCHAR_LOWERCASE,
    125     UCHAR_UPPERCASE,
    126     UCHAR_ID_START,
    127     UCHAR_ID_CONTINUE,
    128     UCHAR_XID_START,
    129     UCHAR_XID_CONTINUE,
    130     UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
    131     UCHAR_FULL_COMPOSITION_EXCLUSION,
    132     UCHAR_GRAPHEME_EXTEND,
    133     UCHAR_GRAPHEME_LINK,
    134     UCHAR_GRAPHEME_BASE,
    135     UCHAR_CASED,
    136     UCHAR_CASE_IGNORABLE,
    137     UCHAR_CHANGES_WHEN_LOWERCASED,
    138     UCHAR_CHANGES_WHEN_UPPERCASED,
    139     UCHAR_CHANGES_WHEN_TITLECASED,
    140     UCHAR_CHANGES_WHEN_CASEFOLDED,
    141     UCHAR_CHANGES_WHEN_CASEMAPPED,
    142     UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
    143 };
    144 
    145 static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
    146 
    147 enum { MAX_ERRORS=50 };
    148 
    149 U_CFUNC void U_CALLCONV
    150 derivedPropsLineFn(void *context,
    151                    char *fields[][2], int32_t /* fieldCount */,
    152                    UErrorCode *pErrorCode)
    153 {
    154     UnicodeTest *me=(UnicodeTest *)context;
    155     uint32_t start, end;
    156     int32_t i;
    157 
    158     u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
    159     if(U_FAILURE(*pErrorCode)) {
    160         me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
    161         return;
    162     }
    163 
    164     /* parse derived binary property name, ignore unknown names */
    165     i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
    166     if(i<0) {
    167         UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
    168         propName.trim();
    169         if(me->unknownPropertyNames->find(propName)==NULL) {
    170             UErrorCode errorCode=U_ZERO_ERROR;
    171             me->unknownPropertyNames->puti(propName, 1, errorCode);
    172             me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
    173         }
    174         return;
    175     }
    176 
    177     me->derivedProps[i].add(start, end);
    178 }
    179 
    180 void UnicodeTest::TestAdditionalProperties() {
    181 #if !UCONFIG_NO_NORMALIZATION
    182     // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
    183     if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
    184         errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
    185               LENGTHOF(derivedPropsNames));
    186         return;
    187     }
    188     if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
    189         errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
    190         return;
    191     }
    192 
    193     char newPath[256];
    194     char backupPath[256];
    195     char *fields[2][2];
    196     UErrorCode errorCode=U_ZERO_ERROR;
    197 
    198     /* Look inside ICU_DATA first */
    199     strcpy(newPath, pathToDataDirectory());
    200     strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
    201 
    202     // As a fallback, try to guess where the source data was located
    203     // at the time ICU was built, and look there.
    204 #   ifdef U_TOPSRCDIR
    205         strcpy(backupPath, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
    206 #   else
    207         strcpy(backupPath, loadTestData(errorCode));
    208         strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
    209 #   endif
    210     strcat(backupPath, U_FILE_SEP_STRING);
    211     strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
    212 
    213     char *path=newPath;
    214     u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
    215 
    216     if(errorCode==U_FILE_ACCESS_ERROR) {
    217         errorCode=U_ZERO_ERROR;
    218         path=backupPath;
    219         u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
    220     }
    221     if(U_FAILURE(errorCode)) {
    222         errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
    223         return;
    224     }
    225     char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
    226     strcpy(basename, "DerivedNormalizationProps.txt");
    227     u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
    228     if(U_FAILURE(errorCode)) {
    229         errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
    230         return;
    231     }
    232 
    233     // now we have all derived core properties in the UnicodeSets
    234     // run them all through the API
    235     int32_t rangeCount, range;
    236     uint32_t i;
    237     UChar32 start, end;
    238 
    239     // test all TRUE properties
    240     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
    241         rangeCount=derivedProps[i].getRangeCount();
    242         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
    243             start=derivedProps[i].getRangeStart(range);
    244             end=derivedProps[i].getRangeEnd(range);
    245             for(; start<=end; ++start) {
    246                 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
    247                     dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
    248                     if(++numErrors[i]>=MAX_ERRORS) {
    249                       dataerrln("Too many errors, moving to the next test");
    250                       break;
    251                     }
    252                 }
    253             }
    254         }
    255     }
    256 
    257     // invert all properties
    258     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
    259         derivedProps[i].complement();
    260     }
    261 
    262     // test all FALSE properties
    263     for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
    264         rangeCount=derivedProps[i].getRangeCount();
    265         for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
    266             start=derivedProps[i].getRangeStart(range);
    267             end=derivedProps[i].getRangeEnd(range);
    268             for(; start<=end; ++start) {
    269                 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
    270                     errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
    271                     if(++numErrors[i]>=MAX_ERRORS) {
    272                       errln("Too many errors, moving to the next test");
    273                       break;
    274                     }
    275                 }
    276             }
    277         }
    278     }
    279 #endif /* !UCONFIG_NO_NORMALIZATION */
    280 }
    281 
    282 void UnicodeTest::TestBinaryValues() {
    283     /*
    284      * Unicode 5.1 explicitly defines binary property value aliases.
    285      * Verify that they are all recognized.
    286      */
    287     UErrorCode errorCode=U_ZERO_ERROR;
    288     UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
    289     if(U_FAILURE(errorCode)) {
    290         dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
    291         return;
    292     }
    293 
    294     static const char *const falseValues[]={ "N", "No", "F", "False" };
    295     static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
    296     int32_t i;
    297     for(i=0; i<LENGTHOF(falseValues); ++i) {
    298         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
    299         pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
    300         errorCode=U_ZERO_ERROR;
    301         UnicodeSet set(pattern, errorCode);
    302         if(U_FAILURE(errorCode)) {
    303             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
    304             continue;
    305         }
    306         set.complement();
    307         if(set!=alpha) {
    308             errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
    309         }
    310     }
    311     for(i=0; i<LENGTHOF(trueValues); ++i) {
    312         UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
    313         pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
    314         errorCode=U_ZERO_ERROR;
    315         UnicodeSet set(pattern, errorCode);
    316         if(U_FAILURE(errorCode)) {
    317             errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
    318             continue;
    319         }
    320         if(set!=alpha) {
    321             errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
    322         }
    323     }
    324 }
    325