1 /******************************************************************** 2 * COPYRIGHT: 3 * Copyright (c) 1997-2010, International Business Machines Corporation and 4 * others. All Rights Reserved. 5 ********************************************************************/ 6 7 #include "unicode/ustring.h" 8 #include "unicode/uchar.h" 9 #include "unicode/uniset.h" 10 #include "unicode/putil.h" 11 #include "cstring.h" 12 #include "hash.h" 13 #include "uparse.h" 14 #include "ucdtest.h" 15 16 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0])) 17 18 static const char *ignorePropNames[]={ 19 "FC_NFKC", 20 "NFD_QC", 21 "NFC_QC", 22 "NFKD_QC", 23 "NFKC_QC", 24 "Expands_On_NFD", 25 "Expands_On_NFC", 26 "Expands_On_NFKD", 27 "Expands_On_NFKC", 28 "NFKC_CF" 29 }; 30 31 UnicodeTest::UnicodeTest() 32 { 33 UErrorCode errorCode=U_ZERO_ERROR; 34 unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode); 35 if(U_FAILURE(errorCode)) { 36 delete unknownPropertyNames; 37 unknownPropertyNames=NULL; 38 } 39 // Ignore some property names altogether. 40 for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) { 41 unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode); 42 } 43 } 44 45 UnicodeTest::~UnicodeTest() 46 { 47 delete unknownPropertyNames; 48 } 49 50 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ ) 51 { 52 if (exec) logln("TestSuite UnicodeTest: "); 53 switch (index) { 54 case 0: name = "TestAdditionalProperties"; if(exec) TestAdditionalProperties(); break; 55 case 1: name = "TestBinaryValues"; if(exec) TestBinaryValues(); break; 56 default: name = ""; break; //needed to end loop 57 } 58 } 59 60 //==================================================== 61 // private data used by the tests 62 //==================================================== 63 64 // test DerivedCoreProperties.txt ------------------------------------------- 65 66 // copied from genprops.c 67 static int32_t 68 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) { 69 const char *t, *z; 70 int32_t i, j; 71 72 s=u_skipWhitespace(s); 73 for(i=0; i<countTokens; ++i) { 74 t=tokens[i]; 75 if(t!=NULL) { 76 for(j=0;; ++j) { 77 if(t[j]!=0) { 78 if(s[j]!=t[j]) { 79 break; 80 } 81 } else { 82 z=u_skipWhitespace(s+j); 83 if(*z==';' || *z==0) { 84 return i; 85 } else { 86 break; 87 } 88 } 89 } 90 } 91 } 92 return -1; 93 } 94 95 static const char *const 96 derivedPropsNames[]={ 97 "Math", 98 "Alphabetic", 99 "Lowercase", 100 "Uppercase", 101 "ID_Start", 102 "ID_Continue", 103 "XID_Start", 104 "XID_Continue", 105 "Default_Ignorable_Code_Point", 106 "Full_Composition_Exclusion", 107 "Grapheme_Extend", 108 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */ 109 "Grapheme_Base", 110 "Cased", 111 "Case_Ignorable", 112 "Changes_When_Lowercased", 113 "Changes_When_Uppercased", 114 "Changes_When_Titlecased", 115 "Changes_When_Casefolded", 116 "Changes_When_Casemapped", 117 "Changes_When_NFKC_Casefolded" 118 }; 119 120 static const UProperty 121 derivedPropsIndex[]={ 122 UCHAR_MATH, 123 UCHAR_ALPHABETIC, 124 UCHAR_LOWERCASE, 125 UCHAR_UPPERCASE, 126 UCHAR_ID_START, 127 UCHAR_ID_CONTINUE, 128 UCHAR_XID_START, 129 UCHAR_XID_CONTINUE, 130 UCHAR_DEFAULT_IGNORABLE_CODE_POINT, 131 UCHAR_FULL_COMPOSITION_EXCLUSION, 132 UCHAR_GRAPHEME_EXTEND, 133 UCHAR_GRAPHEME_LINK, 134 UCHAR_GRAPHEME_BASE, 135 UCHAR_CASED, 136 UCHAR_CASE_IGNORABLE, 137 UCHAR_CHANGES_WHEN_LOWERCASED, 138 UCHAR_CHANGES_WHEN_UPPERCASED, 139 UCHAR_CHANGES_WHEN_TITLECASED, 140 UCHAR_CHANGES_WHEN_CASEFOLDED, 141 UCHAR_CHANGES_WHEN_CASEMAPPED, 142 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED 143 }; 144 145 static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 }; 146 147 enum { MAX_ERRORS=50 }; 148 149 U_CFUNC void U_CALLCONV 150 derivedPropsLineFn(void *context, 151 char *fields[][2], int32_t /* fieldCount */, 152 UErrorCode *pErrorCode) 153 { 154 UnicodeTest *me=(UnicodeTest *)context; 155 uint32_t start, end; 156 int32_t i; 157 158 u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); 159 if(U_FAILURE(*pErrorCode)) { 160 me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]); 161 return; 162 } 163 164 /* parse derived binary property name, ignore unknown names */ 165 i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]); 166 if(i<0) { 167 UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0])); 168 propName.trim(); 169 if(me->unknownPropertyNames->find(propName)==NULL) { 170 UErrorCode errorCode=U_ZERO_ERROR; 171 me->unknownPropertyNames->puti(propName, 1, errorCode); 172 me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]); 173 } 174 return; 175 } 176 177 me->derivedProps[i].add(start, end); 178 } 179 180 void UnicodeTest::TestAdditionalProperties() { 181 #if !UCONFIG_NO_NORMALIZATION 182 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt 183 if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) { 184 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n", 185 LENGTHOF(derivedPropsNames)); 186 return; 187 } 188 if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) { 189 errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n"); 190 return; 191 } 192 193 char newPath[256]; 194 char backupPath[256]; 195 char *fields[2][2]; 196 UErrorCode errorCode=U_ZERO_ERROR; 197 198 /* Look inside ICU_DATA first */ 199 strcpy(newPath, pathToDataDirectory()); 200 strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt"); 201 202 // As a fallback, try to guess where the source data was located 203 // at the time ICU was built, and look there. 204 # ifdef U_TOPSRCDIR 205 strcpy(backupPath, U_TOPSRCDIR U_FILE_SEP_STRING "data"); 206 # else 207 strcpy(backupPath, loadTestData(errorCode)); 208 strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data"); 209 # endif 210 strcat(backupPath, U_FILE_SEP_STRING); 211 strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt"); 212 213 char *path=newPath; 214 u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode); 215 216 if(errorCode==U_FILE_ACCESS_ERROR) { 217 errorCode=U_ZERO_ERROR; 218 path=backupPath; 219 u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode); 220 } 221 if(U_FAILURE(errorCode)) { 222 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode)); 223 return; 224 } 225 char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt"); 226 strcpy(basename, "DerivedNormalizationProps.txt"); 227 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode); 228 if(U_FAILURE(errorCode)) { 229 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode)); 230 return; 231 } 232 233 // now we have all derived core properties in the UnicodeSets 234 // run them all through the API 235 int32_t rangeCount, range; 236 uint32_t i; 237 UChar32 start, end; 238 239 // test all TRUE properties 240 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) { 241 rangeCount=derivedProps[i].getRangeCount(); 242 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) { 243 start=derivedProps[i].getRangeStart(range); 244 end=derivedProps[i].getRangeEnd(range); 245 for(; start<=end; ++start) { 246 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) { 247 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]); 248 if(++numErrors[i]>=MAX_ERRORS) { 249 dataerrln("Too many errors, moving to the next test"); 250 break; 251 } 252 } 253 } 254 } 255 } 256 257 // invert all properties 258 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) { 259 derivedProps[i].complement(); 260 } 261 262 // test all FALSE properties 263 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) { 264 rangeCount=derivedProps[i].getRangeCount(); 265 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) { 266 start=derivedProps[i].getRangeStart(range); 267 end=derivedProps[i].getRangeEnd(range); 268 for(; start<=end; ++start) { 269 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) { 270 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]); 271 if(++numErrors[i]>=MAX_ERRORS) { 272 errln("Too many errors, moving to the next test"); 273 break; 274 } 275 } 276 } 277 } 278 } 279 #endif /* !UCONFIG_NO_NORMALIZATION */ 280 } 281 282 void UnicodeTest::TestBinaryValues() { 283 /* 284 * Unicode 5.1 explicitly defines binary property value aliases. 285 * Verify that they are all recognized. 286 */ 287 UErrorCode errorCode=U_ZERO_ERROR; 288 UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode); 289 if(U_FAILURE(errorCode)) { 290 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode)); 291 return; 292 } 293 294 static const char *const falseValues[]={ "N", "No", "F", "False" }; 295 static const char *const trueValues[]={ "Y", "Yes", "T", "True" }; 296 int32_t i; 297 for(i=0; i<LENGTHOF(falseValues); ++i) { 298 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]"); 299 pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV)); 300 errorCode=U_ZERO_ERROR; 301 UnicodeSet set(pattern, errorCode); 302 if(U_FAILURE(errorCode)) { 303 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode)); 304 continue; 305 } 306 set.complement(); 307 if(set!=alpha) { 308 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]); 309 } 310 } 311 for(i=0; i<LENGTHOF(trueValues); ++i) { 312 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]"); 313 pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV)); 314 errorCode=U_ZERO_ERROR; 315 UnicodeSet set(pattern, errorCode); 316 if(U_FAILURE(errorCode)) { 317 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode)); 318 continue; 319 } 320 if(set!=alpha) { 321 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]); 322 } 323 } 324 } 325