1 /******************************************************************** 2 * COPYRIGHT: 3 * Copyright (c) 1997-2013, International Business Machines Corporation and 4 * others. All Rights Reserved. 5 ********************************************************************/ 6 7 #include "unicode/ustring.h" 8 #include "unicode/uchar.h" 9 #include "unicode/uniset.h" 10 #include "unicode/putil.h" 11 #include "unicode/uscript.h" 12 #include "cstring.h" 13 #include "hash.h" 14 #include "patternprops.h" 15 #include "normalizer2impl.h" 16 #include "uparse.h" 17 #include "ucdtest.h" 18 19 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0])) 20 21 static const char *ignorePropNames[]={ 22 "FC_NFKC", 23 "NFD_QC", 24 "NFC_QC", 25 "NFKD_QC", 26 "NFKC_QC", 27 "Expands_On_NFD", 28 "Expands_On_NFC", 29 "Expands_On_NFKD", 30 "Expands_On_NFKC", 31 "NFKC_CF" 32 }; 33 34 UnicodeTest::UnicodeTest() 35 { 36 UErrorCode errorCode=U_ZERO_ERROR; 37 unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode); 38 if(U_FAILURE(errorCode)) { 39 delete unknownPropertyNames; 40 unknownPropertyNames=NULL; 41 } 42 // Ignore some property names altogether. 43 for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) { 44 unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode); 45 } 46 } 47 48 UnicodeTest::~UnicodeTest() 49 { 50 delete unknownPropertyNames; 51 } 52 53 void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ ) 54 { 55 if(exec) { 56 logln("TestSuite UnicodeTest: "); 57 } 58 TESTCASE_AUTO_BEGIN; 59 TESTCASE_AUTO(TestAdditionalProperties); 60 TESTCASE_AUTO(TestBinaryValues); 61 TESTCASE_AUTO(TestConsistency); 62 TESTCASE_AUTO(TestPatternProperties); 63 TESTCASE_AUTO(TestScriptMetadata); 64 TESTCASE_AUTO_END; 65 } 66 67 //==================================================== 68 // private data used by the tests 69 //==================================================== 70 71 // test DerivedCoreProperties.txt ------------------------------------------- 72 73 // copied from genprops.c 74 static int32_t 75 getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) { 76 const char *t, *z; 77 int32_t i, j; 78 79 s=u_skipWhitespace(s); 80 for(i=0; i<countTokens; ++i) { 81 t=tokens[i]; 82 if(t!=NULL) { 83 for(j=0;; ++j) { 84 if(t[j]!=0) { 85 if(s[j]!=t[j]) { 86 break; 87 } 88 } else { 89 z=u_skipWhitespace(s+j); 90 if(*z==';' || *z==0) { 91 return i; 92 } else { 93 break; 94 } 95 } 96 } 97 } 98 } 99 return -1; 100 } 101 102 static const char *const 103 derivedPropsNames[]={ 104 "Math", 105 "Alphabetic", 106 "Lowercase", 107 "Uppercase", 108 "ID_Start", 109 "ID_Continue", 110 "XID_Start", 111 "XID_Continue", 112 "Default_Ignorable_Code_Point", 113 "Full_Composition_Exclusion", 114 "Grapheme_Extend", 115 "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */ 116 "Grapheme_Base", 117 "Cased", 118 "Case_Ignorable", 119 "Changes_When_Lowercased", 120 "Changes_When_Uppercased", 121 "Changes_When_Titlecased", 122 "Changes_When_Casefolded", 123 "Changes_When_Casemapped", 124 "Changes_When_NFKC_Casefolded" 125 }; 126 127 static const UProperty 128 derivedPropsIndex[]={ 129 UCHAR_MATH, 130 UCHAR_ALPHABETIC, 131 UCHAR_LOWERCASE, 132 UCHAR_UPPERCASE, 133 UCHAR_ID_START, 134 UCHAR_ID_CONTINUE, 135 UCHAR_XID_START, 136 UCHAR_XID_CONTINUE, 137 UCHAR_DEFAULT_IGNORABLE_CODE_POINT, 138 UCHAR_FULL_COMPOSITION_EXCLUSION, 139 UCHAR_GRAPHEME_EXTEND, 140 UCHAR_GRAPHEME_LINK, 141 UCHAR_GRAPHEME_BASE, 142 UCHAR_CASED, 143 UCHAR_CASE_IGNORABLE, 144 UCHAR_CHANGES_WHEN_LOWERCASED, 145 UCHAR_CHANGES_WHEN_UPPERCASED, 146 UCHAR_CHANGES_WHEN_TITLECASED, 147 UCHAR_CHANGES_WHEN_CASEFOLDED, 148 UCHAR_CHANGES_WHEN_CASEMAPPED, 149 UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED 150 }; 151 152 static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 }; 153 154 enum { MAX_ERRORS=50 }; 155 156 U_CFUNC void U_CALLCONV 157 derivedPropsLineFn(void *context, 158 char *fields[][2], int32_t /* fieldCount */, 159 UErrorCode *pErrorCode) 160 { 161 UnicodeTest *me=(UnicodeTest *)context; 162 uint32_t start, end; 163 int32_t i; 164 165 u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode); 166 if(U_FAILURE(*pErrorCode)) { 167 me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]); 168 return; 169 } 170 171 /* parse derived binary property name, ignore unknown names */ 172 i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]); 173 if(i<0) { 174 UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0])); 175 propName.trim(); 176 if(me->unknownPropertyNames->find(propName)==NULL) { 177 UErrorCode errorCode=U_ZERO_ERROR; 178 me->unknownPropertyNames->puti(propName, 1, errorCode); 179 me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]); 180 } 181 return; 182 } 183 184 me->derivedProps[i].add(start, end); 185 } 186 187 void UnicodeTest::TestAdditionalProperties() { 188 #if !UCONFIG_NO_NORMALIZATION 189 // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt 190 if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) { 191 errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n", 192 LENGTHOF(derivedPropsNames)); 193 return; 194 } 195 if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) { 196 errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n"); 197 return; 198 } 199 200 char newPath[256]; 201 char backupPath[256]; 202 char *fields[2][2]; 203 UErrorCode errorCode=U_ZERO_ERROR; 204 205 /* Look inside ICU_DATA first */ 206 strcpy(newPath, pathToDataDirectory()); 207 strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt"); 208 209 // As a fallback, try to guess where the source data was located 210 // at the time ICU was built, and look there. 211 # ifdef U_TOPSRCDIR 212 strcpy(backupPath, U_TOPSRCDIR U_FILE_SEP_STRING "data"); 213 # else 214 strcpy(backupPath, loadTestData(errorCode)); 215 strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data"); 216 # endif 217 strcat(backupPath, U_FILE_SEP_STRING); 218 strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt"); 219 220 char *path=newPath; 221 u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode); 222 223 if(errorCode==U_FILE_ACCESS_ERROR) { 224 errorCode=U_ZERO_ERROR; 225 path=backupPath; 226 u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode); 227 } 228 if(U_FAILURE(errorCode)) { 229 errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode)); 230 return; 231 } 232 char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt"); 233 strcpy(basename, "DerivedNormalizationProps.txt"); 234 u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode); 235 if(U_FAILURE(errorCode)) { 236 errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode)); 237 return; 238 } 239 240 // now we have all derived core properties in the UnicodeSets 241 // run them all through the API 242 int32_t rangeCount, range; 243 uint32_t i; 244 UChar32 start, end; 245 246 // test all TRUE properties 247 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) { 248 rangeCount=derivedProps[i].getRangeCount(); 249 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) { 250 start=derivedProps[i].getRangeStart(range); 251 end=derivedProps[i].getRangeEnd(range); 252 for(; start<=end; ++start) { 253 if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) { 254 dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]); 255 if(++numErrors[i]>=MAX_ERRORS) { 256 dataerrln("Too many errors, moving to the next test"); 257 break; 258 } 259 } 260 } 261 } 262 } 263 264 // invert all properties 265 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) { 266 derivedProps[i].complement(); 267 } 268 269 // test all FALSE properties 270 for(i=0; i<LENGTHOF(derivedPropsNames); ++i) { 271 rangeCount=derivedProps[i].getRangeCount(); 272 for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) { 273 start=derivedProps[i].getRangeStart(range); 274 end=derivedProps[i].getRangeEnd(range); 275 for(; start<=end; ++start) { 276 if(u_hasBinaryProperty(start, derivedPropsIndex[i])) { 277 errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]); 278 if(++numErrors[i]>=MAX_ERRORS) { 279 errln("Too many errors, moving to the next test"); 280 break; 281 } 282 } 283 } 284 } 285 } 286 #endif /* !UCONFIG_NO_NORMALIZATION */ 287 } 288 289 void UnicodeTest::TestBinaryValues() { 290 /* 291 * Unicode 5.1 explicitly defines binary property value aliases. 292 * Verify that they are all recognized. 293 */ 294 UErrorCode errorCode=U_ZERO_ERROR; 295 UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode); 296 if(U_FAILURE(errorCode)) { 297 dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode)); 298 return; 299 } 300 301 static const char *const falseValues[]={ "N", "No", "F", "False" }; 302 static const char *const trueValues[]={ "Y", "Yes", "T", "True" }; 303 int32_t i; 304 for(i=0; i<LENGTHOF(falseValues); ++i) { 305 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]"); 306 pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV)); 307 errorCode=U_ZERO_ERROR; 308 UnicodeSet set(pattern, errorCode); 309 if(U_FAILURE(errorCode)) { 310 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode)); 311 continue; 312 } 313 set.complement(); 314 if(set!=alpha) { 315 errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]); 316 } 317 } 318 for(i=0; i<LENGTHOF(trueValues); ++i) { 319 UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]"); 320 pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV)); 321 errorCode=U_ZERO_ERROR; 322 UnicodeSet set(pattern, errorCode); 323 if(U_FAILURE(errorCode)) { 324 errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode)); 325 continue; 326 } 327 if(set!=alpha) { 328 errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]); 329 } 330 } 331 } 332 333 void UnicodeTest::TestConsistency() { 334 #if !UCONFIG_NO_NORMALIZATION 335 /* 336 * Test for an example that getCanonStartSet() delivers 337 * all characters that compose from the input one, 338 * even in multiple steps. 339 * For example, the set for "I" (0049) should contain both 340 * I-diaeresis (00CF) and I-diaeresis-acute (1E2E). 341 * In general, the set for the middle such character should be a subset 342 * of the set for the first. 343 */ 344 IcuTestErrorCode errorCode(*this, "TestConsistency"); 345 const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode); 346 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode); 347 if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) { 348 dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n", 349 errorCode.errorName()); 350 errorCode.reset(); 351 return; 352 } 353 354 UnicodeSet set1, set2; 355 if (nfcImpl->getCanonStartSet(0x49, set1)) { 356 /* enumerate all characters that are plausible to be latin letters */ 357 for(UChar start=0xa0; start<0x2000; ++start) { 358 UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode); 359 if(decomp.length()>1 && decomp[0]==0x49) { 360 set2.add(start); 361 } 362 } 363 364 if (set1!=set2) { 365 errln("[canon start set of 0049] != [all c with canon decomp with 0049]"); 366 } 367 // This was available in cucdtst.c but the test had to move to intltest 368 // because the new internal normalization functions are in C++. 369 //compareUSets(set1, set2, 370 // "[canon start set of 0049]", "[all c with canon decomp with 0049]", 371 // TRUE); 372 } else { 373 errln("NFC.getCanonStartSet() returned FALSE"); 374 } 375 #endif 376 } 377 378 /** 379 * Test various implementations of Pattern_Syntax & Pattern_White_Space. 380 */ 381 void UnicodeTest::TestPatternProperties() { 382 IcuTestErrorCode errorCode(*this, "TestPatternProperties()"); 383 UnicodeSet syn_pp; 384 UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode); 385 UnicodeSet syn_list( 386 "[!-/\\:-@\\[-\\^`\\{-~" 387 "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7" 388 "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775" 389 "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode); 390 UnicodeSet ws_pp; 391 UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode); 392 UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode); 393 UnicodeSet syn_ws_pp; 394 UnicodeSet syn_ws_prop(syn_prop); 395 syn_ws_prop.addAll(ws_prop); 396 for(UChar32 c=0; c<=0xffff; ++c) { 397 if(PatternProps::isSyntax(c)) { 398 syn_pp.add(c); 399 } 400 if(PatternProps::isWhiteSpace(c)) { 401 ws_pp.add(c); 402 } 403 if(PatternProps::isSyntaxOrWhiteSpace(c)) { 404 syn_ws_pp.add(c); 405 } 406 } 407 compareUSets(syn_pp, syn_prop, 408 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE); 409 compareUSets(syn_pp, syn_list, 410 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE); 411 compareUSets(ws_pp, ws_prop, 412 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE); 413 compareUSets(ws_pp, ws_list, 414 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE); 415 compareUSets(syn_ws_pp, syn_ws_prop, 416 "PatternProps.isSyntaxOrWhiteSpace()", 417 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE); 418 } 419 420 // So far only minimal port of Java & cucdtst.c compareUSets(). 421 UBool 422 UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b, 423 const char *a_name, const char *b_name, 424 UBool diffIsError) { 425 UBool same= a==b; 426 if(!same && diffIsError) { 427 errln("Sets are different: %s vs. %s\n", a_name, b_name); 428 } 429 return same; 430 } 431 432 namespace { 433 434 /** 435 * Maps a special script code to the most common script of its encoded characters. 436 */ 437 UScriptCode getCharScript(UScriptCode script) { 438 switch(script) { 439 case USCRIPT_SIMPLIFIED_HAN: 440 case USCRIPT_TRADITIONAL_HAN: 441 return USCRIPT_HAN; 442 case USCRIPT_JAPANESE: 443 return USCRIPT_HIRAGANA; 444 case USCRIPT_KOREAN: 445 return USCRIPT_HANGUL; 446 default: 447 return script; 448 } 449 } 450 451 } // namespace 452 453 void UnicodeTest::TestScriptMetadata() { 454 IcuTestErrorCode errorCode(*this, "TestScriptMetadata()"); 455 UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode); 456 // So far, sample characters are uppercase. 457 // Georgian is special. 458 UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode); 459 for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) { 460 UScriptCode sc = (UScriptCode)sci; 461 // Run the test with -v to see which script has failures: 462 // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 3 FAIL 463 logln(uscript_getShortName(sc)); 464 UScriptUsage usage = uscript_getUsage(sc); 465 UnicodeString sample = uscript_getSampleUnicodeString(sc); 466 UnicodeSet scriptSet; 467 scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode); 468 if(usage == USCRIPT_USAGE_NOT_ENCODED) { 469 assertTrue("not encoded, no sample", sample.isEmpty()); 470 assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc)); 471 assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc)); 472 assertFalse("not encoded, not cased", uscript_isCased(sc)); 473 assertTrue("not encoded, no characters", scriptSet.isEmpty()); 474 } else { 475 assertFalse("encoded, has a sample character", sample.isEmpty()); 476 UChar32 firstChar = sample.char32At(0); 477 UScriptCode charScript = getCharScript(sc); 478 assertEquals("script(sample(script))", 479 charScript, uscript_getScript(firstChar, errorCode)); 480 assertEquals("RTL vs. set", rtl.contains(firstChar), uscript_isRightToLeft(sc)); 481 assertEquals("cased vs. set", cased.contains(firstChar), uscript_isCased(sc)); 482 assertEquals("encoded, has characters", sc == charScript, !scriptSet.isEmpty()); 483 if(uscript_isRightToLeft(sc)) { 484 rtl.removeAll(scriptSet); 485 } 486 if(uscript_isCased(sc)) { 487 cased.removeAll(scriptSet); 488 } 489 } 490 } 491 UnicodeString pattern; 492 assertEquals("no remaining RTL characters", 493 UnicodeString("[]"), rtl.toPattern(pattern)); 494 assertEquals("no remaining cased characters", 495 UnicodeString("[]"), cased.toPattern(pattern)); 496 497 assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN)); 498 assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI)); 499 assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN)); 500 } 501
