1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2009-2010, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: bidiconf.cpp 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2009oct16 14 * created by: Markus W. Scherer 15 * 16 * BiDi conformance test, using the Unicode BidiTest.txt file. 17 */ 18 19 #include <stdio.h> 20 #include <stdlib.h> 21 #include <string.h> 22 #include "unicode/utypes.h" 23 #include "unicode/ubidi.h" 24 #include "unicode/errorcode.h" 25 #include "unicode/localpointer.h" 26 #include "unicode/putil.h" 27 #include "unicode/unistr.h" 28 #include "intltest.h" 29 #include "uparse.h" 30 31 class BiDiConformanceTest : public IntlTest { 32 public: 33 BiDiConformanceTest() : 34 directionBits(0), lineNumber(0), levelsCount(0), orderingCount(0), 35 errorCount(0) {} 36 37 void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=NULL); 38 39 void TestBidiTest(); 40 private: 41 char *getUnidataPath(char path[]); 42 43 UBool parseLevels(const char *start); 44 UBool parseOrdering(const char *start); 45 UBool parseInputStringFromBiDiClasses(const char *&start); 46 47 UBool checkLevels(const UBiDiLevel actualLevels[], int32_t actualCount, 48 const char *paraLevelName); 49 UBool checkOrdering(UBiDi *ubidi, const char *paraLevelName); 50 51 void printErrorLine(const char *paraLevelName); 52 53 char line[10000]; 54 UBiDiLevel levels[1000]; 55 uint32_t directionBits; 56 int32_t ordering[1000]; 57 int32_t lineNumber; 58 int32_t levelsCount; 59 int32_t orderingCount; 60 int32_t errorCount; 61 UnicodeString inputString; 62 }; 63 64 extern IntlTest *createBiDiConformanceTest() { 65 return new BiDiConformanceTest(); 66 } 67 68 void BiDiConformanceTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) { 69 if(exec) { 70 logln("TestSuite BiDiConformanceTest: "); 71 } 72 switch (index) { 73 TESTCASE(0, TestBidiTest); 74 default: 75 name=""; 76 break; // needed to end the loop 77 } 78 } 79 80 // TODO: Move to a common place (IntlTest?) to avoid duplication with UnicodeTest (ucdtest.cpp). 81 char *BiDiConformanceTest::getUnidataPath(char path[]) { 82 IcuTestErrorCode errorCode(*this, "getUnidataPath"); 83 const int kUnicodeDataTxtLength=15; // strlen("UnicodeData.txt") 84 85 // Look inside ICU_DATA first. 86 strcpy(path, pathToDataDirectory()); 87 strcat(path, "unidata" U_FILE_SEP_STRING "UnicodeData.txt"); 88 FILE *f=fopen(path, "r"); 89 if(f!=NULL) { 90 fclose(f); 91 *(strchr(path, 0)-kUnicodeDataTxtLength)=0; // Remove the basename. 92 return path; 93 } 94 95 // As a fallback, try to guess where the source data was located 96 // at the time ICU was built, and look there. 97 # ifdef U_TOPSRCDIR 98 strcpy(path, U_TOPSRCDIR U_FILE_SEP_STRING "data"); 99 # else 100 strcpy(path, loadTestData(errorCode)); 101 strcat(path, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." 102 U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." 103 U_FILE_SEP_STRING "data"); 104 # endif 105 strcat(path, U_FILE_SEP_STRING); 106 strcat(path, "unidata" U_FILE_SEP_STRING "UnicodeData.txt"); 107 f=fopen(path, "r"); 108 if(f!=NULL) { 109 fclose(f); 110 *(strchr(path, 0)-kUnicodeDataTxtLength)=0; // Remove the basename. 111 return path; 112 } 113 return NULL; 114 } 115 116 U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose); 117 118 UBool BiDiConformanceTest::parseLevels(const char *start) { 119 directionBits=0; 120 levelsCount=0; 121 while(*start!=0 && *(start=u_skipWhitespace(start))!=0) { 122 if(*start=='x') { 123 levels[levelsCount++]=UBIDI_DEFAULT_LTR; 124 ++start; 125 } else { 126 char *end; 127 uint32_t value=(uint32_t)strtoul(start, &end, 10); 128 if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=0) || value>(UBIDI_MAX_EXPLICIT_LEVEL+1)) { 129 errln("@Levels: parse error at %s", start); 130 return FALSE; 131 } 132 levels[levelsCount++]=(UBiDiLevel)value; 133 directionBits|=(1<<(value&1)); 134 start=end; 135 } 136 } 137 return TRUE; 138 } 139 140 UBool BiDiConformanceTest::parseOrdering(const char *start) { 141 orderingCount=0; 142 while(*start!=0 && *(start=u_skipWhitespace(start))!=0) { 143 char *end; 144 uint32_t value=(uint32_t)strtoul(start, &end, 10); 145 if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=0) || value>=1000) { 146 errln("@Reorder: parse error at %s", start); 147 return FALSE; 148 } 149 ordering[orderingCount++]=(int32_t)value; 150 start=end; 151 } 152 return TRUE; 153 } 154 155 static const UChar charFromBiDiClass[U_CHAR_DIRECTION_COUNT]={ 156 0x6c, // 'l' for L 157 0x52, // 'R' for R 158 0x33, // '3' for EN 159 0x2d, // '-' for ES 160 0x25, // '%' for ET 161 0x39, // '9' for AN 162 0x2c, // ',' for CS 163 0x2f, // '/' for B 164 0x5f, // '_' for S 165 0x20, // ' ' for WS 166 0x3d, // '=' for ON 167 0x65, // 'e' for LRE 168 0x6f, // 'o' for LRO 169 0x41, // 'A' for AL 170 0x45, // 'E' for RLE 171 0x4f, // 'O' for RLO 172 0x2a, // '*' for PDF 173 0x60, // '`' for NSM 174 0x7c // '|' for BN 175 }; 176 177 U_CDECL_BEGIN 178 179 static UCharDirection U_CALLCONV 180 biDiConfUBiDiClassCallback(const void * /*context*/, UChar32 c) { 181 for(int i=0; i<U_CHAR_DIRECTION_COUNT; ++i) { 182 if(c==charFromBiDiClass[i]) { 183 return (UCharDirection)i; 184 } 185 } 186 // Character not in our hardcoded table. 187 // Should not occur during testing. 188 return U_BIDI_CLASS_DEFAULT; 189 } 190 191 U_CDECL_END 192 193 static const int8_t biDiClassNameLengths[U_CHAR_DIRECTION_COUNT+1]={ 194 1, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 0 195 }; 196 197 UBool BiDiConformanceTest::parseInputStringFromBiDiClasses(const char *&start) { 198 inputString.remove(); 199 /* 200 * Lengthy but fast BiDi class parser. 201 * A simple parser could terminate or extract the name string and use 202 * int32_t biDiClassInt=u_getPropertyValueEnum(UCHAR_BIDI_CLASS, bidiClassString); 203 * but that makes this test take significantly more time. 204 */ 205 while(*start!=0 && *(start=u_skipWhitespace(start))!=0 && *start!=';') { 206 UCharDirection biDiClass=U_CHAR_DIRECTION_COUNT; 207 // Compare each character once until we have a match on 208 // a complete, short BiDi class name. 209 if(start[0]=='L') { 210 if(start[1]=='R') { 211 if(start[2]=='E') { 212 biDiClass=U_LEFT_TO_RIGHT_EMBEDDING; 213 } else if(start[2]=='O') { 214 biDiClass=U_LEFT_TO_RIGHT_OVERRIDE; 215 } 216 } else { 217 biDiClass=U_LEFT_TO_RIGHT; 218 } 219 } else if(start[0]=='R') { 220 if(start[1]=='L') { 221 if(start[2]=='E') { 222 biDiClass=U_RIGHT_TO_LEFT_EMBEDDING; 223 } else if(start[2]=='O') { 224 biDiClass=U_RIGHT_TO_LEFT_OVERRIDE; 225 } 226 } else { 227 biDiClass=U_RIGHT_TO_LEFT; 228 } 229 } else if(start[0]=='E') { 230 if(start[1]=='N') { 231 biDiClass=U_EUROPEAN_NUMBER; 232 } else if(start[1]=='S') { 233 biDiClass=U_EUROPEAN_NUMBER_SEPARATOR; 234 } else if(start[1]=='T') { 235 biDiClass=U_EUROPEAN_NUMBER_TERMINATOR; 236 } 237 } else if(start[0]=='A') { 238 if(start[1]=='L') { 239 biDiClass=U_RIGHT_TO_LEFT_ARABIC; 240 } else if(start[1]=='N') { 241 biDiClass=U_ARABIC_NUMBER; 242 } 243 } else if(start[0]=='C' && start[1]=='S') { 244 biDiClass=U_COMMON_NUMBER_SEPARATOR; 245 } else if(start[0]=='B') { 246 if(start[1]=='N') { 247 biDiClass=U_BOUNDARY_NEUTRAL; 248 } else { 249 biDiClass=U_BLOCK_SEPARATOR; 250 } 251 } else if(start[0]=='S') { 252 biDiClass=U_SEGMENT_SEPARATOR; 253 } else if(start[0]=='W' && start[1]=='S') { 254 biDiClass=U_WHITE_SPACE_NEUTRAL; 255 } else if(start[0]=='O' && start[1]=='N') { 256 biDiClass=U_OTHER_NEUTRAL; 257 } else if(start[0]=='P' && start[1]=='D' && start[2]=='F') { 258 biDiClass=U_POP_DIRECTIONAL_FORMAT; 259 } else if(start[0]=='N' && start[1]=='S' && start[2]=='M') { 260 biDiClass=U_DIR_NON_SPACING_MARK; 261 } 262 // Now we verify that the class name is terminated properly, 263 // and not just the start of a longer word. 264 int8_t biDiClassNameLength=biDiClassNameLengths[biDiClass]; 265 char c=start[biDiClassNameLength]; 266 if(biDiClass==U_CHAR_DIRECTION_COUNT || (!U_IS_INV_WHITESPACE(c) && c!=';' && c!=0)) { 267 errln("BiDi class string not recognized at %s", start); 268 return FALSE; 269 } 270 inputString.append(charFromBiDiClass[biDiClass]); 271 start+=biDiClassNameLength; 272 } 273 return TRUE; 274 } 275 276 void BiDiConformanceTest::TestBidiTest() { 277 IcuTestErrorCode errorCode(*this, "TestBidiTest"); 278 const char *sourceTestDataPath=getSourceTestData(errorCode); 279 if(errorCode.logIfFailureAndReset("unable to find the source/test/testdata " 280 "folder (getSourceTestData())")) { 281 return; 282 } 283 char bidiTestPath[400]; 284 strcpy(bidiTestPath, sourceTestDataPath); 285 strcat(bidiTestPath, "BidiTest.txt"); 286 LocalStdioFilePointer bidiTestFile(fopen(bidiTestPath, "r")); 287 if(bidiTestFile.isNull()) { 288 errln("unable to open %s", bidiTestPath); 289 return; 290 } 291 LocalUBiDiPointer ubidi(ubidi_open()); 292 ubidi_setClassCallback(ubidi.getAlias(), biDiConfUBiDiClassCallback, NULL, 293 NULL, NULL, errorCode); 294 if(errorCode.logIfFailureAndReset("ubidi_setClassCallback()")) { 295 return; 296 } 297 lineNumber=0; 298 levelsCount=0; 299 orderingCount=0; 300 errorCount=0; 301 while(errorCount<10 && fgets(line, (int)sizeof(line), bidiTestFile.getAlias())!=NULL) { 302 ++lineNumber; 303 // Remove trailing comments and whitespace. 304 char *commentStart=strchr(line, '#'); 305 if(commentStart!=NULL) { 306 *commentStart=0; 307 } 308 u_rtrim(line); 309 const char *start=u_skipWhitespace(line); 310 if(*start==0) { 311 continue; // Skip empty and comment-only lines. 312 } 313 if(*start=='@') { 314 ++start; 315 if(0==strncmp(start, "Levels:", 7)) { 316 if(!parseLevels(start+7)) { 317 return; 318 } 319 } else if(0==strncmp(start, "Reorder:", 8)) { 320 if(!parseOrdering(start+8)) { 321 return; 322 } 323 } 324 // Skip unknown @Xyz: ... 325 } else { 326 if(!parseInputStringFromBiDiClasses(start)) { 327 return; 328 } 329 start=u_skipWhitespace(start); 330 if(*start!=';') { 331 errln("missing ; separator on input line %s", line); 332 return; 333 } 334 start=u_skipWhitespace(start+1); 335 char *end; 336 uint32_t bitset=(uint32_t)strtoul(start, &end, 16); 337 if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0)) { 338 errln("input bitset parse error at %s", start); 339 return; 340 } 341 // Loop over the bitset. 342 static const UBiDiLevel paraLevels[]={ UBIDI_DEFAULT_LTR, 0, 1, UBIDI_DEFAULT_RTL }; 343 static const char *const paraLevelNames[]={ "auto/LTR", "LTR", "RTL", "auto/RTL" }; 344 for(int i=0; i<=3; ++i) { 345 if(bitset&(1<<i)) { 346 ubidi_setPara(ubidi.getAlias(), inputString.getBuffer(), inputString.length(), 347 paraLevels[i], NULL, errorCode); 348 const UBiDiLevel *actualLevels=ubidi_getLevels(ubidi.getAlias(), errorCode); 349 if(errorCode.logIfFailureAndReset("ubidi_setPara() or ubidi_getLevels()")) { 350 errln("Input line %d: %s", (int)lineNumber, line); 351 return; 352 } 353 if(!checkLevels(actualLevels, ubidi_getProcessedLength(ubidi.getAlias()), 354 paraLevelNames[i])) { 355 // continue outerLoop; does not exist in C++ 356 // so just break out of the inner loop. 357 break; 358 } 359 if(!checkOrdering(ubidi.getAlias(), paraLevelNames[i])) { 360 // continue outerLoop; does not exist in C++ 361 // so just break out of the inner loop. 362 break; 363 } 364 } 365 } 366 } 367 } 368 } 369 370 static UChar printLevel(UBiDiLevel level) { 371 if(level<UBIDI_DEFAULT_LTR) { 372 return 0x30+level; 373 } else { 374 return 0x78; // 'x' 375 } 376 } 377 378 static uint32_t getDirectionBits(const UBiDiLevel actualLevels[], int32_t actualCount) { 379 uint32_t actualDirectionBits=0; 380 for(int32_t i=0; i<actualCount; ++i) { 381 actualDirectionBits|=(1<<(actualLevels[i]&1)); 382 } 383 return actualDirectionBits; 384 } 385 386 UBool BiDiConformanceTest::checkLevels(const UBiDiLevel actualLevels[], int32_t actualCount, 387 const char *paraLevelName) { 388 UBool isOk=TRUE; 389 if(levelsCount!=actualCount) { 390 errln("Wrong number of level values; expected %d actual %d", 391 (int)levelsCount, (int)actualCount); 392 isOk=FALSE; 393 } else { 394 for(int32_t i=0; i<actualCount; ++i) { 395 if(levels[i]!=actualLevels[i] && levels[i]<UBIDI_DEFAULT_LTR) { 396 if(directionBits!=3 && directionBits==getDirectionBits(actualLevels, actualCount)) { 397 // ICU used a shortcut: 398 // Since the text is unidirectional, it did not store the resolved 399 // levels but just returns all levels as the paragraph level 0 or 1. 400 // The reordering result is the same, so this is fine. 401 break; 402 } else { 403 errln("Wrong level value at index %d; expected %d actual %d", 404 (int)i, levels[i], actualLevels[i]); 405 isOk=FALSE; 406 break; 407 } 408 } 409 } 410 } 411 if(!isOk) { 412 printErrorLine(paraLevelName); 413 UnicodeString els("Expected levels: "); 414 int32_t i; 415 for(i=0; i<levelsCount; ++i) { 416 els.append((UChar)0x20).append(printLevel(levels[i])); 417 } 418 UnicodeString als("Actual levels: "); 419 for(i=0; i<actualCount; ++i) { 420 als.append((UChar)0x20).append(printLevel(actualLevels[i])); 421 } 422 errln(els); 423 errln(als); 424 } 425 return isOk; 426 } 427 428 // Note: ubidi_setReorderingOptions(ubidi, UBIDI_OPTION_REMOVE_CONTROLS); 429 // does not work for custom BiDi class assignments 430 // and anyway also removes LRM/RLM/ZWJ/ZWNJ which is not desirable here. 431 // Therefore we just skip the indexes for BiDi controls while comparing 432 // with the expected ordering that has them omitted. 433 UBool BiDiConformanceTest::checkOrdering(UBiDi *ubidi, const char *paraLevelName) { 434 UBool isOk=TRUE; 435 IcuTestErrorCode errorCode(*this, "TestBidiTest/checkOrdering()"); 436 int32_t resultLength=ubidi_getResultLength(ubidi); // visual length including BiDi controls 437 int32_t i, visualIndex; 438 // Note: It should be faster to call ubidi_countRuns()/ubidi_getVisualRun() 439 // and loop over each run's indexes, but that seems unnecessary for this test code. 440 for(i=visualIndex=0; i<resultLength; ++i) { 441 int32_t logicalIndex=ubidi_getLogicalIndex(ubidi, i, errorCode); 442 if(errorCode.logIfFailureAndReset("ubidi_getLogicalIndex()")) { 443 errln("Input line %d: %s", (int)lineNumber, line); 444 return FALSE; 445 } 446 if(levels[logicalIndex]>=UBIDI_DEFAULT_LTR) { 447 continue; // BiDi control, omitted from expected ordering. 448 } 449 if(visualIndex<orderingCount && logicalIndex!=ordering[visualIndex]) { 450 errln("Wrong ordering value at visual index %d; expected %d actual %d", 451 (int)visualIndex, ordering[visualIndex], logicalIndex); 452 isOk=FALSE; 453 break; 454 } 455 ++visualIndex; 456 } 457 // visualIndex is now the visual length minus the BiDi controls, 458 // which should match the length of the BidiTest.txt ordering. 459 if(isOk && orderingCount!=visualIndex) { 460 errln("Wrong number of ordering values; expected %d actual %d", 461 (int)orderingCount, (int)visualIndex); 462 isOk=FALSE; 463 } 464 if(!isOk) { 465 printErrorLine(paraLevelName); 466 UnicodeString eord("Expected ordering: "); 467 for(i=0; i<orderingCount; ++i) { 468 eord.append((UChar)0x20).append((UChar)(0x30+ordering[i])); 469 } 470 UnicodeString aord("Actual ordering: "); 471 for(i=0; i<resultLength; ++i) { 472 int32_t logicalIndex=ubidi_getLogicalIndex(ubidi, i, errorCode); 473 if(levels[logicalIndex]<UBIDI_DEFAULT_LTR) { 474 aord.append((UChar)0x20).append((UChar)(0x30+logicalIndex)); 475 } 476 } 477 errln(eord); 478 errln(aord); 479 } 480 return isOk; 481 } 482 483 void BiDiConformanceTest::printErrorLine(const char *paraLevelName) { 484 ++errorCount; 485 errln("Input line %5d: %s", (int)lineNumber, line); 486 errln(UnicodeString("Input string: ")+inputString); 487 errln("Para level: %s", paraLevelName); 488 } 489
