1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2003-2013, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: convtest.cpp 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2003jul15 14 * created by: Markus W. Scherer 15 * 16 * Test file for data-driven conversion tests. 17 */ 18 19 #include "unicode/utypes.h" 20 21 #if !UCONFIG_NO_LEGACY_CONVERSION 22 /* 23 * Note: Turning off all of convtest.cpp if !UCONFIG_NO_LEGACY_CONVERSION 24 * is slightly unnecessary - it removes tests for Unicode charsets 25 * like UTF-8 that should work. 26 * However, there is no easy way for the test to detect whether a test case 27 * is for a Unicode charset, so it would be difficult to only exclude those. 28 * Also, regular testing of ICU is done with all modules on, therefore 29 * not testing conversion for a custom configuration like this should be ok. 30 */ 31 32 #include "unicode/ucnv.h" 33 #include "unicode/unistr.h" 34 #include "unicode/parsepos.h" 35 #include "unicode/uniset.h" 36 #include "unicode/ustring.h" 37 #include "unicode/ures.h" 38 #include "convtest.h" 39 #include "unicode/tstdtmod.h" 40 #include <string.h> 41 #include <stdlib.h> 42 43 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) 44 45 enum { 46 // characters used in test data for callbacks 47 SUB_CB='?', 48 SKIP_CB='0', 49 STOP_CB='.', 50 ESC_CB='&' 51 }; 52 53 ConversionTest::ConversionTest() { 54 UErrorCode errorCode=U_ZERO_ERROR; 55 utf8Cnv=ucnv_open("UTF-8", &errorCode); 56 ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode); 57 if(U_FAILURE(errorCode)) { 58 errln("unable to open UTF-8 converter"); 59 } 60 } 61 62 ConversionTest::~ConversionTest() { 63 ucnv_close(utf8Cnv); 64 } 65 66 void 67 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) { 68 if (exec) logln("TestSuite ConversionTest: "); 69 switch (index) { 70 #if !UCONFIG_NO_FILE_IO 71 case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break; 72 case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break; 73 case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break; 74 #else 75 case 0: 76 case 1: 77 case 2: name="skip"; break; 78 #endif 79 case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break; 80 default: name=""; break; //needed to end loop 81 } 82 } 83 84 // test data interface ----------------------------------------------------- *** 85 86 void 87 ConversionTest::TestToUnicode() { 88 ConversionCase cc; 89 char charset[100], cbopt[4]; 90 const char *option; 91 UnicodeString s, unicode; 92 int32_t offsetsLength; 93 UConverterToUCallback callback; 94 95 TestDataModule *dataModule; 96 TestData *testData; 97 const DataMap *testCase; 98 UErrorCode errorCode; 99 int32_t i; 100 101 errorCode=U_ZERO_ERROR; 102 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode); 103 if(U_SUCCESS(errorCode)) { 104 testData=dataModule->createTestData("toUnicode", errorCode); 105 if(U_SUCCESS(errorCode)) { 106 for(i=0; testData->nextCase(testCase, errorCode); ++i) { 107 if(U_FAILURE(errorCode)) { 108 errln("error retrieving conversion/toUnicode test case %d - %s", 109 i, u_errorName(errorCode)); 110 errorCode=U_ZERO_ERROR; 111 continue; 112 } 113 114 cc.caseNr=i; 115 116 s=testCase->getString("charset", errorCode); 117 s.extract(0, 0x7fffffff, charset, sizeof(charset), ""); 118 cc.charset=charset; 119 120 // BEGIN android-added 121 // To save space, Android does not build full ISO-2022-CN tables. 122 // We skip the TestGetKeywordValuesForLocale for counting available collations. 123 if (strlen(charset) >= 8 && 124 strncmp(charset+4, "2022-CN", 4) == 0) { 125 continue; 126 } 127 // END android-added 128 129 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode); 130 unicode=testCase->getString("unicode", errorCode); 131 cc.unicode=unicode.getBuffer(); 132 cc.unicodeLength=unicode.length(); 133 134 offsetsLength=0; 135 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode); 136 if(offsetsLength==0) { 137 cc.offsets=NULL; 138 } else if(offsetsLength!=unicode.length()) { 139 errln("toUnicode[%d] unicode[%d] and offsets[%d] must have the same length", 140 i, unicode.length(), offsetsLength); 141 errorCode=U_ILLEGAL_ARGUMENT_ERROR; 142 } 143 144 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode); 145 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode); 146 147 s=testCase->getString("errorCode", errorCode); 148 if(s==UNICODE_STRING("invalid", 7)) { 149 cc.outErrorCode=U_INVALID_CHAR_FOUND; 150 } else if(s==UNICODE_STRING("illegal", 7)) { 151 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND; 152 } else if(s==UNICODE_STRING("truncated", 9)) { 153 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND; 154 } else if(s==UNICODE_STRING("illesc", 6)) { 155 cc.outErrorCode=U_ILLEGAL_ESCAPE_SEQUENCE; 156 } else if(s==UNICODE_STRING("unsuppesc", 9)) { 157 cc.outErrorCode=U_UNSUPPORTED_ESCAPE_SEQUENCE; 158 } else { 159 cc.outErrorCode=U_ZERO_ERROR; 160 } 161 162 s=testCase->getString("callback", errorCode); 163 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), ""); 164 cc.cbopt=cbopt; 165 switch(cbopt[0]) { 166 case SUB_CB: 167 callback=UCNV_TO_U_CALLBACK_SUBSTITUTE; 168 break; 169 case SKIP_CB: 170 callback=UCNV_TO_U_CALLBACK_SKIP; 171 break; 172 case STOP_CB: 173 callback=UCNV_TO_U_CALLBACK_STOP; 174 break; 175 case ESC_CB: 176 callback=UCNV_TO_U_CALLBACK_ESCAPE; 177 break; 178 default: 179 callback=NULL; 180 break; 181 } 182 option=callback==NULL ? cbopt : cbopt+1; 183 if(*option==0) { 184 option=NULL; 185 } 186 187 cc.invalidChars=testCase->getBinary(cc.invalidLength, "invalidChars", errorCode); 188 189 if(U_FAILURE(errorCode)) { 190 errln("error parsing conversion/toUnicode test case %d - %s", 191 i, u_errorName(errorCode)); 192 errorCode=U_ZERO_ERROR; 193 } else { 194 logln("TestToUnicode[%d] %s", i, charset); 195 ToUnicodeCase(cc, callback, option); 196 } 197 } 198 delete testData; 199 } 200 delete dataModule; 201 } 202 else { 203 dataerrln("Could not load test conversion data"); 204 } 205 } 206 207 void 208 ConversionTest::TestFromUnicode() { 209 ConversionCase cc; 210 char charset[100], cbopt[4]; 211 const char *option; 212 UnicodeString s, unicode, invalidUChars; 213 int32_t offsetsLength, index; 214 UConverterFromUCallback callback; 215 216 TestDataModule *dataModule; 217 TestData *testData; 218 const DataMap *testCase; 219 const UChar *p; 220 UErrorCode errorCode; 221 int32_t i, length; 222 223 errorCode=U_ZERO_ERROR; 224 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode); 225 if(U_SUCCESS(errorCode)) { 226 testData=dataModule->createTestData("fromUnicode", errorCode); 227 if(U_SUCCESS(errorCode)) { 228 for(i=0; testData->nextCase(testCase, errorCode); ++i) { 229 if(U_FAILURE(errorCode)) { 230 errln("error retrieving conversion/fromUnicode test case %d - %s", 231 i, u_errorName(errorCode)); 232 errorCode=U_ZERO_ERROR; 233 continue; 234 } 235 236 cc.caseNr=i; 237 238 s=testCase->getString("charset", errorCode); 239 s.extract(0, 0x7fffffff, charset, sizeof(charset), ""); 240 cc.charset=charset; 241 242 // BEGIN android-added 243 // To save space, Android does not build full ISO-2022-CN tables. 244 // We skip the TestGetKeywordValuesForLocale for counting available collations. 245 if (strlen(charset) >= 8 && 246 strncmp(charset+4, "2022-CN", 4) == 0) { 247 continue; 248 } 249 // END android-added 250 251 unicode=testCase->getString("unicode", errorCode); 252 cc.unicode=unicode.getBuffer(); 253 cc.unicodeLength=unicode.length(); 254 cc.bytes=testCase->getBinary(cc.bytesLength, "bytes", errorCode); 255 256 offsetsLength=0; 257 cc.offsets=testCase->getIntVector(offsetsLength, "offsets", errorCode); 258 if(offsetsLength==0) { 259 cc.offsets=NULL; 260 } else if(offsetsLength!=cc.bytesLength) { 261 errln("fromUnicode[%d] bytes[%d] and offsets[%d] must have the same length", 262 i, cc.bytesLength, offsetsLength); 263 errorCode=U_ILLEGAL_ARGUMENT_ERROR; 264 } 265 266 cc.finalFlush= 0!=testCase->getInt28("flush", errorCode); 267 cc.fallbacks= 0!=testCase->getInt28("fallbacks", errorCode); 268 269 s=testCase->getString("errorCode", errorCode); 270 if(s==UNICODE_STRING("invalid", 7)) { 271 cc.outErrorCode=U_INVALID_CHAR_FOUND; 272 } else if(s==UNICODE_STRING("illegal", 7)) { 273 cc.outErrorCode=U_ILLEGAL_CHAR_FOUND; 274 } else if(s==UNICODE_STRING("truncated", 9)) { 275 cc.outErrorCode=U_TRUNCATED_CHAR_FOUND; 276 } else { 277 cc.outErrorCode=U_ZERO_ERROR; 278 } 279 280 s=testCase->getString("callback", errorCode); 281 cc.setSub=0; // default: no subchar 282 283 if((index=s.indexOf((UChar)0))>0) { 284 // read NUL-separated subchar first, if any 285 // copy the subchar from Latin-1 characters 286 // start after the NUL 287 p=s.getTerminatedBuffer(); 288 length=index+1; 289 p+=length; 290 length=s.length()-length; 291 if(length<=0 || length>=(int32_t)sizeof(cc.subchar)) { 292 errorCode=U_ILLEGAL_ARGUMENT_ERROR; 293 } else { 294 int32_t j; 295 296 for(j=0; j<length; ++j) { 297 cc.subchar[j]=(char)p[j]; 298 } 299 // NUL-terminate the subchar 300 cc.subchar[j]=0; 301 cc.setSub=1; 302 } 303 304 // remove the NUL and subchar from s 305 s.truncate(index); 306 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ { 307 // read a substitution string, separated by an equal sign 308 p=s.getBuffer()+index+1; 309 length=s.length()-(index+1); 310 if(length<0 || length>=LENGTHOF(cc.subString)) { 311 errorCode=U_ILLEGAL_ARGUMENT_ERROR; 312 } else { 313 u_memcpy(cc.subString, p, length); 314 // NUL-terminate the subString 315 cc.subString[length]=0; 316 cc.setSub=-1; 317 } 318 319 // remove the equal sign and subString from s 320 s.truncate(index); 321 } 322 323 s.extract(0, 0x7fffffff, cbopt, sizeof(cbopt), ""); 324 cc.cbopt=cbopt; 325 switch(cbopt[0]) { 326 case SUB_CB: 327 callback=UCNV_FROM_U_CALLBACK_SUBSTITUTE; 328 break; 329 case SKIP_CB: 330 callback=UCNV_FROM_U_CALLBACK_SKIP; 331 break; 332 case STOP_CB: 333 callback=UCNV_FROM_U_CALLBACK_STOP; 334 break; 335 case ESC_CB: 336 callback=UCNV_FROM_U_CALLBACK_ESCAPE; 337 break; 338 default: 339 callback=NULL; 340 break; 341 } 342 option=callback==NULL ? cbopt : cbopt+1; 343 if(*option==0) { 344 option=NULL; 345 } 346 347 invalidUChars=testCase->getString("invalidUChars", errorCode); 348 cc.invalidUChars=invalidUChars.getBuffer(); 349 cc.invalidLength=invalidUChars.length(); 350 351 if(U_FAILURE(errorCode)) { 352 errln("error parsing conversion/fromUnicode test case %d - %s", 353 i, u_errorName(errorCode)); 354 errorCode=U_ZERO_ERROR; 355 } else { 356 logln("TestFromUnicode[%d] %s", i, charset); 357 FromUnicodeCase(cc, callback, option); 358 } 359 } 360 delete testData; 361 } 362 delete dataModule; 363 } 364 else { 365 dataerrln("Could not load test conversion data"); 366 } 367 } 368 369 static const UChar ellipsis[]={ 0x2e, 0x2e, 0x2e }; 370 371 void 372 ConversionTest::TestGetUnicodeSet() { 373 char charset[100]; 374 UnicodeString s, map, mapnot; 375 int32_t which; 376 377 ParsePosition pos; 378 UnicodeSet cnvSet, mapSet, mapnotSet, diffSet; 379 UnicodeSet *cnvSetPtr = &cnvSet; 380 LocalUConverterPointer cnv; 381 382 TestDataModule *dataModule; 383 TestData *testData; 384 const DataMap *testCase; 385 UErrorCode errorCode; 386 int32_t i; 387 388 errorCode=U_ZERO_ERROR; 389 dataModule=TestDataModule::getTestDataModule("conversion", *this, errorCode); 390 if(U_SUCCESS(errorCode)) { 391 testData=dataModule->createTestData("getUnicodeSet", errorCode); 392 if(U_SUCCESS(errorCode)) { 393 for(i=0; testData->nextCase(testCase, errorCode); ++i) { 394 if(U_FAILURE(errorCode)) { 395 errln("error retrieving conversion/getUnicodeSet test case %d - %s", 396 i, u_errorName(errorCode)); 397 errorCode=U_ZERO_ERROR; 398 continue; 399 } 400 401 s=testCase->getString("charset", errorCode); 402 s.extract(0, 0x7fffffff, charset, sizeof(charset), ""); 403 404 // BEGIN android-added 405 // To save space, Android does not build full ISO-2022-CN tables. 406 // We skip the TestGetKeywordValuesForLocale for counting available collations. 407 if (strlen(charset) >= 8 && 408 strncmp(charset+4, "2022-CN", 4) == 0) { 409 continue; 410 } 411 // END android-added 412 413 map=testCase->getString("map", errorCode); 414 mapnot=testCase->getString("mapnot", errorCode); 415 416 which=testCase->getInt28("which", errorCode); 417 418 if(U_FAILURE(errorCode)) { 419 errln("error parsing conversion/getUnicodeSet test case %d - %s", 420 i, u_errorName(errorCode)); 421 errorCode=U_ZERO_ERROR; 422 continue; 423 } 424 425 // test this test case 426 mapSet.clear(); 427 mapnotSet.clear(); 428 429 pos.setIndex(0); 430 mapSet.applyPattern(map, pos, 0, NULL, errorCode); 431 if(U_FAILURE(errorCode) || pos.getIndex()!=map.length()) { 432 errln("error creating the map set for conversion/getUnicodeSet test case %d - %s\n" 433 " error index %d index %d U+%04x", 434 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), map.char32At(pos.getIndex())); 435 errorCode=U_ZERO_ERROR; 436 continue; 437 } 438 439 pos.setIndex(0); 440 mapnotSet.applyPattern(mapnot, pos, 0, NULL, errorCode); 441 if(U_FAILURE(errorCode) || pos.getIndex()!=mapnot.length()) { 442 errln("error creating the mapnot set for conversion/getUnicodeSet test case %d - %s\n" 443 " error index %d index %d U+%04x", 444 i, u_errorName(errorCode), pos.getErrorIndex(), pos.getIndex(), mapnot.char32At(pos.getIndex())); 445 errorCode=U_ZERO_ERROR; 446 continue; 447 } 448 449 logln("TestGetUnicodeSet[%d] %s", i, charset); 450 451 cnv.adoptInstead(cnv_open(charset, errorCode)); 452 if(U_FAILURE(errorCode)) { 453 errcheckln(errorCode, "error opening \"%s\" for conversion/getUnicodeSet test case %d - %s", 454 charset, i, u_errorName(errorCode)); 455 errorCode=U_ZERO_ERROR; 456 continue; 457 } 458 459 ucnv_getUnicodeSet(cnv.getAlias(), cnvSetPtr->toUSet(), (UConverterUnicodeSet)which, &errorCode); 460 461 if(U_FAILURE(errorCode)) { 462 errln("error in ucnv_getUnicodeSet(\"%s\") for conversion/getUnicodeSet test case %d - %s", 463 charset, i, u_errorName(errorCode)); 464 errorCode=U_ZERO_ERROR; 465 continue; 466 } 467 468 // are there items that must be in cnvSet but are not? 469 (diffSet=mapSet).removeAll(cnvSet); 470 if(!diffSet.isEmpty()) { 471 diffSet.toPattern(s, TRUE); 472 if(s.length()>100) { 473 s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis)); 474 } 475 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d", 476 charset, i); 477 errln(s); 478 } 479 480 // are there items that must not be in cnvSet but are? 481 (diffSet=mapnotSet).retainAll(cnvSet); 482 if(!diffSet.isEmpty()) { 483 diffSet.toPattern(s, TRUE); 484 if(s.length()>100) { 485 s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis)); 486 } 487 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d", 488 charset, i); 489 errln(s); 490 } 491 } 492 delete testData; 493 } 494 delete dataModule; 495 } 496 else { 497 dataerrln("Could not load test conversion data"); 498 } 499 } 500 501 U_CDECL_BEGIN 502 static void U_CALLCONV 503 getUnicodeSetCallback(const void *context, 504 UConverterFromUnicodeArgs * /*fromUArgs*/, 505 const UChar* /*codeUnits*/, 506 int32_t /*length*/, 507 UChar32 codePoint, 508 UConverterCallbackReason reason, 509 UErrorCode *pErrorCode) { 510 if(reason<=UCNV_IRREGULAR) { 511 ((UnicodeSet *)context)->remove(codePoint); // the converter cannot convert this code point 512 *pErrorCode=U_ZERO_ERROR; // skip 513 } // else ignore the reset, close and clone calls. 514 } 515 U_CDECL_END 516 517 // Compare ucnv_getUnicodeSet() with the set of characters that can be converted. 518 void 519 ConversionTest::TestGetUnicodeSet2() { 520 // Build a string with all code points. 521 UChar32 cpLimit; 522 int32_t s0Length; 523 if(quick) { 524 cpLimit=s0Length=0x10000; // BMP only 525 } else { 526 cpLimit=0x110000; 527 s0Length=0x10000+0x200000; // BMP + surrogate pairs 528 } 529 UChar *s0=new UChar[s0Length]; 530 if(s0==NULL) { 531 return; 532 } 533 UChar *s=s0; 534 UChar32 c; 535 UChar c2; 536 // low BMP 537 for(c=0; c<=0xd7ff; ++c) { 538 *s++=(UChar)c; 539 } 540 // trail surrogates 541 for(c=0xdc00; c<=0xdfff; ++c) { 542 *s++=(UChar)c; 543 } 544 // lead surrogates 545 // (after trails so that there is not even one surrogate pair in between) 546 for(c=0xd800; c<=0xdbff; ++c) { 547 *s++=(UChar)c; 548 } 549 // high BMP 550 for(c=0xe000; c<=0xffff; ++c) { 551 *s++=(UChar)c; 552 } 553 // supplementary code points = surrogate pairs 554 if(cpLimit==0x110000) { 555 for(c=0xd800; c<=0xdbff; ++c) { 556 for(c2=0xdc00; c2<=0xdfff; ++c2) { 557 *s++=(UChar)c; 558 *s++=c2; 559 } 560 } 561 } 562 563 static const char *const cnvNames[]={ 564 "UTF-8", 565 "UTF-7", 566 "UTF-16", 567 "US-ASCII", 568 "ISO-8859-1", 569 "windows-1252", 570 "Shift-JIS", 571 "ibm-1390", // EBCDIC_STATEFUL table 572 "ibm-16684", // DBCS-only extension table based on EBCDIC_STATEFUL table 573 "HZ", 574 "ISO-2022-JP", 575 "JIS7", 576 "ISO-2022-CN", 577 "ISO-2022-CN-EXT", 578 "LMBCS" 579 }; 580 LocalUConverterPointer cnv; 581 char buffer[1024]; 582 int32_t i; 583 for(i=0; i<LENGTHOF(cnvNames); ++i) { 584 UErrorCode errorCode=U_ZERO_ERROR; 585 cnv.adoptInstead(cnv_open(cnvNames[i], errorCode)); 586 if(U_FAILURE(errorCode)) { 587 errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode)); 588 continue; 589 } 590 UnicodeSet expected; 591 ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode); 592 if(U_FAILURE(errorCode)) { 593 errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode)); 594 continue; 595 } 596 UConverterUnicodeSet which; 597 for(which=UCNV_ROUNDTRIP_SET; which<UCNV_SET_COUNT; which=(UConverterUnicodeSet)((int)which+1)) { 598 if(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) { 599 ucnv_setFallback(cnv.getAlias(), TRUE); 600 } 601 expected.add(0, cpLimit-1); 602 s=s0; 603 UBool flush; 604 do { 605 char *t=buffer; 606 flush=(UBool)(s==s0+s0Length); 607 ucnv_fromUnicode(cnv.getAlias(), &t, buffer+sizeof(buffer), (const UChar **)&s, s0+s0Length, NULL, flush, &errorCode); 608 if(U_FAILURE(errorCode)) { 609 if(errorCode==U_BUFFER_OVERFLOW_ERROR) { 610 errorCode=U_ZERO_ERROR; 611 continue; 612 } else { 613 break; // unexpected error, should not occur 614 } 615 } 616 } while(!flush); 617 UnicodeSet set; 618 ucnv_getUnicodeSet(cnv.getAlias(), set.toUSet(), which, &errorCode); 619 if(cpLimit<0x110000) { 620 set.remove(cpLimit, 0x10ffff); 621 } 622 if(which==UCNV_ROUNDTRIP_SET) { 623 // ignore PUA code points because they will be converted even if they 624 // are fallbacks and when other fallbacks are turned off, 625 // but ucnv_getUnicodeSet(UCNV_ROUNDTRIP_SET) delivers true roundtrips 626 expected.remove(0xe000, 0xf8ff); 627 expected.remove(0xf0000, 0xffffd); 628 expected.remove(0x100000, 0x10fffd); 629 set.remove(0xe000, 0xf8ff); 630 set.remove(0xf0000, 0xffffd); 631 set.remove(0x100000, 0x10fffd); 632 } 633 if(set!=expected) { 634 // First try to see if we have different sets because ucnv_getUnicodeSet() 635 // added strings: The above conversion method does not tell us what strings might be convertible. 636 // Remove strings from the set and compare again. 637 // Unfortunately, there are no good, direct set methods for finding out whether there are strings 638 // in the set, nor for enumerating or removing just them. 639 // Intersect all code points with the set. The intersection will not contain strings. 640 UnicodeSet temp(0, 0x10ffff); 641 temp.retainAll(set); 642 set=temp; 643 } 644 if(set!=expected) { 645 UnicodeSet diffSet; 646 UnicodeString out; 647 648 // are there items that must be in the set but are not? 649 (diffSet=expected).removeAll(set); 650 if(!diffSet.isEmpty()) { 651 diffSet.toPattern(out, TRUE); 652 if(out.length()>100) { 653 out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis)); 654 } 655 errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d", 656 cnvNames[i], which); 657 errln(out); 658 } 659 660 // are there items that must not be in the set but are? 661 (diffSet=set).removeAll(expected); 662 if(!diffSet.isEmpty()) { 663 diffSet.toPattern(out, TRUE); 664 if(out.length()>100) { 665 out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis)); 666 } 667 errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d", 668 cnvNames[i], which); 669 errln(out); 670 } 671 } 672 } 673 } 674 675 delete [] s0; 676 } 677 678 // open testdata or ICU data converter ------------------------------------- *** 679 680 UConverter * 681 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) { 682 if(name!=NULL && *name=='+') { 683 // Converter names that start with '+' are ignored in ICU4J tests. 684 ++name; 685 } 686 if(name!=NULL && *name=='*') { 687 /* loadTestData(): set the data directory */ 688 return ucnv_openPackage(loadTestData(errorCode), name+1, &errorCode); 689 } else { 690 return ucnv_open(name, &errorCode); 691 } 692 } 693 694 // output helpers ---------------------------------------------------------- *** 695 696 static inline char 697 hexDigit(uint8_t digit) { 698 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit); 699 } 700 701 static char * 702 printBytes(const uint8_t *bytes, int32_t length, char *out) { 703 uint8_t b; 704 705 if(length>0) { 706 b=*bytes++; 707 --length; 708 *out++=hexDigit((uint8_t)(b>>4)); 709 *out++=hexDigit((uint8_t)(b&0xf)); 710 } 711 712 while(length>0) { 713 b=*bytes++; 714 --length; 715 *out++=' '; 716 *out++=hexDigit((uint8_t)(b>>4)); 717 *out++=hexDigit((uint8_t)(b&0xf)); 718 } 719 *out++=0; 720 return out; 721 } 722 723 static char * 724 printUnicode(const UChar *unicode, int32_t length, char *out) { 725 UChar32 c; 726 int32_t i; 727 728 for(i=0; i<length;) { 729 if(i>0) { 730 *out++=' '; 731 } 732 U16_NEXT(unicode, i, length, c); 733 // write 4..6 digits 734 if(c>=0x100000) { 735 *out++='1'; 736 } 737 if(c>=0x10000) { 738 *out++=hexDigit((uint8_t)((c>>16)&0xf)); 739 } 740 *out++=hexDigit((uint8_t)((c>>12)&0xf)); 741 *out++=hexDigit((uint8_t)((c>>8)&0xf)); 742 *out++=hexDigit((uint8_t)((c>>4)&0xf)); 743 *out++=hexDigit((uint8_t)(c&0xf)); 744 } 745 *out++=0; 746 return out; 747 } 748 749 static char * 750 printOffsets(const int32_t *offsets, int32_t length, char *out) { 751 int32_t i, o, d; 752 753 if(offsets==NULL) { 754 length=0; 755 } 756 757 for(i=0; i<length; ++i) { 758 if(i>0) { 759 *out++=' '; 760 } 761 o=offsets[i]; 762 763 // print all offsets with 2 characters each (-x, -9..99, xx) 764 if(o<-9) { 765 *out++='-'; 766 *out++='x'; 767 } else if(o<0) { 768 *out++='-'; 769 *out++=(char)('0'-o); 770 } else if(o<=99) { 771 *out++=(d=o/10)==0 ? ' ' : (char)('0'+d); 772 *out++=(char)('0'+o%10); 773 } else /* o>99 */ { 774 *out++='x'; 775 *out++='x'; 776 } 777 } 778 *out++=0; 779 return out; 780 } 781 782 // toUnicode test worker functions ----------------------------------------- *** 783 784 static int32_t 785 stepToUnicode(ConversionCase &cc, UConverter *cnv, 786 UChar *result, int32_t resultCapacity, 787 int32_t *resultOffsets, /* also resultCapacity */ 788 int32_t step, 789 UErrorCode *pErrorCode) { 790 const char *source, *sourceLimit, *bytesLimit; 791 UChar *target, *targetLimit, *resultLimit; 792 UBool flush; 793 794 source=(const char *)cc.bytes; 795 target=result; 796 bytesLimit=source+cc.bytesLength; 797 resultLimit=result+resultCapacity; 798 799 if(step>=0) { 800 // call ucnv_toUnicode() with in/out buffers no larger than (step) at a time 801 // move only one buffer (in vs. out) at a time to be extra mean 802 // step==0 performs bulk conversion and generates offsets 803 804 // initialize the partial limits for the loop 805 if(step==0) { 806 // use the entire buffers 807 sourceLimit=bytesLimit; 808 targetLimit=resultLimit; 809 flush=cc.finalFlush; 810 } else { 811 // start with empty partial buffers 812 sourceLimit=source; 813 targetLimit=target; 814 flush=FALSE; 815 816 // output offsets only for bulk conversion 817 resultOffsets=NULL; 818 } 819 820 for(;;) { 821 // resetting the opposite conversion direction must not affect this one 822 ucnv_resetFromUnicode(cnv); 823 824 // convert 825 ucnv_toUnicode(cnv, 826 &target, targetLimit, 827 &source, sourceLimit, 828 resultOffsets, 829 flush, pErrorCode); 830 831 // check pointers and errors 832 if(source>sourceLimit || target>targetLimit) { 833 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 834 break; 835 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { 836 if(target!=targetLimit) { 837 // buffer overflow must only be set when the target is filled 838 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 839 break; 840 } else if(targetLimit==resultLimit) { 841 // not just a partial overflow 842 break; 843 } 844 845 // the partial target is filled, set a new limit, reset the error and continue 846 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit; 847 *pErrorCode=U_ZERO_ERROR; 848 } else if(U_FAILURE(*pErrorCode)) { 849 // some other error occurred, done 850 break; 851 } else { 852 if(source!=sourceLimit) { 853 // when no error occurs, then the input must be consumed 854 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 855 break; 856 } 857 858 if(sourceLimit==bytesLimit) { 859 // we are done 860 break; 861 } 862 863 // the partial conversion succeeded, set a new limit and continue 864 sourceLimit=(bytesLimit-source)>=step ? source+step : bytesLimit; 865 flush=(UBool)(cc.finalFlush && sourceLimit==bytesLimit); 866 } 867 } 868 } else /* step<0 */ { 869 /* 870 * step==-1: call only ucnv_getNextUChar() 871 * otherwise alternate between ucnv_toUnicode() and ucnv_getNextUChar() 872 * if step==-2 or -3, then give ucnv_toUnicode() the whole remaining input, 873 * else give it at most (-step-2)/2 bytes 874 */ 875 UChar32 c; 876 877 // end the loop by getting an index out of bounds error 878 for(;;) { 879 // resetting the opposite conversion direction must not affect this one 880 ucnv_resetFromUnicode(cnv); 881 882 // convert 883 if((step&1)!=0 /* odd: -1, -3, -5, ... */) { 884 sourceLimit=source; // use sourceLimit not as a real limit 885 // but to remember the pre-getNextUChar source pointer 886 c=ucnv_getNextUChar(cnv, &source, bytesLimit, pErrorCode); 887 888 // check pointers and errors 889 if(*pErrorCode==U_INDEX_OUTOFBOUNDS_ERROR) { 890 if(source!=bytesLimit) { 891 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 892 } else { 893 *pErrorCode=U_ZERO_ERROR; 894 } 895 break; 896 } else if(U_FAILURE(*pErrorCode)) { 897 break; 898 } 899 // source may not move if c is from previous overflow 900 901 if(target==resultLimit) { 902 *pErrorCode=U_BUFFER_OVERFLOW_ERROR; 903 break; 904 } 905 if(c<=0xffff) { 906 *target++=(UChar)c; 907 } else { 908 *target++=U16_LEAD(c); 909 if(target==resultLimit) { 910 *pErrorCode=U_BUFFER_OVERFLOW_ERROR; 911 break; 912 } 913 *target++=U16_TRAIL(c); 914 } 915 916 // alternate between -n-1 and -n but leave -1 alone 917 if(step<-1) { 918 ++step; 919 } 920 } else /* step is even */ { 921 // allow only one UChar output 922 targetLimit=target<resultLimit ? target+1 : resultLimit; 923 924 // as with ucnv_getNextUChar(), we always flush (if we go to bytesLimit) 925 // and never output offsets 926 if(step==-2) { 927 sourceLimit=bytesLimit; 928 } else { 929 sourceLimit=source+(-step-2)/2; 930 if(sourceLimit>bytesLimit) { 931 sourceLimit=bytesLimit; 932 } 933 } 934 935 ucnv_toUnicode(cnv, 936 &target, targetLimit, 937 &source, sourceLimit, 938 NULL, (UBool)(sourceLimit==bytesLimit), pErrorCode); 939 940 // check pointers and errors 941 if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { 942 if(target!=targetLimit) { 943 // buffer overflow must only be set when the target is filled 944 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 945 break; 946 } else if(targetLimit==resultLimit) { 947 // not just a partial overflow 948 break; 949 } 950 951 // the partial target is filled, set a new limit and continue 952 *pErrorCode=U_ZERO_ERROR; 953 } else if(U_FAILURE(*pErrorCode)) { 954 // some other error occurred, done 955 break; 956 } else { 957 if(source!=sourceLimit) { 958 // when no error occurs, then the input must be consumed 959 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 960 break; 961 } 962 963 // we are done (flush==TRUE) but we continue, to get the index out of bounds error above 964 } 965 966 --step; 967 } 968 } 969 } 970 971 return (int32_t)(target-result); 972 } 973 974 UBool 975 ConversionTest::ToUnicodeCase(ConversionCase &cc, UConverterToUCallback callback, const char *option) { 976 // open the converter 977 IcuTestErrorCode errorCode(*this, "ToUnicodeCase"); 978 LocalUConverterPointer cnv(cnv_open(cc.charset, errorCode)); 979 if(errorCode.isFailure()) { 980 errcheckln(errorCode, "toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s", 981 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, errorCode.errorName()); 982 errorCode.reset(); 983 return FALSE; 984 } 985 986 // set the callback 987 if(callback!=NULL) { 988 ucnv_setToUCallBack(cnv.getAlias(), callback, option, NULL, NULL, errorCode); 989 if(U_FAILURE(errorCode)) { 990 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setToUCallBack() failed - %s", 991 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode)); 992 return FALSE; 993 } 994 } 995 996 int32_t resultOffsets[256]; 997 UChar result[256]; 998 int32_t resultLength; 999 UBool ok; 1000 1001 static const struct { 1002 int32_t step; 1003 const char *name; 1004 } steps[]={ 1005 { 0, "bulk" }, // must be first for offsets to be checked 1006 { 1, "step=1" }, 1007 { 3, "step=3" }, 1008 { 7, "step=7" }, 1009 { -1, "getNext" }, 1010 { -2, "toU(bulk)+getNext" }, 1011 { -3, "getNext+toU(bulk)" }, 1012 { -4, "toU(1)+getNext" }, 1013 { -5, "getNext+toU(1)" }, 1014 { -12, "toU(5)+getNext" }, 1015 { -13, "getNext+toU(5)" }, 1016 }; 1017 int32_t i, step; 1018 1019 ok=TRUE; 1020 for(i=0; i<LENGTHOF(steps) && ok; ++i) { 1021 step=steps[i].step; 1022 if(step<0 && !cc.finalFlush) { 1023 // skip ucnv_getNextUChar() if !finalFlush because 1024 // ucnv_getNextUChar() always implies flush 1025 continue; 1026 } 1027 if(step!=0) { 1028 // bulk test is first, then offsets are not checked any more 1029 cc.offsets=NULL; 1030 } 1031 else { 1032 memset(resultOffsets, -1, LENGTHOF(resultOffsets)); 1033 } 1034 memset(result, -1, LENGTHOF(result)); 1035 errorCode.reset(); 1036 resultLength=stepToUnicode(cc, cnv.getAlias(), 1037 result, LENGTHOF(result), 1038 step==0 ? resultOffsets : NULL, 1039 step, errorCode); 1040 ok=checkToUnicode( 1041 cc, cnv.getAlias(), steps[i].name, 1042 result, resultLength, 1043 cc.offsets!=NULL ? resultOffsets : NULL, 1044 errorCode); 1045 if(errorCode.isFailure() || !cc.finalFlush) { 1046 // reset if an error occurred or we did not flush 1047 // otherwise do nothing to make sure that flushing resets 1048 ucnv_resetToUnicode(cnv.getAlias()); 1049 } 1050 if (cc.offsets != NULL && resultOffsets[resultLength] != -1) { 1051 errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d", 1052 cc.caseNr, cc.charset, resultLength); 1053 } 1054 if (result[resultLength] != (UChar)-1) { 1055 errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d", 1056 cc.caseNr, cc.charset, resultLength); 1057 } 1058 } 1059 1060 // not a real loop, just a convenience for breaking out of the block 1061 while(ok && cc.finalFlush) { 1062 // test ucnv_toUChars() 1063 memset(result, 0, sizeof(result)); 1064 1065 errorCode.reset(); 1066 resultLength=ucnv_toUChars(cnv.getAlias(), 1067 result, LENGTHOF(result), 1068 (const char *)cc.bytes, cc.bytesLength, 1069 errorCode); 1070 ok=checkToUnicode( 1071 cc, cnv.getAlias(), "toUChars", 1072 result, resultLength, 1073 NULL, 1074 errorCode); 1075 if(!ok) { 1076 break; 1077 } 1078 1079 // test preflighting 1080 // keep the correct result for simple checking 1081 errorCode.reset(); 1082 resultLength=ucnv_toUChars(cnv.getAlias(), 1083 NULL, 0, 1084 (const char *)cc.bytes, cc.bytesLength, 1085 errorCode); 1086 if(errorCode.get()==U_STRING_NOT_TERMINATED_WARNING || errorCode.get()==U_BUFFER_OVERFLOW_ERROR) { 1087 errorCode.reset(); 1088 } 1089 ok=checkToUnicode( 1090 cc, cnv.getAlias(), "preflight toUChars", 1091 result, resultLength, 1092 NULL, 1093 errorCode); 1094 break; 1095 } 1096 1097 errorCode.reset(); // all errors have already been reported 1098 return ok; 1099 } 1100 1101 UBool 1102 ConversionTest::checkToUnicode(ConversionCase &cc, UConverter *cnv, const char *name, 1103 const UChar *result, int32_t resultLength, 1104 const int32_t *resultOffsets, 1105 UErrorCode resultErrorCode) { 1106 char resultInvalidChars[8]; 1107 int8_t resultInvalidLength; 1108 UErrorCode errorCode; 1109 1110 const char *msg; 1111 1112 // reset the message; NULL will mean "ok" 1113 msg=NULL; 1114 1115 errorCode=U_ZERO_ERROR; 1116 resultInvalidLength=sizeof(resultInvalidChars); 1117 ucnv_getInvalidChars(cnv, resultInvalidChars, &resultInvalidLength, &errorCode); 1118 if(U_FAILURE(errorCode)) { 1119 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidChars() failed - %s", 1120 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode)); 1121 return FALSE; 1122 } 1123 1124 // check everything that might have gone wrong 1125 if(cc.unicodeLength!=resultLength) { 1126 msg="wrong result length"; 1127 } else if(0!=u_memcmp(cc.unicode, result, cc.unicodeLength)) { 1128 msg="wrong result string"; 1129 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.unicodeLength*sizeof(*cc.offsets))) { 1130 msg="wrong offsets"; 1131 } else if(cc.outErrorCode!=resultErrorCode) { 1132 msg="wrong error code"; 1133 } else if(cc.invalidLength!=resultInvalidLength) { 1134 msg="wrong length of last invalid input"; 1135 } else if(0!=memcmp(cc.invalidChars, resultInvalidChars, cc.invalidLength)) { 1136 msg="wrong last invalid input"; 1137 } 1138 1139 if(msg==NULL) { 1140 return TRUE; 1141 } else { 1142 char buffer[2000]; // one buffer for all strings 1143 char *s, *bytesString, *unicodeString, *resultString, 1144 *offsetsString, *resultOffsetsString, 1145 *invalidCharsString, *resultInvalidCharsString; 1146 1147 bytesString=s=buffer; 1148 s=printBytes(cc.bytes, cc.bytesLength, bytesString); 1149 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString=s); 1150 s=printUnicode(result, resultLength, resultString=s); 1151 s=printOffsets(cc.offsets, cc.unicodeLength, offsetsString=s); 1152 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s); 1153 s=printBytes(cc.invalidChars, cc.invalidLength, invalidCharsString=s); 1154 s=printBytes((uint8_t *)resultInvalidChars, resultInvalidLength, resultInvalidCharsString=s); 1155 1156 if((s-buffer)>(int32_t)sizeof(buffer)) { 1157 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkToUnicode() test output buffer overflow writing %d chars\n", 1158 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer)); 1159 exit(1); 1160 } 1161 1162 errln("toUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n" 1163 " bytes <%s>[%d]\n" 1164 " expected <%s>[%d]\n" 1165 " result <%s>[%d]\n" 1166 " offsets <%s>\n" 1167 " result offsets <%s>\n" 1168 " error code expected %s got %s\n" 1169 " invalidChars expected <%s> got <%s>\n", 1170 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg, 1171 bytesString, cc.bytesLength, 1172 unicodeString, cc.unicodeLength, 1173 resultString, resultLength, 1174 offsetsString, 1175 resultOffsetsString, 1176 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode), 1177 invalidCharsString, resultInvalidCharsString); 1178 1179 return FALSE; 1180 } 1181 } 1182 1183 // fromUnicode test worker functions --------------------------------------- *** 1184 1185 static int32_t 1186 stepFromUTF8(ConversionCase &cc, 1187 UConverter *utf8Cnv, UConverter *cnv, 1188 char *result, int32_t resultCapacity, 1189 int32_t step, 1190 UErrorCode *pErrorCode) { 1191 const char *source, *sourceLimit, *utf8Limit; 1192 UChar pivotBuffer[32]; 1193 UChar *pivotSource, *pivotTarget, *pivotLimit; 1194 char *target, *targetLimit, *resultLimit; 1195 UBool flush; 1196 1197 source=cc.utf8; 1198 pivotSource=pivotTarget=pivotBuffer; 1199 target=result; 1200 utf8Limit=source+cc.utf8Length; 1201 resultLimit=result+resultCapacity; 1202 1203 // call ucnv_convertEx() with in/out buffers no larger than (step) at a time 1204 // move only one buffer (in vs. out) at a time to be extra mean 1205 // step==0 performs bulk conversion 1206 1207 // initialize the partial limits for the loop 1208 if(step==0) { 1209 // use the entire buffers 1210 sourceLimit=utf8Limit; 1211 targetLimit=resultLimit; 1212 flush=cc.finalFlush; 1213 1214 pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer); 1215 } else { 1216 // start with empty partial buffers 1217 sourceLimit=source; 1218 targetLimit=target; 1219 flush=FALSE; 1220 1221 // empty pivot is not allowed, make it of length step 1222 pivotLimit=pivotBuffer+step; 1223 } 1224 1225 for(;;) { 1226 // resetting the opposite conversion direction must not affect this one 1227 ucnv_resetFromUnicode(utf8Cnv); 1228 ucnv_resetToUnicode(cnv); 1229 1230 // convert 1231 ucnv_convertEx(cnv, utf8Cnv, 1232 &target, targetLimit, 1233 &source, sourceLimit, 1234 pivotBuffer, &pivotSource, &pivotTarget, pivotLimit, 1235 FALSE, flush, pErrorCode); 1236 1237 // check pointers and errors 1238 if(source>sourceLimit || target>targetLimit) { 1239 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 1240 break; 1241 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { 1242 if(target!=targetLimit) { 1243 // buffer overflow must only be set when the target is filled 1244 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 1245 break; 1246 } else if(targetLimit==resultLimit) { 1247 // not just a partial overflow 1248 break; 1249 } 1250 1251 // the partial target is filled, set a new limit, reset the error and continue 1252 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit; 1253 *pErrorCode=U_ZERO_ERROR; 1254 } else if(U_FAILURE(*pErrorCode)) { 1255 if(pivotSource==pivotBuffer) { 1256 // toUnicode error, should not occur 1257 // toUnicode errors are tested in cintltst TestConvertExFromUTF8() 1258 break; 1259 } else { 1260 // fromUnicode error 1261 // some other error occurred, done 1262 break; 1263 } 1264 } else { 1265 if(source!=sourceLimit) { 1266 // when no error occurs, then the input must be consumed 1267 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 1268 break; 1269 } 1270 1271 if(sourceLimit==utf8Limit) { 1272 // we are done 1273 if(*pErrorCode==U_STRING_NOT_TERMINATED_WARNING) { 1274 // ucnv_convertEx() warns about not terminating the output 1275 // but ucnv_fromUnicode() does not and so 1276 // checkFromUnicode() does not expect it 1277 *pErrorCode=U_ZERO_ERROR; 1278 } 1279 break; 1280 } 1281 1282 // the partial conversion succeeded, set a new limit and continue 1283 sourceLimit=(utf8Limit-source)>=step ? source+step : utf8Limit; 1284 flush=(UBool)(cc.finalFlush && sourceLimit==utf8Limit); 1285 } 1286 } 1287 1288 return (int32_t)(target-result); 1289 } 1290 1291 static int32_t 1292 stepFromUnicode(ConversionCase &cc, UConverter *cnv, 1293 char *result, int32_t resultCapacity, 1294 int32_t *resultOffsets, /* also resultCapacity */ 1295 int32_t step, 1296 UErrorCode *pErrorCode) { 1297 const UChar *source, *sourceLimit, *unicodeLimit; 1298 char *target, *targetLimit, *resultLimit; 1299 UBool flush; 1300 1301 source=cc.unicode; 1302 target=result; 1303 unicodeLimit=source+cc.unicodeLength; 1304 resultLimit=result+resultCapacity; 1305 1306 // call ucnv_fromUnicode() with in/out buffers no larger than (step) at a time 1307 // move only one buffer (in vs. out) at a time to be extra mean 1308 // step==0 performs bulk conversion and generates offsets 1309 1310 // initialize the partial limits for the loop 1311 if(step==0) { 1312 // use the entire buffers 1313 sourceLimit=unicodeLimit; 1314 targetLimit=resultLimit; 1315 flush=cc.finalFlush; 1316 } else { 1317 // start with empty partial buffers 1318 sourceLimit=source; 1319 targetLimit=target; 1320 flush=FALSE; 1321 1322 // output offsets only for bulk conversion 1323 resultOffsets=NULL; 1324 } 1325 1326 for(;;) { 1327 // resetting the opposite conversion direction must not affect this one 1328 ucnv_resetToUnicode(cnv); 1329 1330 // convert 1331 ucnv_fromUnicode(cnv, 1332 &target, targetLimit, 1333 &source, sourceLimit, 1334 resultOffsets, 1335 flush, pErrorCode); 1336 1337 // check pointers and errors 1338 if(source>sourceLimit || target>targetLimit) { 1339 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 1340 break; 1341 } else if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { 1342 if(target!=targetLimit) { 1343 // buffer overflow must only be set when the target is filled 1344 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 1345 break; 1346 } else if(targetLimit==resultLimit) { 1347 // not just a partial overflow 1348 break; 1349 } 1350 1351 // the partial target is filled, set a new limit, reset the error and continue 1352 targetLimit=(resultLimit-target)>=step ? target+step : resultLimit; 1353 *pErrorCode=U_ZERO_ERROR; 1354 } else if(U_FAILURE(*pErrorCode)) { 1355 // some other error occurred, done 1356 break; 1357 } else { 1358 if(source!=sourceLimit) { 1359 // when no error occurs, then the input must be consumed 1360 *pErrorCode=U_INTERNAL_PROGRAM_ERROR; 1361 break; 1362 } 1363 1364 if(sourceLimit==unicodeLimit) { 1365 // we are done 1366 break; 1367 } 1368 1369 // the partial conversion succeeded, set a new limit and continue 1370 sourceLimit=(unicodeLimit-source)>=step ? source+step : unicodeLimit; 1371 flush=(UBool)(cc.finalFlush && sourceLimit==unicodeLimit); 1372 } 1373 } 1374 1375 return (int32_t)(target-result); 1376 } 1377 1378 UBool 1379 ConversionTest::FromUnicodeCase(ConversionCase &cc, UConverterFromUCallback callback, const char *option) { 1380 UConverter *cnv; 1381 UErrorCode errorCode; 1382 1383 // open the converter 1384 errorCode=U_ZERO_ERROR; 1385 cnv=cnv_open(cc.charset, errorCode); 1386 if(U_FAILURE(errorCode)) { 1387 errcheckln(errorCode, "fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_open() failed - %s", 1388 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode)); 1389 return FALSE; 1390 } 1391 ucnv_resetToUnicode(utf8Cnv); 1392 1393 // set the callback 1394 if(callback!=NULL) { 1395 ucnv_setFromUCallBack(cnv, callback, option, NULL, NULL, &errorCode); 1396 if(U_FAILURE(errorCode)) { 1397 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setFromUCallBack() failed - %s", 1398 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode)); 1399 ucnv_close(cnv); 1400 return FALSE; 1401 } 1402 } 1403 1404 // set the fallbacks flag 1405 // TODO change with Jitterbug 2401, then add a similar call for toUnicode too 1406 ucnv_setFallback(cnv, cc.fallbacks); 1407 1408 // set the subchar 1409 int32_t length; 1410 1411 if(cc.setSub>0) { 1412 length=(int32_t)strlen(cc.subchar); 1413 ucnv_setSubstChars(cnv, cc.subchar, (int8_t)length, &errorCode); 1414 if(U_FAILURE(errorCode)) { 1415 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstChars() failed - %s", 1416 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode)); 1417 ucnv_close(cnv); 1418 return FALSE; 1419 } 1420 } else if(cc.setSub<0) { 1421 ucnv_setSubstString(cnv, cc.subString, -1, &errorCode); 1422 if(U_FAILURE(errorCode)) { 1423 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s", 1424 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode)); 1425 ucnv_close(cnv); 1426 return FALSE; 1427 } 1428 } 1429 1430 // convert unicode to utf8 1431 char utf8[256]; 1432 cc.utf8=utf8; 1433 u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length, 1434 cc.unicode, cc.unicodeLength, 1435 &errorCode); 1436 if(U_FAILURE(errorCode)) { 1437 // skip UTF-8 testing of a string with an unpaired surrogate, 1438 // or of one that's too long 1439 // toUnicode errors are tested in cintltst TestConvertExFromUTF8() 1440 cc.utf8Length=-1; 1441 } 1442 1443 int32_t resultOffsets[256]; 1444 char result[256]; 1445 int32_t resultLength; 1446 UBool ok; 1447 1448 static const struct { 1449 int32_t step; 1450 const char *name, *utf8Name; 1451 } steps[]={ 1452 { 0, "bulk", "utf8" }, // must be first for offsets to be checked 1453 { 1, "step=1", "utf8 step=1" }, 1454 { 3, "step=3", "utf8 step=3" }, 1455 { 7, "step=7", "utf8 step=7" } 1456 }; 1457 int32_t i, step; 1458 1459 ok=TRUE; 1460 for(i=0; i<LENGTHOF(steps) && ok; ++i) { 1461 step=steps[i].step; 1462 memset(resultOffsets, -1, LENGTHOF(resultOffsets)); 1463 memset(result, -1, LENGTHOF(result)); 1464 errorCode=U_ZERO_ERROR; 1465 resultLength=stepFromUnicode(cc, cnv, 1466 result, LENGTHOF(result), 1467 step==0 ? resultOffsets : NULL, 1468 step, &errorCode); 1469 ok=checkFromUnicode( 1470 cc, cnv, steps[i].name, 1471 (uint8_t *)result, resultLength, 1472 cc.offsets!=NULL ? resultOffsets : NULL, 1473 errorCode); 1474 if(U_FAILURE(errorCode) || !cc.finalFlush) { 1475 // reset if an error occurred or we did not flush 1476 // otherwise do nothing to make sure that flushing resets 1477 ucnv_resetFromUnicode(cnv); 1478 } 1479 if (resultOffsets[resultLength] != -1) { 1480 errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d", 1481 cc.caseNr, cc.charset, resultLength); 1482 } 1483 if (result[resultLength] != (char)-1) { 1484 errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d", 1485 cc.caseNr, cc.charset, resultLength); 1486 } 1487 1488 // bulk test is first, then offsets are not checked any more 1489 cc.offsets=NULL; 1490 1491 // test direct conversion from UTF-8 1492 if(cc.utf8Length>=0) { 1493 errorCode=U_ZERO_ERROR; 1494 resultLength=stepFromUTF8(cc, utf8Cnv, cnv, 1495 result, LENGTHOF(result), 1496 step, &errorCode); 1497 ok=checkFromUnicode( 1498 cc, cnv, steps[i].utf8Name, 1499 (uint8_t *)result, resultLength, 1500 NULL, 1501 errorCode); 1502 if(U_FAILURE(errorCode) || !cc.finalFlush) { 1503 // reset if an error occurred or we did not flush 1504 // otherwise do nothing to make sure that flushing resets 1505 ucnv_resetToUnicode(utf8Cnv); 1506 ucnv_resetFromUnicode(cnv); 1507 } 1508 } 1509 } 1510 1511 // not a real loop, just a convenience for breaking out of the block 1512 while(ok && cc.finalFlush) { 1513 // test ucnv_fromUChars() 1514 memset(result, 0, sizeof(result)); 1515 1516 errorCode=U_ZERO_ERROR; 1517 resultLength=ucnv_fromUChars(cnv, 1518 result, LENGTHOF(result), 1519 cc.unicode, cc.unicodeLength, 1520 &errorCode); 1521 ok=checkFromUnicode( 1522 cc, cnv, "fromUChars", 1523 (uint8_t *)result, resultLength, 1524 NULL, 1525 errorCode); 1526 if(!ok) { 1527 break; 1528 } 1529 1530 // test preflighting 1531 // keep the correct result for simple checking 1532 errorCode=U_ZERO_ERROR; 1533 resultLength=ucnv_fromUChars(cnv, 1534 NULL, 0, 1535 cc.unicode, cc.unicodeLength, 1536 &errorCode); 1537 if(errorCode==U_STRING_NOT_TERMINATED_WARNING || errorCode==U_BUFFER_OVERFLOW_ERROR) { 1538 errorCode=U_ZERO_ERROR; 1539 } 1540 ok=checkFromUnicode( 1541 cc, cnv, "preflight fromUChars", 1542 (uint8_t *)result, resultLength, 1543 NULL, 1544 errorCode); 1545 break; 1546 } 1547 1548 ucnv_close(cnv); 1549 return ok; 1550 } 1551 1552 UBool 1553 ConversionTest::checkFromUnicode(ConversionCase &cc, UConverter *cnv, const char *name, 1554 const uint8_t *result, int32_t resultLength, 1555 const int32_t *resultOffsets, 1556 UErrorCode resultErrorCode) { 1557 UChar resultInvalidUChars[8]; 1558 int8_t resultInvalidLength; 1559 UErrorCode errorCode; 1560 1561 const char *msg; 1562 1563 // reset the message; NULL will mean "ok" 1564 msg=NULL; 1565 1566 errorCode=U_ZERO_ERROR; 1567 resultInvalidLength=LENGTHOF(resultInvalidUChars); 1568 ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode); 1569 if(U_FAILURE(errorCode)) { 1570 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s", 1571 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode)); 1572 return FALSE; 1573 } 1574 1575 // check everything that might have gone wrong 1576 if(cc.bytesLength!=resultLength) { 1577 msg="wrong result length"; 1578 } else if(0!=memcmp(cc.bytes, result, cc.bytesLength)) { 1579 msg="wrong result string"; 1580 } else if(cc.offsets!=NULL && 0!=memcmp(cc.offsets, resultOffsets, cc.bytesLength*sizeof(*cc.offsets))) { 1581 msg="wrong offsets"; 1582 } else if(cc.outErrorCode!=resultErrorCode) { 1583 msg="wrong error code"; 1584 } else if(cc.invalidLength!=resultInvalidLength) { 1585 msg="wrong length of last invalid input"; 1586 } else if(0!=u_memcmp(cc.invalidUChars, resultInvalidUChars, cc.invalidLength)) { 1587 msg="wrong last invalid input"; 1588 } 1589 1590 if(msg==NULL) { 1591 return TRUE; 1592 } else { 1593 char buffer[2000]; // one buffer for all strings 1594 char *s, *unicodeString, *bytesString, *resultString, 1595 *offsetsString, *resultOffsetsString, 1596 *invalidCharsString, *resultInvalidUCharsString; 1597 1598 unicodeString=s=buffer; 1599 s=printUnicode(cc.unicode, cc.unicodeLength, unicodeString); 1600 s=printBytes(cc.bytes, cc.bytesLength, bytesString=s); 1601 s=printBytes(result, resultLength, resultString=s); 1602 s=printOffsets(cc.offsets, cc.bytesLength, offsetsString=s); 1603 s=printOffsets(resultOffsets, resultLength, resultOffsetsString=s); 1604 s=printUnicode(cc.invalidUChars, cc.invalidLength, invalidCharsString=s); 1605 s=printUnicode(resultInvalidUChars, resultInvalidLength, resultInvalidUCharsString=s); 1606 1607 if((s-buffer)>(int32_t)sizeof(buffer)) { 1608 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) fatal error: checkFromUnicode() test output buffer overflow writing %d chars\n", 1609 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, (int)(s-buffer)); 1610 exit(1); 1611 } 1612 1613 errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) failed: %s\n" 1614 " unicode <%s>[%d]\n" 1615 " expected <%s>[%d]\n" 1616 " result <%s>[%d]\n" 1617 " offsets <%s>\n" 1618 " result offsets <%s>\n" 1619 " error code expected %s got %s\n" 1620 " invalidChars expected <%s> got <%s>\n", 1621 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, msg, 1622 unicodeString, cc.unicodeLength, 1623 bytesString, cc.bytesLength, 1624 resultString, resultLength, 1625 offsetsString, 1626 resultOffsetsString, 1627 u_errorName(cc.outErrorCode), u_errorName(resultErrorCode), 1628 invalidCharsString, resultInvalidUCharsString); 1629 1630 return FALSE; 1631 } 1632 } 1633 1634 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ 1635
