1 // 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * Copyright (C) 2010-2014, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ******************************************************************************* 8 * file name: uts46test.cpp 9 * encoding: UTF-8 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2010may05 14 * created by: Markus W. Scherer 15 */ 16 17 #include "unicode/utypes.h" 18 19 #if !UCONFIG_NO_IDNA 20 21 #include <string.h> 22 #include "unicode/bytestream.h" 23 #include "unicode/idna.h" 24 #include "unicode/localpointer.h" 25 #include "unicode/std_string.h" 26 #include "unicode/stringpiece.h" 27 #include "unicode/uidna.h" 28 #include "unicode/unistr.h" 29 #include "charstr.h" 30 #include "cmemory.h" 31 #include "intltest.h" 32 #include "uparse.h" 33 34 class UTS46Test : public IntlTest { 35 public: 36 UTS46Test() : trans(NULL), nontrans(NULL) {} 37 virtual ~UTS46Test(); 38 39 void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=NULL); 40 void TestAPI(); 41 void TestNotSTD3(); 42 void TestSomeCases(); 43 void IdnaTest(); 44 45 void checkIdnaTestResult(const char *line, const char *type, 46 const UnicodeString &expected, const UnicodeString &result, 47 const char *status, const IDNAInfo &info); 48 void idnaTestOneLine(char *fields[][2], UErrorCode &errorCode); 49 50 private: 51 IDNA *trans, *nontrans; 52 }; 53 54 extern IntlTest *createUTS46Test() { 55 return new UTS46Test(); 56 } 57 58 UTS46Test::~UTS46Test() { 59 delete trans; 60 delete nontrans; 61 } 62 63 void UTS46Test::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) { 64 if(exec) { 65 logln("TestSuite UTS46Test: "); 66 if(trans==NULL) { 67 IcuTestErrorCode errorCode(*this, "init/createUTS46Instance()"); 68 uint32_t commonOptions= 69 UIDNA_USE_STD3_RULES|UIDNA_CHECK_BIDI| 70 UIDNA_CHECK_CONTEXTJ|UIDNA_CHECK_CONTEXTO; 71 trans=IDNA::createUTS46Instance(commonOptions, errorCode); 72 nontrans=IDNA::createUTS46Instance( 73 commonOptions| 74 UIDNA_NONTRANSITIONAL_TO_ASCII|UIDNA_NONTRANSITIONAL_TO_UNICODE, 75 errorCode); 76 if(errorCode.errDataIfFailureAndReset("createUTS46Instance()")) { 77 name=""; 78 return; 79 } 80 } 81 } 82 TESTCASE_AUTO_BEGIN; 83 TESTCASE_AUTO(TestAPI); 84 TESTCASE_AUTO(TestNotSTD3); 85 TESTCASE_AUTO(TestSomeCases); 86 TESTCASE_AUTO(IdnaTest); 87 TESTCASE_AUTO_END; 88 } 89 90 const uint32_t severeErrors= 91 UIDNA_ERROR_LEADING_COMBINING_MARK| 92 UIDNA_ERROR_DISALLOWED| 93 UIDNA_ERROR_PUNYCODE| 94 UIDNA_ERROR_LABEL_HAS_DOT| 95 UIDNA_ERROR_INVALID_ACE_LABEL; 96 97 static UBool isASCII(const UnicodeString &str) { 98 const UChar *s=str.getBuffer(); 99 int32_t length=str.length(); 100 for(int32_t i=0; i<length; ++i) { 101 if(s[i]>=0x80) { 102 return FALSE; 103 } 104 } 105 return TRUE; 106 } 107 108 class TestCheckedArrayByteSink : public CheckedArrayByteSink { 109 public: 110 TestCheckedArrayByteSink(char* outbuf, int32_t capacity) 111 : CheckedArrayByteSink(outbuf, capacity), calledFlush(FALSE) {} 112 virtual CheckedArrayByteSink& Reset() { 113 CheckedArrayByteSink::Reset(); 114 calledFlush = FALSE; 115 return *this; 116 } 117 virtual void Flush() { calledFlush = TRUE; } 118 UBool calledFlush; 119 }; 120 121 void UTS46Test::TestAPI() { 122 UErrorCode errorCode=U_ZERO_ERROR; 123 UnicodeString result; 124 IDNAInfo info; 125 UnicodeString input=UNICODE_STRING_SIMPLE("www.eXample.cOm"); 126 UnicodeString expected=UNICODE_STRING_SIMPLE("www.example.com"); 127 trans->nameToASCII(input, result, info, errorCode); 128 if(U_FAILURE(errorCode) || info.hasErrors() || result!=expected) { 129 errln("T.nameToASCII(www.example.com) info.errors=%04lx result matches=%d %s", 130 (long)info.getErrors(), result==expected, u_errorName(errorCode)); 131 } 132 errorCode=U_USELESS_COLLATOR_ERROR; 133 trans->nameToUnicode(input, result, info, errorCode); 134 if(errorCode!=U_USELESS_COLLATOR_ERROR || !result.isBogus()) { 135 errln("T.nameToUnicode(U_FAILURE) did not preserve the errorCode " 136 "or not result.setToBogus() - %s", 137 u_errorName(errorCode)); 138 } 139 errorCode=U_ZERO_ERROR; 140 input.setToBogus(); 141 result=UNICODE_STRING_SIMPLE("quatsch"); 142 nontrans->labelToASCII(input, result, info, errorCode); 143 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || !result.isBogus()) { 144 errln("N.labelToASCII(bogus) did not set illegal-argument-error " 145 "or not result.setToBogus() - %s", 146 u_errorName(errorCode)); 147 } 148 errorCode=U_ZERO_ERROR; 149 input=UNICODE_STRING_SIMPLE("xn--bcher.de-65a"); 150 expected=UNICODE_STRING_SIMPLE("xn--bcher\\uFFFDde-65a").unescape(); 151 nontrans->labelToASCII(input, result, info, errorCode); 152 if( U_FAILURE(errorCode) || 153 info.getErrors()!=(UIDNA_ERROR_LABEL_HAS_DOT|UIDNA_ERROR_INVALID_ACE_LABEL) || 154 result!=expected 155 ) { 156 errln("N.labelToASCII(label-with-dot) failed with errors %04lx - %s", 157 info.getErrors(), u_errorName(errorCode)); 158 } 159 // UTF-8 160 char buffer[100]; 161 TestCheckedArrayByteSink sink(buffer, UPRV_LENGTHOF(buffer)); 162 errorCode=U_ZERO_ERROR; 163 nontrans->labelToUnicodeUTF8(StringPiece(NULL, 5), sink, info, errorCode); 164 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || sink.NumberOfBytesWritten()!=0) { 165 errln("N.labelToUnicodeUTF8(StringPiece(NULL, 5)) did not set illegal-argument-error ", 166 "or did output something - %s", 167 u_errorName(errorCode)); 168 } 169 170 sink.Reset(); 171 errorCode=U_ZERO_ERROR; 172 nontrans->nameToASCII_UTF8(StringPiece(), sink, info, errorCode); 173 if(U_FAILURE(errorCode) || sink.NumberOfBytesWritten()!=0 || !sink.calledFlush) { 174 errln("N.nameToASCII_UTF8(empty) failed - %s", 175 u_errorName(errorCode)); 176 } 177 178 static const char s[]={ 0x61, (char)0xc3, (char)0x9f }; 179 sink.Reset(); 180 errorCode=U_USELESS_COLLATOR_ERROR; 181 nontrans->nameToUnicodeUTF8(StringPiece(s, 3), sink, info, errorCode); 182 if(errorCode!=U_USELESS_COLLATOR_ERROR || sink.NumberOfBytesWritten()!=0) { 183 errln("N.nameToUnicode_UTF8(U_FAILURE) did not preserve the errorCode " 184 "or did output something - %s", 185 u_errorName(errorCode)); 186 } 187 188 sink.Reset(); 189 errorCode=U_ZERO_ERROR; 190 trans->labelToUnicodeUTF8(StringPiece(s, 3), sink, info, errorCode); 191 if( U_FAILURE(errorCode) || sink.NumberOfBytesWritten()!=3 || 192 buffer[0]!=0x61 || buffer[1]!=0x73 || buffer[2]!=0x73 || 193 !sink.calledFlush 194 ) { 195 errln("T.labelToUnicodeUTF8(a sharp-s) failed - %s", 196 u_errorName(errorCode)); 197 } 198 199 sink.Reset(); 200 errorCode=U_ZERO_ERROR; 201 // "eXampLe.cOm" 202 static const char eX[]={ 0x65, 0x58, 0x61, 0x6d, 0x70, 0x4c, 0x65, 0x2e, 0x63, 0x4f, 0x6d, 0 }; 203 // "example.com" 204 static const char ex[]={ 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d }; 205 trans->nameToUnicodeUTF8(eX, sink, info, errorCode); 206 if( U_FAILURE(errorCode) || sink.NumberOfBytesWritten()!=11 || 207 0!=memcmp(ex, buffer, 11) || !sink.calledFlush 208 ) { 209 errln("T.nameToUnicodeUTF8(eXampLe.cOm) failed - %s", 210 u_errorName(errorCode)); 211 } 212 } 213 214 void UTS46Test::TestNotSTD3() { 215 IcuTestErrorCode errorCode(*this, "TestNotSTD3()"); 216 char buffer[400]; 217 LocalPointer<IDNA> not3(IDNA::createUTS46Instance(UIDNA_CHECK_BIDI, errorCode)); 218 if(errorCode.isFailure()) { 219 return; 220 } 221 UnicodeString input=UNICODE_STRING_SIMPLE("\\u0000A_2+2=4\\u000A.e\\u00DFen.net").unescape(); 222 UnicodeString result; 223 IDNAInfo info; 224 if( not3->nameToUnicode(input, result, info, errorCode)!= 225 UNICODE_STRING_SIMPLE("\\u0000a_2+2=4\\u000A.essen.net").unescape() || 226 info.hasErrors() 227 ) { 228 prettify(result).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 229 errln("notSTD3.nameToUnicode(non-LDH ASCII) unexpected errors %04lx string %s", 230 (long)info.getErrors(), buffer); 231 } 232 // A space (BiDi class WS) is not allowed in a BiDi domain name. 233 input=UNICODE_STRING_SIMPLE("a z.xn--4db.edu"); 234 not3->nameToASCII(input, result, info, errorCode); 235 if(result!=input || info.getErrors()!=UIDNA_ERROR_BIDI) { 236 errln("notSTD3.nameToASCII(ASCII-with-space.alef.edu) failed"); 237 } 238 // Characters that are canonically equivalent to sequences with non-LDH ASCII. 239 input=UNICODE_STRING_SIMPLE("a\\u2260b\\u226Ec\\u226Fd").unescape(); 240 not3->nameToUnicode(input, result, info, errorCode); 241 if(result!=input || info.hasErrors()) { 242 prettify(result).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 243 errln("notSTD3.nameToUnicode(equiv to non-LDH ASCII) unexpected errors %04lx string %s", 244 (long)info.getErrors(), buffer); 245 } 246 } 247 248 struct TestCase { 249 // Input string and options string (Nontransitional/Transitional/Both). 250 const char *s, *o; 251 // Expected Unicode result string. 252 const char *u; 253 uint32_t errors; 254 }; 255 256 static const TestCase testCases[]={ 257 { "www.eXample.cOm", "B", // all ASCII 258 "www.example.com", 0 }, 259 { "B\\u00FCcher.de", "B", // u-umlaut 260 "b\\u00FCcher.de", 0 }, 261 { "\\u00D6BB", "B", // O-umlaut 262 "\\u00F6bb", 0 }, 263 { "fa\\u00DF.de", "N", // sharp s 264 "fa\\u00DF.de", 0 }, 265 { "fa\\u00DF.de", "T", // sharp s 266 "fass.de", 0 }, 267 { "XN--fA-hia.dE", "B", // sharp s in Punycode 268 "fa\\u00DF.de", 0 }, 269 { "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2.com", "N", // Greek with final sigma 270 "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2.com", 0 }, 271 { "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2.com", "T", // Greek with final sigma 272 "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C3.com", 0 }, 273 { "xn--nxasmm1c", "B", // Greek with final sigma in Punycode 274 "\\u03B2\\u03CC\\u03BB\\u03BF\\u03C2", 0 }, 275 { "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", "N", // "Sri" in "Sri Lanka" has a ZWJ 276 "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", 0 }, 277 { "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", "T", // "Sri" in "Sri Lanka" has a ZWJ 278 "www.\\u0DC1\\u0DCA\\u0DBB\\u0DD3.com", 0 }, 279 { "www.xn--10cl1a0b660p.com", "B", // "Sri" in Punycode 280 "www.\\u0DC1\\u0DCA\\u200D\\u0DBB\\u0DD3.com", 0 }, 281 { "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC", "N", // ZWNJ 282 "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC", 0 }, 283 { "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC", "T", // ZWNJ 284 "\\u0646\\u0627\\u0645\\u0647\\u0627\\u06CC", 0 }, 285 { "xn--mgba3gch31f060k.com", "B", // ZWNJ in Punycode 286 "\\u0646\\u0627\\u0645\\u0647\\u200C\\u0627\\u06CC.com", 0 }, 287 { "a.b\\uFF0Ec\\u3002d\\uFF61", "B", 288 "a.b.c.d.", 0 }, 289 { "U\\u0308.xn--tda", "B", // U+umlaut.u-umlaut 290 "\\u00FC.\\u00FC", 0 }, 291 { "xn--u-ccb", "B", // u+umlaut in Punycode 292 "xn--u-ccb\\uFFFD", UIDNA_ERROR_INVALID_ACE_LABEL }, 293 { "a\\u2488com", "B", // contains 1-dot 294 "a\\uFFFDcom", UIDNA_ERROR_DISALLOWED }, 295 { "xn--a-ecp.ru", "B", // contains 1-dot in Punycode 296 "xn--a-ecp\\uFFFD.ru", UIDNA_ERROR_INVALID_ACE_LABEL }, 297 { "xn--0.pt", "B", // invalid Punycode 298 "xn--0\\uFFFD.pt", UIDNA_ERROR_PUNYCODE }, 299 { "xn--a.pt", "B", // U+0080 300 "xn--a\\uFFFD.pt", UIDNA_ERROR_INVALID_ACE_LABEL }, 301 { "xn--a-\\u00C4.pt", "B", // invalid Punycode 302 "xn--a-\\u00E4.pt", UIDNA_ERROR_PUNYCODE }, 303 { "\\u65E5\\u672C\\u8A9E\\u3002\\uFF2A\\uFF30", "B", // Japanese with fullwidth ".jp" 304 "\\u65E5\\u672C\\u8A9E.jp", 0 }, 305 { "\\u2615", "B", "\\u2615", 0 }, // Unicode 4.0 HOT BEVERAGE 306 // some characters are disallowed because they are canonically equivalent 307 // to sequences with non-LDH ASCII 308 { "a\\u2260b\\u226Ec\\u226Fd", "B", 309 "a\\uFFFDb\\uFFFDc\\uFFFDd", UIDNA_ERROR_DISALLOWED }, 310 // many deviation characters, test the special mapping code 311 { "1.a\\u00DF\\u200C\\u200Db\\u200C\\u200Dc\\u00DF\\u00DF\\u00DF\\u00DFd" 312 "\\u03C2\\u03C3\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFe" 313 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFx" 314 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFy" 315 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u0302\\u00DFz", "N", 316 "1.a\\u00DF\\u200C\\u200Db\\u200C\\u200Dc\\u00DF\\u00DF\\u00DF\\u00DFd" 317 "\\u03C2\\u03C3\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFe" 318 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFx" 319 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFy" 320 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u0302\\u00DFz", 321 UIDNA_ERROR_LABEL_TOO_LONG|UIDNA_ERROR_CONTEXTJ }, 322 { "1.a\\u00DF\\u200C\\u200Db\\u200C\\u200Dc\\u00DF\\u00DF\\u00DF\\u00DFd" 323 "\\u03C2\\u03C3\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFe" 324 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFx" 325 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DFy" 326 "\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u00DF\\u0302\\u00DFz", "T", 327 "1.assbcssssssssd" 328 "\\u03C3\\u03C3sssssssssssssssse" 329 "ssssssssssssssssssssx" 330 "ssssssssssssssssssssy" 331 "sssssssssssssss\\u015Dssz", UIDNA_ERROR_LABEL_TOO_LONG }, 332 // "xn--bss" with deviation characters 333 { "\\u200Cx\\u200Dn\\u200C-\\u200D-b\\u00DF", "N", 334 "\\u200Cx\\u200Dn\\u200C-\\u200D-b\\u00DF", UIDNA_ERROR_CONTEXTJ }, 335 { "\\u200Cx\\u200Dn\\u200C-\\u200D-b\\u00DF", "T", 336 "\\u5919", 0 }, 337 // "xn--bssffl" written as: 338 // 02E3 MODIFIER LETTER SMALL X 339 // 034F COMBINING GRAPHEME JOINER (ignored) 340 // 2115 DOUBLE-STRUCK CAPITAL N 341 // 200B ZERO WIDTH SPACE (ignored) 342 // FE63 SMALL HYPHEN-MINUS 343 // 00AD SOFT HYPHEN (ignored) 344 // FF0D FULLWIDTH HYPHEN-MINUS 345 // 180C MONGOLIAN FREE VARIATION SELECTOR TWO (ignored) 346 // 212C SCRIPT CAPITAL B 347 // FE00 VARIATION SELECTOR-1 (ignored) 348 // 017F LATIN SMALL LETTER LONG S 349 // 2064 INVISIBLE PLUS (ignored) 350 // 1D530 MATHEMATICAL FRAKTUR SMALL S 351 // E01EF VARIATION SELECTOR-256 (ignored) 352 // FB04 LATIN SMALL LIGATURE FFL 353 { "\\u02E3\\u034F\\u2115\\u200B\\uFE63\\u00AD\\uFF0D\\u180C" 354 "\\u212C\\uFE00\\u017F\\u2064\\U0001D530\\U000E01EF\\uFB04", "B", 355 "\\u5921\\u591E\\u591C\\u5919", 0 }, 356 { "123456789012345678901234567890123456789012345678901234567890123." 357 "123456789012345678901234567890123456789012345678901234567890123." 358 "123456789012345678901234567890123456789012345678901234567890123." 359 "1234567890123456789012345678901234567890123456789012345678901", "B", 360 "123456789012345678901234567890123456789012345678901234567890123." 361 "123456789012345678901234567890123456789012345678901234567890123." 362 "123456789012345678901234567890123456789012345678901234567890123." 363 "1234567890123456789012345678901234567890123456789012345678901", 0 }, 364 { "123456789012345678901234567890123456789012345678901234567890123." 365 "123456789012345678901234567890123456789012345678901234567890123." 366 "123456789012345678901234567890123456789012345678901234567890123." 367 "1234567890123456789012345678901234567890123456789012345678901.", "B", 368 "123456789012345678901234567890123456789012345678901234567890123." 369 "123456789012345678901234567890123456789012345678901234567890123." 370 "123456789012345678901234567890123456789012345678901234567890123." 371 "1234567890123456789012345678901234567890123456789012345678901.", 0 }, 372 // Domain name >256 characters, forces slow path in UTF-8 processing. 373 { "123456789012345678901234567890123456789012345678901234567890123." 374 "123456789012345678901234567890123456789012345678901234567890123." 375 "123456789012345678901234567890123456789012345678901234567890123." 376 "123456789012345678901234567890123456789012345678901234567890123." 377 "12345678901234567890123456789012345678901234567890123456789012", "B", 378 "123456789012345678901234567890123456789012345678901234567890123." 379 "123456789012345678901234567890123456789012345678901234567890123." 380 "123456789012345678901234567890123456789012345678901234567890123." 381 "123456789012345678901234567890123456789012345678901234567890123." 382 "12345678901234567890123456789012345678901234567890123456789012", 383 UIDNA_ERROR_DOMAIN_NAME_TOO_LONG }, 384 { "123456789012345678901234567890123456789012345678901234567890123." 385 "123456789012345678901234567890123456789012345678901234567890123." 386 "123456789012345678901234567890123456789012345678901234567890123." 387 "123456789012345678901234567890123456789012345678901234567890123." 388 "1234567890123456789012345678901234567890123456789\\u05D0", "B", 389 "123456789012345678901234567890123456789012345678901234567890123." 390 "123456789012345678901234567890123456789012345678901234567890123." 391 "123456789012345678901234567890123456789012345678901234567890123." 392 "123456789012345678901234567890123456789012345678901234567890123." 393 "1234567890123456789012345678901234567890123456789\\u05D0", 394 UIDNA_ERROR_DOMAIN_NAME_TOO_LONG|UIDNA_ERROR_BIDI }, 395 { "123456789012345678901234567890123456789012345678901234567890123." 396 "1234567890123456789012345678901234567890123456789012345678901234." 397 "123456789012345678901234567890123456789012345678901234567890123." 398 "123456789012345678901234567890123456789012345678901234567890", "B", 399 "123456789012345678901234567890123456789012345678901234567890123." 400 "1234567890123456789012345678901234567890123456789012345678901234." 401 "123456789012345678901234567890123456789012345678901234567890123." 402 "123456789012345678901234567890123456789012345678901234567890", 403 UIDNA_ERROR_LABEL_TOO_LONG }, 404 { "123456789012345678901234567890123456789012345678901234567890123." 405 "1234567890123456789012345678901234567890123456789012345678901234." 406 "123456789012345678901234567890123456789012345678901234567890123." 407 "123456789012345678901234567890123456789012345678901234567890.", "B", 408 "123456789012345678901234567890123456789012345678901234567890123." 409 "1234567890123456789012345678901234567890123456789012345678901234." 410 "123456789012345678901234567890123456789012345678901234567890123." 411 "123456789012345678901234567890123456789012345678901234567890.", 412 UIDNA_ERROR_LABEL_TOO_LONG }, 413 { "123456789012345678901234567890123456789012345678901234567890123." 414 "1234567890123456789012345678901234567890123456789012345678901234." 415 "123456789012345678901234567890123456789012345678901234567890123." 416 "1234567890123456789012345678901234567890123456789012345678901", "B", 417 "123456789012345678901234567890123456789012345678901234567890123." 418 "1234567890123456789012345678901234567890123456789012345678901234." 419 "123456789012345678901234567890123456789012345678901234567890123." 420 "1234567890123456789012345678901234567890123456789012345678901", 421 UIDNA_ERROR_LABEL_TOO_LONG|UIDNA_ERROR_DOMAIN_NAME_TOO_LONG }, 422 // label length 63: xn--1234567890123456789012345678901234567890123456789012345-9te 423 { "\\u00E41234567890123456789012345678901234567890123456789012345", "B", 424 "\\u00E41234567890123456789012345678901234567890123456789012345", 0 }, 425 { "1234567890\\u00E41234567890123456789012345678901234567890123456", "B", 426 "1234567890\\u00E41234567890123456789012345678901234567890123456", UIDNA_ERROR_LABEL_TOO_LONG }, 427 { "123456789012345678901234567890123456789012345678901234567890123." 428 "1234567890\\u00E4123456789012345678901234567890123456789012345." 429 "123456789012345678901234567890123456789012345678901234567890123." 430 "1234567890123456789012345678901234567890123456789012345678901", "B", 431 "123456789012345678901234567890123456789012345678901234567890123." 432 "1234567890\\u00E4123456789012345678901234567890123456789012345." 433 "123456789012345678901234567890123456789012345678901234567890123." 434 "1234567890123456789012345678901234567890123456789012345678901", 0 }, 435 { "123456789012345678901234567890123456789012345678901234567890123." 436 "1234567890\\u00E4123456789012345678901234567890123456789012345." 437 "123456789012345678901234567890123456789012345678901234567890123." 438 "1234567890123456789012345678901234567890123456789012345678901.", "B", 439 "123456789012345678901234567890123456789012345678901234567890123." 440 "1234567890\\u00E4123456789012345678901234567890123456789012345." 441 "123456789012345678901234567890123456789012345678901234567890123." 442 "1234567890123456789012345678901234567890123456789012345678901.", 0 }, 443 { "123456789012345678901234567890123456789012345678901234567890123." 444 "1234567890\\u00E4123456789012345678901234567890123456789012345." 445 "123456789012345678901234567890123456789012345678901234567890123." 446 "12345678901234567890123456789012345678901234567890123456789012", "B", 447 "123456789012345678901234567890123456789012345678901234567890123." 448 "1234567890\\u00E4123456789012345678901234567890123456789012345." 449 "123456789012345678901234567890123456789012345678901234567890123." 450 "12345678901234567890123456789012345678901234567890123456789012", 451 UIDNA_ERROR_DOMAIN_NAME_TOO_LONG }, 452 { "123456789012345678901234567890123456789012345678901234567890123." 453 "1234567890\\u00E41234567890123456789012345678901234567890123456." 454 "123456789012345678901234567890123456789012345678901234567890123." 455 "123456789012345678901234567890123456789012345678901234567890", "B", 456 "123456789012345678901234567890123456789012345678901234567890123." 457 "1234567890\\u00E41234567890123456789012345678901234567890123456." 458 "123456789012345678901234567890123456789012345678901234567890123." 459 "123456789012345678901234567890123456789012345678901234567890", 460 UIDNA_ERROR_LABEL_TOO_LONG }, 461 { "123456789012345678901234567890123456789012345678901234567890123." 462 "1234567890\\u00E41234567890123456789012345678901234567890123456." 463 "123456789012345678901234567890123456789012345678901234567890123." 464 "123456789012345678901234567890123456789012345678901234567890.", "B", 465 "123456789012345678901234567890123456789012345678901234567890123." 466 "1234567890\\u00E41234567890123456789012345678901234567890123456." 467 "123456789012345678901234567890123456789012345678901234567890123." 468 "123456789012345678901234567890123456789012345678901234567890.", 469 UIDNA_ERROR_LABEL_TOO_LONG }, 470 { "123456789012345678901234567890123456789012345678901234567890123." 471 "1234567890\\u00E41234567890123456789012345678901234567890123456." 472 "123456789012345678901234567890123456789012345678901234567890123." 473 "1234567890123456789012345678901234567890123456789012345678901", "B", 474 "123456789012345678901234567890123456789012345678901234567890123." 475 "1234567890\\u00E41234567890123456789012345678901234567890123456." 476 "123456789012345678901234567890123456789012345678901234567890123." 477 "1234567890123456789012345678901234567890123456789012345678901", 478 UIDNA_ERROR_LABEL_TOO_LONG|UIDNA_ERROR_DOMAIN_NAME_TOO_LONG }, 479 // hyphen errors and empty-label errors 480 // Ticket #10883: ToUnicode also checks for empty labels. 481 { ".", "B", ".", UIDNA_ERROR_EMPTY_LABEL }, 482 { "\\uFF0E", "B", ".", UIDNA_ERROR_EMPTY_LABEL }, 483 // "xn---q----jra"=="-q--a-umlaut-" 484 { "a.b..-q--a-.e", "B", "a.b..-q--a-.e", 485 UIDNA_ERROR_EMPTY_LABEL|UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN| 486 UIDNA_ERROR_HYPHEN_3_4 }, 487 { "a.b..-q--\\u00E4-.e", "B", "a.b..-q--\\u00E4-.e", 488 UIDNA_ERROR_EMPTY_LABEL|UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN| 489 UIDNA_ERROR_HYPHEN_3_4 }, 490 { "a.b..xn---q----jra.e", "B", "a.b..-q--\\u00E4-.e", 491 UIDNA_ERROR_EMPTY_LABEL|UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN| 492 UIDNA_ERROR_HYPHEN_3_4 }, 493 { "a..c", "B", "a..c", UIDNA_ERROR_EMPTY_LABEL }, 494 { "a.xn--.c", "B", "a..c", UIDNA_ERROR_EMPTY_LABEL }, 495 { "a.-b.", "B", "a.-b.", UIDNA_ERROR_LEADING_HYPHEN }, 496 { "a.b-.c", "B", "a.b-.c", UIDNA_ERROR_TRAILING_HYPHEN }, 497 { "a.-.c", "B", "a.-.c", UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN }, 498 { "a.bc--de.f", "B", "a.bc--de.f", UIDNA_ERROR_HYPHEN_3_4 }, 499 { "\\u00E4.\\u00AD.c", "B", "\\u00E4..c", UIDNA_ERROR_EMPTY_LABEL }, 500 { "\\u00E4.xn--.c", "B", "\\u00E4..c", UIDNA_ERROR_EMPTY_LABEL }, 501 { "\\u00E4.-b.", "B", "\\u00E4.-b.", UIDNA_ERROR_LEADING_HYPHEN }, 502 { "\\u00E4.b-.c", "B", "\\u00E4.b-.c", UIDNA_ERROR_TRAILING_HYPHEN }, 503 { "\\u00E4.-.c", "B", "\\u00E4.-.c", UIDNA_ERROR_LEADING_HYPHEN|UIDNA_ERROR_TRAILING_HYPHEN }, 504 { "\\u00E4.bc--de.f", "B", "\\u00E4.bc--de.f", UIDNA_ERROR_HYPHEN_3_4 }, 505 { "a.b.\\u0308c.d", "B", "a.b.\\uFFFDc.d", UIDNA_ERROR_LEADING_COMBINING_MARK }, 506 { "a.b.xn--c-bcb.d", "B", 507 "a.b.xn--c-bcb\\uFFFD.d", UIDNA_ERROR_LEADING_COMBINING_MARK|UIDNA_ERROR_INVALID_ACE_LABEL }, 508 // BiDi 509 { "A0", "B", "a0", 0 }, 510 { "0A", "B", "0a", 0 }, // all-LTR is ok to start with a digit (EN) 511 { "0A.\\u05D0", "B", // ASCII label does not start with L/R/AL 512 "0a.\\u05D0", UIDNA_ERROR_BIDI }, 513 { "c.xn--0-eha.xn--4db", "B", // 2nd label does not start with L/R/AL 514 "c.0\\u00FC.\\u05D0", UIDNA_ERROR_BIDI }, 515 { "b-.\\u05D0", "B", // label does not end with L/EN 516 "b-.\\u05D0", UIDNA_ERROR_TRAILING_HYPHEN|UIDNA_ERROR_BIDI }, 517 { "d.xn----dha.xn--4db", "B", // 2nd label does not end with L/EN 518 "d.\\u00FC-.\\u05D0", UIDNA_ERROR_TRAILING_HYPHEN|UIDNA_ERROR_BIDI }, 519 { "a\\u05D0", "B", "a\\u05D0", UIDNA_ERROR_BIDI }, // first dir != last dir 520 { "\\u05D0\\u05C7", "B", "\\u05D0\\u05C7", 0 }, 521 { "\\u05D09\\u05C7", "B", "\\u05D09\\u05C7", 0 }, 522 { "\\u05D0a\\u05C7", "B", "\\u05D0a\\u05C7", UIDNA_ERROR_BIDI }, // first dir != last dir 523 { "\\u05D0\\u05EA", "B", "\\u05D0\\u05EA", 0 }, 524 { "\\u05D0\\u05F3\\u05EA", "B", "\\u05D0\\u05F3\\u05EA", 0 }, 525 { "a\\u05D0Tz", "B", "a\\u05D0tz", UIDNA_ERROR_BIDI }, // mixed dir 526 { "\\u05D0T\\u05EA", "B", "\\u05D0t\\u05EA", UIDNA_ERROR_BIDI }, // mixed dir 527 { "\\u05D07\\u05EA", "B", "\\u05D07\\u05EA", 0 }, 528 { "\\u05D0\\u0667\\u05EA", "B", "\\u05D0\\u0667\\u05EA", 0 }, // Arabic 7 in the middle 529 { "a7\\u0667z", "B", "a7\\u0667z", UIDNA_ERROR_BIDI }, // AN digit in LTR 530 { "a7\\u0667", "B", "a7\\u0667", UIDNA_ERROR_BIDI }, // AN digit in LTR 531 { "\\u05D07\\u0667\\u05EA", "B", // mixed EN/AN digits in RTL 532 "\\u05D07\\u0667\\u05EA", UIDNA_ERROR_BIDI }, 533 { "\\u05D07\\u0667", "B", // mixed EN/AN digits in RTL 534 "\\u05D07\\u0667", UIDNA_ERROR_BIDI }, 535 // ZWJ 536 { "\\u0BB9\\u0BCD\\u200D", "N", "\\u0BB9\\u0BCD\\u200D", 0 }, // Virama+ZWJ 537 { "\\u0BB9\\u200D", "N", "\\u0BB9\\u200D", UIDNA_ERROR_CONTEXTJ }, // no Virama 538 { "\\u200D", "N", "\\u200D", UIDNA_ERROR_CONTEXTJ }, // no Virama 539 // ZWNJ 540 { "\\u0BB9\\u0BCD\\u200C", "N", "\\u0BB9\\u0BCD\\u200C", 0 }, // Virama+ZWNJ 541 { "\\u0BB9\\u200C", "N", "\\u0BB9\\u200C", UIDNA_ERROR_CONTEXTJ }, // no Virama 542 { "\\u200C", "N", "\\u200C", UIDNA_ERROR_CONTEXTJ }, // no Virama 543 { "\\u0644\\u0670\\u200C\\u06ED\\u06EF", "N", // Joining types D T ZWNJ T R 544 "\\u0644\\u0670\\u200C\\u06ED\\u06EF", 0 }, 545 { "\\u0644\\u0670\\u200C\\u06EF", "N", // D T ZWNJ R 546 "\\u0644\\u0670\\u200C\\u06EF", 0 }, 547 { "\\u0644\\u200C\\u06ED\\u06EF", "N", // D ZWNJ T R 548 "\\u0644\\u200C\\u06ED\\u06EF", 0 }, 549 { "\\u0644\\u200C\\u06EF", "N", // D ZWNJ R 550 "\\u0644\\u200C\\u06EF", 0 }, 551 { "\\u0644\\u0670\\u200C\\u06ED", "N", // D T ZWNJ T 552 "\\u0644\\u0670\\u200C\\u06ED", UIDNA_ERROR_BIDI|UIDNA_ERROR_CONTEXTJ }, 553 { "\\u06EF\\u200C\\u06EF", "N", // R ZWNJ R 554 "\\u06EF\\u200C\\u06EF", UIDNA_ERROR_CONTEXTJ }, 555 { "\\u0644\\u200C", "N", // D ZWNJ 556 "\\u0644\\u200C", UIDNA_ERROR_BIDI|UIDNA_ERROR_CONTEXTJ }, 557 { "\\u0660\\u0661", "B", // Arabic-Indic Digits alone 558 "\\u0660\\u0661", UIDNA_ERROR_BIDI }, 559 { "\\u06F0\\u06F1", "B", // Extended Arabic-Indic Digits alone 560 "\\u06F0\\u06F1", 0 }, 561 { "\\u0660\\u06F1", "B", // Mixed Arabic-Indic Digits 562 "\\u0660\\u06F1", UIDNA_ERROR_CONTEXTO_DIGITS|UIDNA_ERROR_BIDI }, 563 // All of the CONTEXTO "Would otherwise have been DISALLOWED" characters 564 // in their correct contexts, 565 // then each in incorrect context. 566 { "l\\u00B7l\\u4E00\\u0375\\u03B1\\u05D0\\u05F3\\u05F4\\u30FB", "B", 567 "l\\u00B7l\\u4E00\\u0375\\u03B1\\u05D0\\u05F3\\u05F4\\u30FB", UIDNA_ERROR_BIDI }, 568 { "l\\u00B7", "B", 569 "l\\u00B7", UIDNA_ERROR_CONTEXTO_PUNCTUATION }, 570 { "\\u00B7l", "B", 571 "\\u00B7l", UIDNA_ERROR_CONTEXTO_PUNCTUATION }, 572 { "\\u0375", "B", 573 "\\u0375", UIDNA_ERROR_CONTEXTO_PUNCTUATION }, 574 { "\\u03B1\\u05F3", "B", 575 "\\u03B1\\u05F3", UIDNA_ERROR_CONTEXTO_PUNCTUATION|UIDNA_ERROR_BIDI }, 576 { "\\u05F4", "B", 577 "\\u05F4", UIDNA_ERROR_CONTEXTO_PUNCTUATION }, 578 { "l\\u30FB", "B", 579 "l\\u30FB", UIDNA_ERROR_CONTEXTO_PUNCTUATION }, 580 // Ticket #8137: UTS #46 toUnicode() fails with non-ASCII labels that turn 581 // into 15 characters (UChars). 582 // The bug was in u_strFromPunycode() which did not write the last character 583 // if it just so fit into the end of the destination buffer. 584 // The UTS #46 code gives a default-capacity UnicodeString as the destination buffer, 585 // and the internal UnicodeString capacity is currently 15 UChars on 64-bit machines 586 // but 13 on 32-bit machines. 587 // Label with 15 UChars, for 64-bit-machine testing: 588 { "aaaaaaaaaaaaa\\u00FCa.de", "B", "aaaaaaaaaaaaa\\u00FCa.de", 0 }, 589 { "xn--aaaaaaaaaaaaaa-ssb.de", "B", "aaaaaaaaaaaaa\\u00FCa.de", 0 }, 590 { "abschlu\\u00DFpr\\u00FCfung.de", "N", "abschlu\\u00DFpr\\u00FCfung.de", 0 }, 591 { "xn--abschluprfung-hdb15b.de", "B", "abschlu\\u00DFpr\\u00FCfung.de", 0 }, 592 // Label with 13 UChars, for 32-bit-machine testing: 593 { "xn--aaaaaaaaaaaa-nlb.de", "B", "aaaaaaaaaaa\\u00FCa.de", 0 }, 594 { "xn--schluprfung-z6a39a.de", "B", "schlu\\u00DFpr\\u00FCfung.de", 0 }, 595 // { "", "B", 596 // "", 0 }, 597 }; 598 599 void UTS46Test::TestSomeCases() { 600 IcuTestErrorCode errorCode(*this, "TestSomeCases"); 601 char buffer[400], buffer2[400]; 602 int32_t i; 603 for(i=0; i<UPRV_LENGTHOF(testCases); ++i) { 604 const TestCase &testCase=testCases[i]; 605 UnicodeString input(ctou(testCase.s)); 606 UnicodeString expected(ctou(testCase.u)); 607 // ToASCII/ToUnicode, transitional/nontransitional 608 UnicodeString aT, uT, aN, uN; 609 IDNAInfo aTInfo, uTInfo, aNInfo, uNInfo; 610 trans->nameToASCII(input, aT, aTInfo, errorCode); 611 trans->nameToUnicode(input, uT, uTInfo, errorCode); 612 nontrans->nameToASCII(input, aN, aNInfo, errorCode); 613 nontrans->nameToUnicode(input, uN, uNInfo, errorCode); 614 if(errorCode.errIfFailureAndReset("first-level processing [%d/%s] %s", 615 (int)i, testCase.o, testCase.s) 616 ) { 617 continue; 618 } 619 // ToUnicode does not set length-overflow errors. 620 uint32_t uniErrors=testCase.errors&~ 621 (UIDNA_ERROR_LABEL_TOO_LONG| 622 UIDNA_ERROR_DOMAIN_NAME_TOO_LONG); 623 char mode=testCase.o[0]; 624 if(mode=='B' || mode=='N') { 625 if(uNInfo.getErrors()!=uniErrors) { 626 errln("N.nameToUnicode([%d] %s) unexpected errors %04lx", 627 (int)i, testCase.s, (long)uNInfo.getErrors()); 628 continue; 629 } 630 if(uN!=expected) { 631 prettify(uN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 632 errln("N.nameToUnicode([%d] %s) unexpected string %s", 633 (int)i, testCase.s, buffer); 634 continue; 635 } 636 if(aNInfo.getErrors()!=testCase.errors) { 637 errln("N.nameToASCII([%d] %s) unexpected errors %04lx", 638 (int)i, testCase.s, (long)aNInfo.getErrors()); 639 continue; 640 } 641 } 642 if(mode=='B' || mode=='T') { 643 if(uTInfo.getErrors()!=uniErrors) { 644 errln("T.nameToUnicode([%d] %s) unexpected errors %04lx", 645 (int)i, testCase.s, (long)uTInfo.getErrors()); 646 continue; 647 } 648 if(uT!=expected) { 649 prettify(uT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 650 errln("T.nameToUnicode([%d] %s) unexpected string %s", 651 (int)i, testCase.s, buffer); 652 continue; 653 } 654 if(aTInfo.getErrors()!=testCase.errors) { 655 errln("T.nameToASCII([%d] %s) unexpected errors %04lx", 656 (int)i, testCase.s, (long)aTInfo.getErrors()); 657 continue; 658 } 659 } 660 // ToASCII is all-ASCII if no severe errors 661 if((aNInfo.getErrors()&severeErrors)==0 && !isASCII(aN)) { 662 prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 663 errln("N.nameToASCII([%d] %s) (errors %04lx) result is not ASCII %s", 664 (int)i, testCase.s, aNInfo.getErrors(), buffer); 665 continue; 666 } 667 if((aTInfo.getErrors()&severeErrors)==0 && !isASCII(aT)) { 668 prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 669 errln("T.nameToASCII([%d] %s) (errors %04lx) result is not ASCII %s", 670 (int)i, testCase.s, aTInfo.getErrors(), buffer); 671 continue; 672 } 673 if(verbose) { 674 char m= mode=='B' ? mode : 'N'; 675 prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 676 logln("%c.nameToASCII([%d] %s) (errors %04lx) result string: %s", 677 m, (int)i, testCase.s, aNInfo.getErrors(), buffer); 678 if(mode!='B') { 679 prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 680 logln("T.nameToASCII([%d] %s) (errors %04lx) result string: %s", 681 (int)i, testCase.s, aTInfo.getErrors(), buffer); 682 } 683 } 684 // second-level processing 685 UnicodeString aTuN, uTaN, aNuN, uNaN; 686 IDNAInfo aTuNInfo, uTaNInfo, aNuNInfo, uNaNInfo; 687 nontrans->nameToUnicode(aT, aTuN, aTuNInfo, errorCode); 688 nontrans->nameToASCII(uT, uTaN, uTaNInfo, errorCode); 689 nontrans->nameToUnicode(aN, aNuN, aNuNInfo, errorCode); 690 nontrans->nameToASCII(uN, uNaN, uNaNInfo, errorCode); 691 if(errorCode.errIfFailureAndReset("second-level processing [%d/%s] %s", 692 (int)i, testCase.o, testCase.s) 693 ) { 694 continue; 695 } 696 if(aN!=uNaN) { 697 prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 698 prettify(uNaN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 699 errln("N.nameToASCII([%d] %s)!=N.nameToUnicode().N.nameToASCII() " 700 "(errors %04lx) %s vs. %s", 701 (int)i, testCase.s, aNInfo.getErrors(), buffer, buffer2); 702 continue; 703 } 704 if(aT!=uTaN) { 705 prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 706 prettify(uTaN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 707 errln("T.nameToASCII([%d] %s)!=T.nameToUnicode().N.nameToASCII() " 708 "(errors %04lx) %s vs. %s", 709 (int)i, testCase.s, aNInfo.getErrors(), buffer, buffer2); 710 continue; 711 } 712 if(uN!=aNuN) { 713 prettify(uN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 714 prettify(aNuN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 715 errln("N.nameToUnicode([%d] %s)!=N.nameToASCII().N.nameToUnicode() " 716 "(errors %04lx) %s vs. %s", 717 (int)i, testCase.s, uNInfo.getErrors(), buffer, buffer2); 718 continue; 719 } 720 if(uT!=aTuN) { 721 prettify(uT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 722 prettify(aTuN).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 723 errln("T.nameToUnicode([%d] %s)!=T.nameToASCII().N.nameToUnicode() " 724 "(errors %04lx) %s vs. %s", 725 (int)i, testCase.s, uNInfo.getErrors(), buffer, buffer2); 726 continue; 727 } 728 // labelToUnicode 729 UnicodeString aTL, uTL, aNL, uNL; 730 IDNAInfo aTLInfo, uTLInfo, aNLInfo, uNLInfo; 731 trans->labelToASCII(input, aTL, aTLInfo, errorCode); 732 trans->labelToUnicode(input, uTL, uTLInfo, errorCode); 733 nontrans->labelToASCII(input, aNL, aNLInfo, errorCode); 734 nontrans->labelToUnicode(input, uNL, uNLInfo, errorCode); 735 if(errorCode.errIfFailureAndReset("labelToXYZ processing [%d/%s] %s", 736 (int)i, testCase.o, testCase.s) 737 ) { 738 continue; 739 } 740 if(aN.indexOf((UChar)0x2e)<0) { 741 if(aN!=aNL || aNInfo.getErrors()!=aNLInfo.getErrors()) { 742 prettify(aN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 743 prettify(aNL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 744 errln("N.nameToASCII([%d] %s)!=N.labelToASCII() " 745 "(errors %04lx vs %04lx) %s vs. %s", 746 (int)i, testCase.s, aNInfo.getErrors(), aNLInfo.getErrors(), buffer, buffer2); 747 continue; 748 } 749 } else { 750 if((aNLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) { 751 errln("N.labelToASCII([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT", 752 (int)i, testCase.s, (long)aNLInfo.getErrors()); 753 continue; 754 } 755 } 756 if(aT.indexOf((UChar)0x2e)<0) { 757 if(aT!=aTL || aTInfo.getErrors()!=aTLInfo.getErrors()) { 758 prettify(aT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 759 prettify(aTL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 760 errln("T.nameToASCII([%d] %s)!=T.labelToASCII() " 761 "(errors %04lx vs %04lx) %s vs. %s", 762 (int)i, testCase.s, aTInfo.getErrors(), aTLInfo.getErrors(), buffer, buffer2); 763 continue; 764 } 765 } else { 766 if((aTLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) { 767 errln("T.labelToASCII([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT", 768 (int)i, testCase.s, (long)aTLInfo.getErrors()); 769 continue; 770 } 771 } 772 if(uN.indexOf((UChar)0x2e)<0) { 773 if(uN!=uNL || uNInfo.getErrors()!=uNLInfo.getErrors()) { 774 prettify(uN).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 775 prettify(uNL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 776 errln("N.nameToUnicode([%d] %s)!=N.labelToUnicode() " 777 "(errors %04lx vs %04lx) %s vs. %s", 778 (int)i, testCase.s, uNInfo.getErrors(), uNLInfo.getErrors(), buffer, buffer2); 779 continue; 780 } 781 } else { 782 if((uNLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) { 783 errln("N.labelToUnicode([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT", 784 (int)i, testCase.s, (long)uNLInfo.getErrors()); 785 continue; 786 } 787 } 788 if(uT.indexOf((UChar)0x2e)<0) { 789 if(uT!=uTL || uTInfo.getErrors()!=uTLInfo.getErrors()) { 790 prettify(uT).extract(0, 0x7fffffff, buffer, UPRV_LENGTHOF(buffer)); 791 prettify(uTL).extract(0, 0x7fffffff, buffer2, UPRV_LENGTHOF(buffer2)); 792 errln("T.nameToUnicode([%d] %s)!=T.labelToUnicode() " 793 "(errors %04lx vs %04lx) %s vs. %s", 794 (int)i, testCase.s, uTInfo.getErrors(), uTLInfo.getErrors(), buffer, buffer2); 795 continue; 796 } 797 } else { 798 if((uTLInfo.getErrors()&UIDNA_ERROR_LABEL_HAS_DOT)==0) { 799 errln("T.labelToUnicode([%d] %s) errors %04lx missing UIDNA_ERROR_LABEL_HAS_DOT", 800 (int)i, testCase.s, (long)uTLInfo.getErrors()); 801 continue; 802 } 803 } 804 // Differences between transitional and nontransitional processing 805 if(mode=='B') { 806 if( aNInfo.isTransitionalDifferent() || 807 aTInfo.isTransitionalDifferent() || 808 uNInfo.isTransitionalDifferent() || 809 uTInfo.isTransitionalDifferent() || 810 aNLInfo.isTransitionalDifferent() || 811 aTLInfo.isTransitionalDifferent() || 812 uNLInfo.isTransitionalDifferent() || 813 uTLInfo.isTransitionalDifferent() 814 ) { 815 errln("B.process([%d] %s) isTransitionalDifferent()", (int)i, testCase.s); 816 continue; 817 } 818 if( aN!=aT || uN!=uT || aNL!=aTL || uNL!=uTL || 819 aNInfo.getErrors()!=aTInfo.getErrors() || uNInfo.getErrors()!=uTInfo.getErrors() || 820 aNLInfo.getErrors()!=aTLInfo.getErrors() || uNLInfo.getErrors()!=uTLInfo.getErrors() 821 ) { 822 errln("N.process([%d] %s) vs. T.process() different errors or result strings", 823 (int)i, testCase.s); 824 continue; 825 } 826 } else { 827 if( !aNInfo.isTransitionalDifferent() || 828 !aTInfo.isTransitionalDifferent() || 829 !uNInfo.isTransitionalDifferent() || 830 !uTInfo.isTransitionalDifferent() || 831 !aNLInfo.isTransitionalDifferent() || 832 !aTLInfo.isTransitionalDifferent() || 833 !uNLInfo.isTransitionalDifferent() || 834 !uTLInfo.isTransitionalDifferent() 835 ) { 836 errln("%s.process([%d] %s) !isTransitionalDifferent()", 837 testCase.o, (int)i, testCase.s); 838 continue; 839 } 840 if(aN==aT || uN==uT || aNL==aTL || uNL==uTL) { 841 errln("N.process([%d] %s) vs. T.process() same result strings", 842 (int)i, testCase.s); 843 continue; 844 } 845 } 846 // UTF-8 847 std::string input8, aT8, uT8, aN8, uN8; 848 StringByteSink<std::string> aT8Sink(&aT8), uT8Sink(&uT8), aN8Sink(&aN8), uN8Sink(&uN8); 849 IDNAInfo aT8Info, uT8Info, aN8Info, uN8Info; 850 input.toUTF8String(input8); 851 trans->nameToASCII_UTF8(input8, aT8Sink, aT8Info, errorCode); 852 trans->nameToUnicodeUTF8(input8, uT8Sink, uT8Info, errorCode); 853 nontrans->nameToASCII_UTF8(input8, aN8Sink, aN8Info, errorCode); 854 nontrans->nameToUnicodeUTF8(input8, uN8Sink, uN8Info, errorCode); 855 if(errorCode.errIfFailureAndReset("UTF-8 processing [%d/%s] %s", 856 (int)i, testCase.o, testCase.s) 857 ) { 858 continue; 859 } 860 UnicodeString aT16(UnicodeString::fromUTF8(aT8)); 861 UnicodeString uT16(UnicodeString::fromUTF8(uT8)); 862 UnicodeString aN16(UnicodeString::fromUTF8(aN8)); 863 UnicodeString uN16(UnicodeString::fromUTF8(uN8)); 864 if( aN8Info.getErrors()!=aNInfo.getErrors() || 865 uN8Info.getErrors()!=uNInfo.getErrors() 866 ) { 867 errln("N.xyzUTF8([%d] %s) vs. UTF-16 processing different errors %04lx vs. %04lx", 868 (int)i, testCase.s, 869 (long)aN8Info.getErrors(), (long)aNInfo.getErrors()); 870 continue; 871 } 872 if( aT8Info.getErrors()!=aTInfo.getErrors() || 873 uT8Info.getErrors()!=uTInfo.getErrors() 874 ) { 875 errln("T.xyzUTF8([%d] %s) vs. UTF-16 processing different errors %04lx vs. %04lx", 876 (int)i, testCase.s, 877 (long)aT8Info.getErrors(), (long)aTInfo.getErrors()); 878 continue; 879 } 880 if(aT16!=aT || uT16!=uT || aN16!=aN || uN16!=uN) { 881 errln("%s.xyzUTF8([%d] %s) vs. UTF-16 processing different string results", 882 testCase.o, (int)i, testCase.s, (long)aTInfo.getErrors()); 883 continue; 884 } 885 if( aT8Info.isTransitionalDifferent()!=aTInfo.isTransitionalDifferent() || 886 uT8Info.isTransitionalDifferent()!=uTInfo.isTransitionalDifferent() || 887 aN8Info.isTransitionalDifferent()!=aNInfo.isTransitionalDifferent() || 888 uN8Info.isTransitionalDifferent()!=uNInfo.isTransitionalDifferent() 889 ) { 890 errln("%s.xyzUTF8([%d] %s) vs. UTF-16 processing different isTransitionalDifferent()", 891 testCase.o, (int)i, testCase.s); 892 continue; 893 } 894 } 895 } 896 897 namespace { 898 899 const int32_t kNumFields = 7; 900 901 void U_CALLCONV 902 idnaTestLineFn(void *context, 903 char *fields[][2], int32_t /* fieldCount */, 904 UErrorCode *pErrorCode) { 905 reinterpret_cast<UTS46Test *>(context)->idnaTestOneLine(fields, *pErrorCode); 906 } 907 908 UnicodeString s16FromField(char *(&field)[2]) { 909 int32_t length = (int32_t)(field[1] - field[0]); 910 return UnicodeString::fromUTF8(StringPiece(field[0], length)).trim().unescape(); 911 } 912 913 std::string statusFromField(char *(&field)[2]) { 914 const char *start = u_skipWhitespace(field[0]); 915 std::string status; 916 if (start != field[1]) { 917 int32_t length = (int32_t)(field[1] - start); 918 while (length > 0 && (start[length - 1] == u' ' || start[length - 1] == u'\t')) { 919 --length; 920 } 921 status.assign(start, length); 922 } 923 return status; 924 } 925 926 } // namespace 927 928 void UTS46Test::checkIdnaTestResult(const char *line, const char *type, 929 const UnicodeString &expected, const UnicodeString &result, 930 const char *status, const IDNAInfo &info) { 931 // An error in toUnicode or toASCII is indicated by a value in square brackets, 932 // such as "[B5 B6]". 933 UBool expectedHasErrors = FALSE; 934 if (*status != 0) { 935 if (*status != u'[') { 936 errln("%s status field does not start with '[': %s\n %s", type, status, line); 937 } 938 if (strcmp(status, u8"[]") != 0) { 939 expectedHasErrors = TRUE; 940 } 941 } 942 if (expectedHasErrors != info.hasErrors()) { 943 errln("%s expected errors %s %d != %d = actual has errors: %04lx\n %s", 944 type, status, expectedHasErrors, info.hasErrors(), (long)info.getErrors(), line); 945 } 946 if (!expectedHasErrors && expected != result) { 947 errln("%s expected != actual\n %s", type, line); 948 errln(UnicodeString(u" ") + expected); 949 errln(UnicodeString(u" ") + result); 950 } 951 } 952 953 void UTS46Test::idnaTestOneLine(char *fields[][2], UErrorCode &errorCode) { 954 // IdnaTestV2.txt (since Unicode 11) 955 // Column 1: source 956 // The source string to be tested 957 UnicodeString source = s16FromField(fields[0]); 958 959 // Column 2: toUnicode 960 // The result of applying toUnicode to the source, with Transitional_Processing=false. 961 // A blank value means the same as the source value. 962 UnicodeString toUnicode = s16FromField(fields[1]); 963 if (toUnicode.isEmpty()) { 964 toUnicode = source; 965 } 966 967 // Column 3: toUnicodeStatus 968 // A set of status codes, each corresponding to a particular test. 969 // A blank value means []. 970 std::string toUnicodeStatus = statusFromField(fields[2]); 971 972 // Column 4: toAsciiN 973 // The result of applying toASCII to the source, with Transitional_Processing=false. 974 // A blank value means the same as the toUnicode value. 975 UnicodeString toAsciiN = s16FromField(fields[3]); 976 if (toAsciiN.isEmpty()) { 977 toAsciiN = toUnicode; 978 } 979 980 // Column 5: toAsciiNStatus 981 // A set of status codes, each corresponding to a particular test. 982 // A blank value means the same as the toUnicodeStatus value. 983 std::string toAsciiNStatus = statusFromField(fields[4]); 984 if (toAsciiNStatus.empty()) { 985 toAsciiNStatus = toUnicodeStatus; 986 } 987 988 // Column 6: toAsciiT 989 // The result of applying toASCII to the source, with Transitional_Processing=true. 990 // A blank value means the same as the toAsciiN value. 991 UnicodeString toAsciiT = s16FromField(fields[5]); 992 if (toAsciiT.isEmpty()) { 993 toAsciiT = toAsciiN; 994 } 995 996 // Column 7: toAsciiTStatus 997 // A set of status codes, each corresponding to a particular test. 998 // A blank value means the same as the toAsciiNStatus value. 999 std::string toAsciiTStatus = statusFromField(fields[6]); 1000 if (toAsciiTStatus.empty()) { 1001 toAsciiTStatus = toAsciiNStatus; 1002 } 1003 1004 // ToASCII/ToUnicode, transitional/nontransitional 1005 UnicodeString uN, aN, aT; 1006 IDNAInfo uNInfo, aNInfo, aTInfo; 1007 nontrans->nameToUnicode(source, uN, uNInfo, errorCode); 1008 checkIdnaTestResult(fields[0][0], "toUnicodeNontrans", toUnicode, uN, 1009 toUnicodeStatus.c_str(), uNInfo); 1010 nontrans->nameToASCII(source, aN, aNInfo, errorCode); 1011 checkIdnaTestResult(fields[0][0], "toASCIINontrans", toAsciiN, aN, 1012 toAsciiNStatus.c_str(), aNInfo); 1013 trans->nameToASCII(source, aT, aTInfo, errorCode); 1014 checkIdnaTestResult(fields[0][0], "toASCIITrans", toAsciiT, aT, 1015 toAsciiTStatus.c_str(), aTInfo); 1016 } 1017 1018 // TODO: de-duplicate 1019 U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose); 1020 1021 // http://www.unicode.org/Public/idna/latest/IdnaTest.txt 1022 void UTS46Test::IdnaTest() { 1023 IcuTestErrorCode errorCode(*this, "IdnaTest"); 1024 const char *sourceTestDataPath = getSourceTestData(errorCode); 1025 if (errorCode.errIfFailureAndReset("unable to find the source/test/testdata " 1026 "folder (getSourceTestData())")) { 1027 return; 1028 } 1029 CharString path(sourceTestDataPath, errorCode); 1030 path.appendPathPart("IdnaTestV2.txt", errorCode); 1031 LocalStdioFilePointer idnaTestFile(fopen(path.data(), "r")); 1032 if (idnaTestFile.isNull()) { 1033 errln("unable to open %s", path.data()); 1034 return; 1035 } 1036 1037 // Columns (c1, c2,...) are separated by semicolons. 1038 // Leading and trailing spaces and tabs in each column are ignored. 1039 // Comments are indicated with hash marks. 1040 char *fields[kNumFields][2]; 1041 u_parseDelimitedFile(path.data(), ';', fields, kNumFields, idnaTestLineFn, this, errorCode); 1042 if (errorCode.errIfFailureAndReset("error parsing IdnaTest.txt")) { 1043 return; 1044 } 1045 } 1046 1047 #endif // UCONFIG_NO_IDNA 1048
