1 /******************************************************************** 2 * COPYRIGHT: 3 * Copyright (c) 1998-2012, International Business Machines Corporation and 4 * others. All Rights Reserved. 5 ********************************************************************/ 6 /* 7 * File utf8tst.c 8 * 9 * Modification History: 10 * 11 * Date Name Description 12 * 07/24/2000 Madhu Creation 13 ******************************************************************************* 14 */ 15 16 #include "unicode/utypes.h" 17 #include "unicode/utf8.h" 18 #include "cmemory.h" 19 #include "cintltst.h" 20 21 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) 22 23 /* lenient UTF-8 ------------------------------------------------------------ */ 24 25 /* 26 * Lenient UTF-8 differs from conformant UTF-8 in that it allows surrogate 27 * code points with their "natural" encoding. 28 * Effectively, this allows a mix of UTF-8 and CESU-8 as well as encodings of 29 * single surrogates. 30 * 31 * This is not conformant with UTF-8. 32 * 33 * Supplementary code points may be encoded as pairs of 3-byte sequences, but 34 * the macros below do not attempt to assemble such pairs. 35 */ 36 37 #define L8_NEXT(s, i, length, c) { \ 38 (c)=(uint8_t)(s)[(i)++]; \ 39 if((c)>=0x80) { \ 40 if(U8_IS_LEAD(c)) { \ 41 (c)=utf8_nextCharSafeBody((const uint8_t *)s, &(i), (int32_t)(length), c, -2); \ 42 } else { \ 43 (c)=U_SENTINEL; \ 44 } \ 45 } \ 46 } 47 48 #define L8_PREV(s, start, i, c) { \ 49 (c)=(uint8_t)(s)[--(i)]; \ 50 if((c)>=0x80) { \ 51 if((c)<=0xbf) { \ 52 (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -2); \ 53 } else { \ 54 (c)=U_SENTINEL; \ 55 } \ 56 } \ 57 } 58 59 /* -------------------------------------------------------------------------- */ 60 61 static void printUChars(const uint8_t *uchars, int16_t len); 62 63 static void TestCodeUnitValues(void); 64 static void TestCharLength(void); 65 static void TestGetChar(void); 66 static void TestNextPrevChar(void); 67 static void TestNulTerminated(void); 68 static void TestNextPrevNonCharacters(void); 69 static void TestNextPrevCharUnsafe(void); 70 static void TestFwdBack(void); 71 static void TestFwdBackUnsafe(void); 72 static void TestSetChar(void); 73 static void TestSetCharUnsafe(void); 74 static void TestAppendChar(void); 75 static void TestAppend(void); 76 static void TestSurrogates(void); 77 78 void addUTF8Test(TestNode** root); 79 80 void 81 addUTF8Test(TestNode** root) 82 { 83 addTest(root, &TestCodeUnitValues, "utf8tst/TestCodeUnitValues"); 84 addTest(root, &TestCharLength, "utf8tst/TestCharLength"); 85 addTest(root, &TestGetChar, "utf8tst/TestGetChar"); 86 addTest(root, &TestNextPrevChar, "utf8tst/TestNextPrevChar"); 87 addTest(root, &TestNulTerminated, "utf8tst/TestNulTerminated"); 88 addTest(root, &TestNextPrevNonCharacters, "utf8tst/TestNextPrevNonCharacters"); 89 addTest(root, &TestNextPrevCharUnsafe, "utf8tst/TestNextPrevCharUnsafe"); 90 addTest(root, &TestFwdBack, "utf8tst/TestFwdBack"); 91 addTest(root, &TestFwdBackUnsafe, "utf8tst/TestFwdBackUnsafe"); 92 addTest(root, &TestSetChar, "utf8tst/TestSetChar"); 93 addTest(root, &TestSetCharUnsafe, "utf8tst/TestSetCharUnsafe"); 94 addTest(root, &TestAppendChar, "utf8tst/TestAppendChar"); 95 addTest(root, &TestAppend, "utf8tst/TestAppend"); 96 addTest(root, &TestSurrogates, "utf8tst/TestSurrogates"); 97 } 98 99 static void TestCodeUnitValues() 100 { 101 static const uint8_t codeunit[]={0x00, 0x65, 0x7e, 0x7f, 0xc0, 0xc4, 0xf0, 0xfd, 0x80, 0x81, 0xbc, 0xbe,}; 102 103 int16_t i; 104 for(i=0; i<LENGTHOF(codeunit); i++){ 105 uint8_t c=codeunit[i]; 106 log_verbose("Testing code unit value of %x\n", c); 107 if(i<4){ 108 if(!UTF8_IS_SINGLE(c) || UTF8_IS_LEAD(c) || UTF8_IS_TRAIL(c) || !U8_IS_SINGLE(c) || U8_IS_LEAD(c) || U8_IS_TRAIL(c)){ 109 log_err("ERROR: 0x%02x is a single byte but results in single: %c lead: %c trail: %c\n", 110 c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n'); 111 } 112 } else if(i< 8){ 113 if(!UTF8_IS_LEAD(c) || UTF8_IS_SINGLE(c) || UTF8_IS_TRAIL(c) || !U8_IS_LEAD(c) || U8_IS_SINGLE(c) || U8_IS_TRAIL(c)){ 114 log_err("ERROR: 0x%02x is a lead byte but results in single: %c lead: %c trail: %c\n", 115 c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n'); 116 } 117 } else if(i< 12){ 118 if(!UTF8_IS_TRAIL(c) || UTF8_IS_SINGLE(c) || UTF8_IS_LEAD(c) || !U8_IS_TRAIL(c) || U8_IS_SINGLE(c) || U8_IS_LEAD(c)){ 119 log_err("ERROR: 0x%02x is a trail byte but results in single: %c lead: %c trail: %c\n", 120 c, UTF8_IS_SINGLE(c) ? 'y' : 'n', UTF8_IS_LEAD(c) ? 'y' : 'n', UTF8_IS_TRAIL(c) ? 'y' : 'n'); 121 } 122 } 123 } 124 } 125 126 static void TestCharLength() 127 { 128 static const uint32_t codepoint[]={ 129 1, 0x0061, 130 1, 0x007f, 131 2, 0x016f, 132 2, 0x07ff, 133 3, 0x0865, 134 3, 0x20ac, 135 4, 0x20402, 136 4, 0x23456, 137 4, 0x24506, 138 4, 0x20402, 139 4, 0x10402, 140 3, 0xd7ff, 141 3, 0xe000, 142 143 }; 144 145 int16_t i; 146 UBool multiple; 147 for(i=0; i<LENGTHOF(codepoint); i=(int16_t)(i+2)){ 148 UChar32 c=codepoint[i+1]; 149 if(UTF8_CHAR_LENGTH(c) != (uint16_t)codepoint[i] || U8_LENGTH(c) != (uint16_t)codepoint[i]){ 150 log_err("The no: of code units for %lx:- Expected: %d Got: %d\n", c, codepoint[i], UTF8_CHAR_LENGTH(c)); 151 }else{ 152 log_verbose("The no: of code units for %lx is %d\n",c, UTF8_CHAR_LENGTH(c)); 153 } 154 multiple=(UBool)(codepoint[i] == 1 ? FALSE : TRUE); 155 if(UTF8_NEED_MULTIPLE_UCHAR(c) != multiple){ 156 log_err("ERROR: UTF8_NEED_MULTIPLE_UCHAR failed for %lx\n", c); 157 } 158 } 159 } 160 161 static void TestGetChar() 162 { 163 static const uint8_t input[]={ 164 /* code unit,*/ 165 0x61, 166 0x7f, 167 0xe4, 168 0xba, 169 0x8c, 170 0xF0, 171 0x90, 172 0x90, 173 0x81, 174 0xc0, 175 0x65, 176 0x31, 177 0x9a, 178 0xc9 179 }; 180 static const UChar32 result[]={ 181 /* codepoint-unsafe, codepoint-safe(not strict) codepoint-safe(strict) */ 182 0x61, 0x61, 0x61, 183 0x7f, 0x7f, 0x7f, 184 0x4e8c, 0x4e8c, 0x4e8c, 185 0x4e8c, 0x4e8c, 0x4e8c , 186 0x4e8c, 0x4e8c, 0x4e8c, 187 0x10401, 0x10401, 0x10401 , 188 0x10401, 0x10401, 0x10401 , 189 0x10401, 0x10401, 0x10401 , 190 0x10401, 0x10401, 0x10401, 191 0x25, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 192 0x65, 0x65, 0x65, 193 0x31, 0x31, 0x31, 194 0x31, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 195 0x240, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1 196 }; 197 uint16_t i=0; 198 UChar32 c, expected; 199 uint32_t offset=0; 200 201 for(offset=0; offset<sizeof(input); offset++) { 202 if (offset < sizeof(input) - 1) { 203 UTF8_GET_CHAR_UNSAFE(input, offset, c); 204 if(c != result[i]){ 205 log_err("ERROR: UTF8_GET_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c); 206 207 } 208 209 U8_GET_UNSAFE(input, offset, c); 210 if(c != result[i]){ 211 log_err("ERROR: U8_GET_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i], c); 212 213 } 214 } 215 216 UTF8_GET_CHAR_SAFE(input, 0, offset, sizeof(input), c, FALSE); 217 expected=result[i+1]; 218 if(c != expected){ 219 log_err("ERROR: UTF8_GET_CHAR_SAFE failed for offset=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 220 } 221 222 U8_GET(input, 0, offset, sizeof(input), c); 223 if(UTF_IS_ERROR(expected)) { expected=U_SENTINEL; } 224 if(c != expected){ 225 log_err("ERROR: U8_GET failed for offset=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 226 } 227 228 U8_GET_OR_FFFD(input, 0, offset, sizeof(input), c); 229 if(expected<0) { expected=0xfffd; } 230 if(c != expected){ 231 log_err("ERROR: U8_GET_OR_FFFD failed for offset=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 232 } 233 234 UTF8_GET_CHAR_SAFE(input, 0, offset, sizeof(input), c, TRUE); 235 if(c != result[i+2]){ 236 log_err("ERROR: UTF8_GET_CHAR_SAFE(strict) failed for offset=%ld. Expected:%lx Got:%lx\n", offset, result[i+2], c); 237 } 238 239 i=(uint16_t)(i+3); 240 } 241 } 242 243 static void TestNextPrevChar() { 244 static const uint8_t input[]={0x61, 0xf0, 0x90, 0x90, 0x81, 0xc0, 0x80, 0xfd, 0xbe, 0xc2, 0x61, 0x81, 0x90, 0x90, 0xf0, 0x00}; 245 static const UChar32 result[]={ 246 /* next_unsafe next_safe_ns next_safe_s prev_unsafe prev_safe_ns prev_safe_s */ 247 0x0061, 0x0061, 0x0061, 0x0000, 0x0000, 0x0000, 248 0x10401, 0x10401, 0x10401, 0xf0, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 249 0x90, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x2841410, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 250 0x90, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0xa1050, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 251 0x81, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x2841, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 252 0x00, UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2, 0x61, 0x61, 0x61, 253 0x80, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0xc2, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 254 0xfd, UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2, 0x77e, UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2, 255 0xbe, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0xfd, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 256 0xa1, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x00, UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2, 257 0x61, 0x61, 0x61, 0xc0, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 258 0x81, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x10401, 0x10401, 0x10401, 259 0x90, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x410, UTF_ERROR_VALUE, UTF_ERROR_VALUE, 260 0x90, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0x410, UTF8_ERROR_VALUE_2, UTF8_ERROR_VALUE_2, 261 0x0840, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 0xf0, UTF8_ERROR_VALUE_1, UTF8_ERROR_VALUE_1, 262 0x0000, 0x0000, 0x0000, 0x0061, 0x0061, 0x0061 263 }; 264 static const int32_t movedOffset[]={ 265 /* next_unsafe next_safe_ns next_safe_s prev_unsafe prev_safe_ns prev_safe_s */ 266 1, 1, 1, 15, 15, 15, 267 5, 5, 5, 14, 14 , 14, 268 3, 3, 3, 9, 13, 13, 269 4, 4, 4, 9, 12, 12, 270 5, 5, 5, 9, 11, 11, 271 7, 7, 7, 10, 10, 10, 272 7, 7, 7, 9, 9, 9, 273 8, 9, 9, 7, 7, 7, 274 9, 9, 9, 7, 7, 7, 275 11, 10, 10, 5, 5, 5, 276 11, 11, 11, 5, 5, 5, 277 12, 12, 12, 1, 1, 1, 278 13, 13, 13, 1, 1, 1, 279 14, 14, 14, 1, 1, 1, 280 14, 15, 15, 1, 1, 1, 281 14, 16, 16, 0, 0, 0, 282 }; 283 /* TODO: remove unused columns for next_unsafe & prev_unsafe, and adjust the test code */ 284 285 UChar32 c, expected; 286 uint32_t i=0; 287 uint32_t offset=0; 288 int32_t setOffset=0; 289 for(offset=0; offset<sizeof(input); offset++){ 290 setOffset=offset; 291 UTF8_NEXT_CHAR_SAFE(input, setOffset, sizeof(input), c, FALSE); 292 if(setOffset != movedOffset[i+1]){ 293 log_err("ERROR: UTF8_NEXT_CHAR_SAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 294 offset, movedOffset[i+1], setOffset); 295 } 296 expected=result[i+1]; 297 if(c != expected){ 298 log_err("ERROR: UTF8_NEXT_CHAR_SAFE failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 299 } 300 301 setOffset=offset; 302 U8_NEXT(input, setOffset, sizeof(input), c); 303 if(setOffset != movedOffset[i+1]){ 304 log_err("ERROR: U8_NEXT failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 305 offset, movedOffset[i+1], setOffset); 306 } 307 if(UTF_IS_ERROR(expected)) { expected=U_SENTINEL; } 308 if(c != expected){ 309 log_err("ERROR: U8_NEXT failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 310 } 311 312 setOffset=offset; 313 U8_NEXT_OR_FFFD(input, setOffset, sizeof(input), c); 314 if(setOffset != movedOffset[i+1]){ 315 log_err("ERROR: U8_NEXT_OR_FFFD failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 316 offset, movedOffset[i+1], setOffset); 317 } 318 if(expected<0) { expected=0xfffd; } 319 if(c != expected){ 320 log_err("ERROR: U8_NEXT_OR_FFFD failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 321 } 322 323 setOffset=offset; 324 UTF8_NEXT_CHAR_SAFE(input, setOffset, sizeof(input), c, TRUE); 325 if(setOffset != movedOffset[i+1]){ 326 log_err("ERROR: UTF8_NEXT_CHAR_SAFE(strict) failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 327 offset, movedOffset[i+2], setOffset); 328 } 329 if(c != result[i+2]){ 330 log_err("ERROR: UTF8_NEXT_CHAR_SAFE(strict) failed for input=%ld. Expected:%lx Got:%lx\n", offset, result[i+2], c); 331 } 332 333 i=i+6; 334 } 335 336 i=0; 337 for(offset=sizeof(input); offset > 0; --offset){ 338 setOffset=offset; 339 UTF8_PREV_CHAR_SAFE(input, 0, setOffset, c, FALSE); 340 if(setOffset != movedOffset[i+4]){ 341 log_err("ERROR: UTF8_PREV_CHAR_SAFE failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 342 offset, movedOffset[i+4], setOffset); 343 } 344 expected=result[i+4]; 345 if(c != expected){ 346 log_err("ERROR: UTF8_PREV_CHAR_SAFE failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 347 } 348 349 setOffset=offset; 350 U8_PREV(input, 0, setOffset, c); 351 if(setOffset != movedOffset[i+4]){ 352 log_err("ERROR: U8_PREV failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 353 offset, movedOffset[i+4], setOffset); 354 } 355 if(UTF_IS_ERROR(expected)) { expected=U_SENTINEL; } 356 if(c != expected){ 357 log_err("ERROR: U8_PREV failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 358 } 359 360 setOffset=offset; 361 U8_PREV_OR_FFFD(input, 0, setOffset, c); 362 if(setOffset != movedOffset[i+4]){ 363 log_err("ERROR: U8_PREV_OR_FFFD failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 364 offset, movedOffset[i+4], setOffset); 365 } 366 if(expected<0) { expected=0xfffd; } 367 if(c != expected){ 368 log_err("ERROR: U8_PREV_OR_FFFD failed for input=%ld. Expected:%lx Got:%lx\n", offset, expected, c); 369 } 370 371 setOffset=offset; 372 UTF8_PREV_CHAR_SAFE(input, 0, setOffset, c, TRUE); 373 if(setOffset != movedOffset[i+5]){ 374 log_err("ERROR: UTF8_PREV_CHAR_SAFE(strict) failed to move the offset correctly at %d\n ExpectedOffset:%d Got %d\n", 375 offset, movedOffset[i+5], setOffset); 376 } 377 if(c != result[i+5]){ 378 log_err("ERROR: UTF8_PREV_CHAR_SAFE(strict) failed for input=%ld. Expected:%lx Got:%lx\n", offset, result[i+5], c); 379 } 380 381 i=i+6; 382 } 383 } 384 385 /* keep this in sync with utf16tst.c's TestNulTerminated() */ 386 static void TestNulTerminated() { 387 static const uint8_t input[]={ 388 /* 0 */ 0x61, 389 /* 1 */ 0xf0, 0x90, 0x90, 0x81, 390 /* 5 */ 0xc0, 0x80, 391 /* 7 */ 0xdf, 0x80, 392 /* 9 */ 0xc2, 393 /* 10 */ 0x62, 394 /* 11 */ 0xfd, 0xbe, 395 /* 13 */ 0xe0, 0xa0, 0x80, 396 /* 16 */ 0xe2, 0x82, 0xac, 397 /* 19 */ 0xf0, 0x90, 0x90, 398 /* 22 */ 0x00 399 /* 23 */ 400 }; 401 static const UChar32 result[]={ 402 0x61, 403 0x10401, 404 U_SENTINEL, 405 0x7c0, 406 U_SENTINEL, 407 0x62, 408 U_SENTINEL, 409 0x800, 410 0x20ac, 411 U_SENTINEL, 412 0 413 }; 414 415 UChar32 c, c2, expected; 416 int32_t i0, i=0, j, k, expectedIndex; 417 int32_t cpIndex=0; 418 do { 419 i0=i; 420 U8_NEXT(input, i, -1, c); 421 expected=result[cpIndex]; 422 if(c!=expected) { 423 log_err("U8_NEXT(from %d)=U+%04x != U+%04x\n", i0, c, expected); 424 } 425 j=i0; 426 U8_NEXT_OR_FFFD(input, j, -1, c); 427 if(expected<0) { expected=0xfffd; } 428 if(c!=expected) { 429 log_err("U8_NEXT_OR_FFFD(from %d)=U+%04x != U+%04x\n", i0, c, expected); 430 } 431 if(j!=i) { 432 log_err("U8_NEXT_OR_FFFD() moved to index %d but U8_NEXT() moved to %d\n", j, i); 433 } 434 j=i0; 435 U8_FWD_1(input, j, -1); 436 if(j!=i) { 437 log_err("U8_FWD_1() moved to index %d but U8_NEXT() moved to %d\n", j, i); 438 } 439 ++cpIndex; 440 /* 441 * Move by this many code points from the start. 442 * U8_FWD_N() stops at the end of the string, that is, at the NUL if necessary. 443 */ 444 expectedIndex= (c==0) ? i-1 : i; 445 k=0; 446 U8_FWD_N(input, k, -1, cpIndex); 447 if(k!=expectedIndex) { 448 log_err("U8_FWD_N(code points from 0) moved to index %d but expected %d\n", k, expectedIndex); 449 } 450 } while(c!=0); 451 452 i=0; 453 do { 454 j=i0=i; 455 U8_NEXT(input, i, -1, c); 456 do { 457 U8_GET(input, 0, j, -1, c2); 458 if(c2!=c) { 459 log_err("U8_NEXT(from %d)=U+%04x != U+%04x=U8_GET(at %d)\n", i0, c, c2, j); 460 } 461 U8_GET_OR_FFFD(input, 0, j, -1, c2); 462 expected= (c>=0) ? c : 0xfffd; 463 if(c2!=expected) { 464 log_err("U8_NEXT_OR_FFFD(from %d)=U+%04x != U+%04x=U8_GET_OR_FFFD(at %d)\n", i0, expected, c2, j); 465 } 466 /* U8_SET_CP_LIMIT moves from a non-lead byte to the limit of the code point */ 467 k=j+1; 468 U8_SET_CP_LIMIT(input, 0, k, -1); 469 if(k!=i) { 470 log_err("U8_NEXT() moved to %d but U8_SET_CP_LIMIT(%d) moved to %d\n", i, j+1, k); 471 } 472 } while(++j<i); 473 } while(c!=0); 474 } 475 476 static void TestNextPrevNonCharacters() { 477 /* test non-characters */ 478 static const uint8_t nonChars[]={ 479 0xef, 0xb7, 0x90, /* U+fdd0 */ 480 0xef, 0xbf, 0xbf, /* U+feff */ 481 0xf0, 0x9f, 0xbf, 0xbe, /* U+1fffe */ 482 0xf0, 0xbf, 0xbf, 0xbf, /* U+3ffff */ 483 0xf4, 0x8f, 0xbf, 0xbe /* U+10fffe */ 484 }; 485 486 UChar32 ch; 487 int32_t idx; 488 489 for(idx=0; idx<(int32_t)sizeof(nonChars);) { 490 U8_NEXT(nonChars, idx, sizeof(nonChars), ch); 491 if(!U_IS_UNICODE_NONCHAR(ch)) { 492 log_err("U8_NEXT(before %d) failed to read a non-character\n", idx); 493 } 494 } 495 for(idx=(int32_t)sizeof(nonChars); idx>0;) { 496 U8_PREV(nonChars, 0, idx, ch); 497 if(!U_IS_UNICODE_NONCHAR(ch)) { 498 log_err("U8_PREV(at %d) failed to read a non-character\n", idx); 499 } 500 } 501 } 502 503 static void TestNextPrevCharUnsafe() { 504 /* 505 * Use a (mostly) well-formed UTF-8 string and test at code point boundaries. 506 * The behavior of _UNSAFE macros for ill-formed strings is undefined. 507 */ 508 static const uint8_t input[]={ 509 0x61, 510 0xf0, 0x90, 0x90, 0x81, 511 0xc0, 0x80, /* non-shortest form */ 512 0xe2, 0x82, 0xac, 513 0xc2, 0xa1, 514 0xf4, 0x8f, 0xbf, 0xbf, 515 0x00 516 }; 517 static const UChar32 codePoints[]={ 518 0x61, 519 0x10401, 520 0, 521 0x20ac, 522 0xa1, 523 0x10ffff, 524 0 525 }; 526 527 UChar32 c; 528 int32_t i; 529 uint32_t offset; 530 for(i=0, offset=0; offset<sizeof(input); ++i) { 531 UTF8_NEXT_CHAR_UNSAFE(input, offset, c); 532 if(c != codePoints[i]){ 533 log_err("ERROR: UTF8_NEXT_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", 534 offset, codePoints[i], c); 535 } 536 } 537 for(i=0, offset=0; offset<sizeof(input); ++i) { 538 U8_NEXT_UNSAFE(input, offset, c); 539 if(c != codePoints[i]){ 540 log_err("ERROR: U8_NEXT_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", 541 offset, codePoints[i], c); 542 } 543 } 544 545 for(i=LENGTHOF(codePoints)-1, offset=sizeof(input); offset > 0; --i){ 546 UTF8_PREV_CHAR_UNSAFE(input, offset, c); 547 if(c != codePoints[i]){ 548 log_err("ERROR: UTF8_PREV_CHAR_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", 549 offset, codePoints[i], c); 550 } 551 } 552 for(i=LENGTHOF(codePoints)-1, offset=sizeof(input); offset > 0; --i){ 553 U8_PREV_UNSAFE(input, offset, c); 554 if(c != codePoints[i]){ 555 log_err("ERROR: U8_PREV_UNSAFE failed for offset=%ld. Expected:%lx Got:%lx\n", 556 offset, codePoints[i], c); 557 } 558 } 559 } 560 561 static void TestFwdBack() { 562 static const uint8_t input[]={0x61, 0xF0, 0x90, 0x90, 0x81, 0xff, 0x62, 0xc0, 0x80, 0x7f, 0x8f, 0xc0, 0x63, 0x81, 0x90, 0x90, 0xF0, 0x00}; 563 static const uint16_t fwd_safe[] ={1, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}; 564 static const uint16_t back_safe[] ={17, 16, 15, 14, 13, 12, 11, 10, 9, 7, 6, 5, 1, 0}; 565 566 static const uint16_t Nvalue[]= {0, 1, 2, 3, 1, 2, 1, 5}; 567 static const uint16_t fwd_N_safe[] ={0, 1, 6, 10, 11, 13, 14, 18}; /*safe macro keeps it at the end of the string */ 568 static const uint16_t back_N_safe[] ={18, 17, 15, 12, 11, 9, 7, 0}; 569 570 uint32_t offsafe=0; 571 572 uint32_t i=0; 573 while(offsafe < sizeof(input)){ 574 UTF8_FWD_1_SAFE(input, offsafe, sizeof(input)); 575 if(offsafe != fwd_safe[i]){ 576 log_err("ERROR: Forward_safe offset expected:%d, Got:%d\n", fwd_safe[i], offsafe); 577 } 578 i++; 579 } 580 581 i=0; 582 while(offsafe < sizeof(input)){ 583 U8_FWD_1(input, offsafe, sizeof(input)); 584 if(offsafe != fwd_safe[i]){ 585 log_err("ERROR: U8_FWD_1 offset expected:%d, Got:%d\n", fwd_safe[i], offsafe); 586 } 587 i++; 588 } 589 590 i=0; 591 offsafe=sizeof(input); 592 while(offsafe > 0){ 593 UTF8_BACK_1_SAFE(input, 0, offsafe); 594 if(offsafe != back_safe[i]){ 595 log_err("ERROR: Backward_safe offset expected:%d, Got:%d\n", back_safe[i], offsafe); 596 } 597 i++; 598 } 599 600 i=0; 601 offsafe=sizeof(input); 602 while(offsafe > 0){ 603 U8_BACK_1(input, 0, offsafe); 604 if(offsafe != back_safe[i]){ 605 log_err("ERROR: U8_BACK_1 offset expected:%d, Got:%d\n", back_safe[i], offsafe); 606 } 607 i++; 608 } 609 610 offsafe=0; 611 for(i=0; i<LENGTHOF(Nvalue); i++){ 612 UTF8_FWD_N_SAFE(input, offsafe, sizeof(input), Nvalue[i]); 613 if(offsafe != fwd_N_safe[i]){ 614 log_err("ERROR: Forward_N_safe offset=%d expected:%d, Got:%d\n", i, fwd_N_safe[i], offsafe); 615 } 616 617 } 618 619 offsafe=0; 620 for(i=0; i<LENGTHOF(Nvalue); i++){ 621 U8_FWD_N(input, offsafe, sizeof(input), Nvalue[i]); 622 if(offsafe != fwd_N_safe[i]){ 623 log_err("ERROR: U8_FWD_N offset=%d expected:%d, Got:%d\n", i, fwd_N_safe[i], offsafe); 624 } 625 626 } 627 628 offsafe=sizeof(input); 629 for(i=0; i<LENGTHOF(Nvalue); i++){ 630 UTF8_BACK_N_SAFE(input, 0, offsafe, Nvalue[i]); 631 if(offsafe != back_N_safe[i]){ 632 log_err("ERROR: backward_N_safe offset=%d expected:%d, Got:%ld\n", i, back_N_safe[i], offsafe); 633 } 634 } 635 636 offsafe=sizeof(input); 637 for(i=0; i<LENGTHOF(Nvalue); i++){ 638 U8_BACK_N(input, 0, offsafe, Nvalue[i]); 639 if(offsafe != back_N_safe[i]){ 640 log_err("ERROR: U8_BACK_N offset=%d expected:%d, Got:%ld\n", i, back_N_safe[i], offsafe); 641 } 642 } 643 } 644 645 static void TestFwdBackUnsafe() { 646 /* 647 * Use a (mostly) well-formed UTF-8 string and test at code point boundaries. 648 * The behavior of _UNSAFE macros for ill-formed strings is undefined. 649 */ 650 static const uint8_t input[]={ 651 0x61, 652 0xf0, 0x90, 0x90, 0x81, 653 0xc0, 0x80, /* non-shortest form */ 654 0xe2, 0x82, 0xac, 655 0xc2, 0xa1, 656 0xf4, 0x8f, 0xbf, 0xbf, 657 0x00 658 }; 659 static const int8_t boundaries[]={ 0, 1, 5, 7, 10, 12, 16, 17 }; 660 661 int32_t offset; 662 int32_t i; 663 for(i=1, offset=0; offset<LENGTHOF(input); ++i) { 664 UTF8_FWD_1_UNSAFE(input, offset); 665 if(offset != boundaries[i]){ 666 log_err("ERROR: UTF8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset); 667 } 668 } 669 for(i=1, offset=0; offset<LENGTHOF(input); ++i) { 670 U8_FWD_1_UNSAFE(input, offset); 671 if(offset != boundaries[i]){ 672 log_err("ERROR: U8_FWD_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset); 673 } 674 } 675 676 for(i=LENGTHOF(boundaries)-2, offset=LENGTHOF(input); offset>0; --i) { 677 UTF8_BACK_1_UNSAFE(input, offset); 678 if(offset != boundaries[i]){ 679 log_err("ERROR: UTF8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset); 680 } 681 } 682 for(i=LENGTHOF(boundaries)-2, offset=LENGTHOF(input); offset>0; --i) { 683 U8_BACK_1_UNSAFE(input, offset); 684 if(offset != boundaries[i]){ 685 log_err("ERROR: U8_BACK_1_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset); 686 } 687 } 688 689 for(i=0; i<LENGTHOF(boundaries); ++i) { 690 offset=0; 691 UTF8_FWD_N_UNSAFE(input, offset, i); 692 if(offset != boundaries[i]) { 693 log_err("ERROR: UTF8_FWD_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset); 694 } 695 } 696 for(i=0; i<LENGTHOF(boundaries); ++i) { 697 offset=0; 698 U8_FWD_N_UNSAFE(input, offset, i); 699 if(offset != boundaries[i]) { 700 log_err("ERROR: U8_FWD_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[i], offset); 701 } 702 } 703 704 for(i=0; i<LENGTHOF(boundaries); ++i) { 705 int32_t j=LENGTHOF(boundaries)-1-i; 706 offset=LENGTHOF(input); 707 UTF8_BACK_N_UNSAFE(input, offset, i); 708 if(offset != boundaries[j]) { 709 log_err("ERROR: UTF8_BACK_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[j], offset); 710 } 711 } 712 for(i=0; i<LENGTHOF(boundaries); ++i) { 713 int32_t j=LENGTHOF(boundaries)-1-i; 714 offset=LENGTHOF(input); 715 U8_BACK_N_UNSAFE(input, offset, i); 716 if(offset != boundaries[j]) { 717 log_err("ERROR: U8_BACK_N_UNSAFE offset expected:%d, Got:%d\n", boundaries[j], offset); 718 } 719 } 720 } 721 722 static void TestSetChar() { 723 static const uint8_t input[] 724 = {0x61, 0xe4, 0xba, 0x8c, 0x7f, 0xfe, 0x62, 0xc5, 0x7f, 0x61, 0x80, 0x80, 0xe0, 0x00 }; 725 static const int16_t start_safe[] 726 = {0, 1, 1, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; 727 static const int16_t limit_safe[] 728 = {0, 1, 4, 4, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; 729 730 uint32_t i=0; 731 int32_t offset=0, setOffset=0; 732 for(offset=0; offset<=LENGTHOF(input); offset++){ 733 if (offset<LENGTHOF(input)){ 734 setOffset=offset; 735 UTF8_SET_CHAR_START_SAFE(input, 0, setOffset); 736 if(setOffset != start_safe[i]){ 737 log_err("ERROR: UTF8_SET_CHAR_START_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset); 738 } 739 740 setOffset=offset; 741 U8_SET_CP_START(input, 0, setOffset); 742 if(setOffset != start_safe[i]){ 743 log_err("ERROR: U8_SET_CP_START failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_safe[i], setOffset); 744 } 745 } 746 747 setOffset=offset; 748 UTF8_SET_CHAR_LIMIT_SAFE(input,0, setOffset, sizeof(input)); 749 if(setOffset != limit_safe[i]){ 750 log_err("ERROR: UTF8_SET_CHAR_LIMIT_SAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset); 751 } 752 753 setOffset=offset; 754 U8_SET_CP_LIMIT(input,0, setOffset, sizeof(input)); 755 if(setOffset != limit_safe[i]){ 756 log_err("ERROR: U8_SET_CP_LIMIT failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_safe[i], setOffset); 757 } 758 759 i++; 760 } 761 } 762 763 static void TestSetCharUnsafe() { 764 static const uint8_t input[] 765 = {0x61, 0xe4, 0xba, 0x8c, 0x7f, 0x2e, 0x62, 0xc5, 0x7f, 0x61, 0x80, 0x80, 0xe0, 0x80, 0x80, 0x00 }; 766 static const int16_t start_unsafe[] 767 = {0, 1, 1, 1, 4, 5, 6, 7, 8, 9, 9, 9, 12, 12, 12, 15 }; 768 static const int16_t limit_unsafe[] 769 = {0, 1, 4, 4, 4, 5, 6, 7, 9, 9, 10, 10, 10, 15, 15, 15, 16 }; 770 771 uint32_t i=0; 772 int32_t offset=0, setOffset=0; 773 for(offset=0; offset<=LENGTHOF(input); offset++){ 774 if (offset<LENGTHOF(input)){ 775 setOffset=offset; 776 UTF8_SET_CHAR_START_UNSAFE(input, setOffset); 777 if(setOffset != start_unsafe[i]){ 778 log_err("ERROR: UTF8_SET_CHAR_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset); 779 } 780 781 setOffset=offset; 782 U8_SET_CP_START_UNSAFE(input, setOffset); 783 if(setOffset != start_unsafe[i]){ 784 log_err("ERROR: U8_SET_CP_START_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, start_unsafe[i], setOffset); 785 } 786 } 787 788 if (offset != 0) { /* Can't have it go off the end of the array */ 789 setOffset=offset; 790 UTF8_SET_CHAR_LIMIT_UNSAFE(input, setOffset); 791 if(setOffset != limit_unsafe[i]){ 792 log_err("ERROR: UTF8_SET_CHAR_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset); 793 } 794 795 setOffset=offset; 796 U8_SET_CP_LIMIT_UNSAFE(input, setOffset); 797 if(setOffset != limit_unsafe[i]){ 798 log_err("ERROR: U8_SET_CP_LIMIT_UNSAFE failed for offset=%ld. Expected:%ld Got:%ld\n", offset, limit_unsafe[i], setOffset); 799 } 800 } 801 802 i++; 803 } 804 } 805 806 static void TestAppendChar(){ 807 static const uint8_t s[11]={0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}; 808 static const uint32_t test[]={ 809 /* append-position(unsafe), CHAR to be appended */ 810 0, 0x10401, 811 2, 0x0028, 812 2, 0x007f, 813 3, 0xd801, 814 1, 0x20402, 815 8, 0x10401, 816 5, 0xc0, 817 5, 0xc1, 818 5, 0xfd, 819 6, 0x80, 820 6, 0x81, 821 6, 0xbf, 822 7, 0xfe, 823 824 /* append-position(safe), CHAR to be appended */ 825 0, 0x10401, 826 2, 0x0028, 827 3, 0x7f, 828 3, 0xd801, /* illegal for UTF-8 starting with Unicode 3.2 */ 829 1, 0x20402, 830 9, 0x10401, 831 5, 0xc0, 832 5, 0xc1, 833 5, 0xfd, 834 6, 0x80, 835 6, 0x81, 836 6, 0xbf, 837 7, 0xfe, 838 839 }; 840 static const uint16_t movedOffset[]={ 841 /* offset-moved-to(unsafe) */ 842 4, /*for append-pos: 0 , CHAR 0x10401*/ 843 3, 844 3, 845 6, 846 5, 847 12, 848 7, 849 7, 850 7, 851 8, 852 8, 853 8, 854 9, 855 856 /* offset-moved-to(safe) */ 857 4, /*for append-pos: 0, CHAR 0x10401*/ 858 3, 859 4, 860 6, 861 5, 862 11, 863 7, 864 7, 865 7, 866 8, 867 8, 868 8, 869 9, 870 871 }; 872 873 static const uint8_t result[][11]={ 874 /*unsafe*/ 875 {0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 876 {0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 877 {0x61, 0x62, 0x7f, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 878 {0x61, 0x62, 0x63, 0xed, 0xa0, 0x81, 0x67, 0x68, 0x69, 0x6a, 0x00}, 879 {0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 880 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0xF0, 0x90, 0x90}, 881 882 {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x80, 0x68, 0x69, 0x6a, 0x00}, 883 {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x81, 0x68, 0x69, 0x6a, 0x00}, 884 {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0xbd, 0x68, 0x69, 0x6a, 0x00}, 885 886 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x80, 0x69, 0x6a, 0x00}, 887 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x81, 0x69, 0x6a, 0x00}, 888 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0xbf, 0x69, 0x6a, 0x00}, 889 890 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0xc3, 0xbe, 0x6a, 0x00}, 891 /*safe*/ 892 {0xF0, 0x90, 0x90, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 893 {0x61, 0x62, 0x28, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 894 {0x61, 0x62, 0x63, 0x7f, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 895 {0x61, 0x62, 0x63, 0xef, 0xbf, 0xbf, 0x67, 0x68, 0x69, 0x6a, 0x00}, 896 {0x61, 0xF0, 0xa0, 0x90, 0x82, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x00}, 897 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xc2, 0x9f}, /*gets UTF8_ERROR_VALUE_2 which takes 2 bytes 0xc0, 0x9f*/ 898 899 {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x80, 0x68, 0x69, 0x6a, 0x00}, 900 {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0x81, 0x68, 0x69, 0x6a, 0x00}, 901 {0x61, 0x62, 0x63, 0x64, 0x65, 0xc3, 0xbd, 0x68, 0x69, 0x6a, 0x00}, 902 903 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x80, 0x69, 0x6a, 0x00}, 904 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0x81, 0x69, 0x6a, 0x00}, 905 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0xc2, 0xbf, 0x69, 0x6a, 0x00}, 906 907 {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0xc3, 0xbe, 0x6a, 0x00}, 908 909 }; 910 uint16_t i, count=0; 911 uint8_t str[12]; 912 uint32_t offset; 913 /* UChar32 c=0;*/ 914 uint16_t size=LENGTHOF(s); 915 for(i=0; i<LENGTHOF(test); i=(uint16_t)(i+2)){ 916 uprv_memcpy(str, s, size); 917 offset=test[i]; 918 if(count<13){ 919 UTF8_APPEND_CHAR_UNSAFE(str, offset, test[i+1]); 920 if(offset != movedOffset[count]){ 921 log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n", 922 count, movedOffset[count], offset); 923 924 } 925 if(uprv_memcmp(str, result[count], size) !=0){ 926 log_err("ERROR: UTF8_APPEND_CHAR_UNSAFE failed for count=%d. \nExpected:", count); 927 printUChars(result[count], size); 928 log_err("\nGot: "); 929 printUChars(str, size); 930 log_err("\n"); 931 } 932 }else{ 933 UTF8_APPEND_CHAR_SAFE(str, offset, size, test[i+1]); 934 if(offset != movedOffset[count]){ 935 log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n", 936 count, movedOffset[count], offset); 937 938 } 939 if(uprv_memcmp(str, result[count], size) !=0){ 940 log_err("ERROR: UTF8_APPEND_CHAR_SAFE failed for count=%d. \nExpected:", count); 941 printUChars(result[count], size); 942 log_err("\nGot: "); 943 printUChars(str, size); 944 log_err("\n"); 945 } 946 /*call the API instead of MACRO 947 uprv_memcpy(str, s, size); 948 offset=test[i]; 949 c=test[i+1]; 950 if((uint32_t)(c)<=0x7f) { 951 (str)[(offset)++]=(uint8_t)(c); 952 } else { 953 (offset)=utf8_appendCharSafeBody(str, (int32_t)(offset), (int32_t)(size), c); 954 } 955 if(offset != movedOffset[count]){ 956 log_err("ERROR: utf8_appendCharSafeBody() failed to move the offset correctly for count=%d.\nExpectedOffset=%d currentOffset=%d\n", 957 count, movedOffset[count], offset); 958 959 } 960 if(uprv_memcmp(str, result[count], size) !=0){ 961 log_err("ERROR: utf8_appendCharSafeBody() failed for count=%d. \nExpected:", count); 962 printUChars(result[count], size); 963 printf("\nGot: "); 964 printUChars(str, size); 965 printf("\n"); 966 } 967 */ 968 } 969 count++; 970 } 971 972 973 } 974 975 static void TestAppend() { 976 static const UChar32 codePoints[]={ 977 0x61, 0xdf, 0x901, 0x3040, 978 0xac00, 0xd800, 0xdbff, 0xdcde, 979 0xdffd, 0xe000, 0xffff, 0x10000, 980 0x12345, 0xe0021, 0x10ffff, 0x110000, 981 0x234567, 0x7fffffff, -1, -1000, 982 0, 0x400 983 }; 984 static const uint8_t expectUnsafe[]={ 985 0x61, 0xc3, 0x9f, 0xe0, 0xa4, 0x81, 0xe3, 0x81, 0x80, 986 0xea, 0xb0, 0x80, 0xed, 0xa0, 0x80, 0xed, 0xaf, 0xbf, 0xed, 0xb3, 0x9e, 987 0xed, 0xbf, 0xbd, 0xee, 0x80, 0x80, 0xef, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80, 988 0xf0, 0x92, 0x8d, 0x85, 0xf3, 0xa0, 0x80, 0xa1, 0xf4, 0x8f, 0xbf, 0xbf, /* not 0x110000 */ 989 /* none from this line */ 990 0, 0xd0, 0x80 991 }, expectSafe[]={ 992 0x61, 0xc3, 0x9f, 0xe0, 0xa4, 0x81, 0xe3, 0x81, 0x80, 993 0xea, 0xb0, 0x80, /* no surrogates */ 994 /* no surrogates */ 0xee, 0x80, 0x80, 0xef, 0xbf, 0xbf, 0xf0, 0x90, 0x80, 0x80, 995 0xf0, 0x92, 0x8d, 0x85, 0xf3, 0xa0, 0x80, 0xa1, 0xf4, 0x8f, 0xbf, 0xbf, /* not 0x110000 */ 996 /* none from this line */ 997 0, 0xd0, 0x80 998 }; 999 1000 uint8_t buffer[100]; 1001 UChar32 c; 1002 int32_t i, length; 1003 UBool isError, expectIsError, wrongIsError; 1004 1005 length=0; 1006 for(i=0; i<LENGTHOF(codePoints); ++i) { 1007 c=codePoints[i]; 1008 if(c<0 || 0x10ffff<c) { 1009 continue; /* skip non-code points for U8_APPEND_UNSAFE */ 1010 } 1011 1012 U8_APPEND_UNSAFE(buffer, length, c); 1013 } 1014 if(length!=LENGTHOF(expectUnsafe) || 0!=memcmp(buffer, expectUnsafe, length)) { 1015 log_err("U8_APPEND_UNSAFE did not generate the expected output\n"); 1016 } 1017 1018 length=0; 1019 wrongIsError=FALSE; 1020 for(i=0; i<LENGTHOF(codePoints); ++i) { 1021 c=codePoints[i]; 1022 expectIsError= c<0 || 0x10ffff<c || U_IS_SURROGATE(c); 1023 isError=FALSE; 1024 1025 U8_APPEND(buffer, length, LENGTHOF(buffer), c, isError); 1026 wrongIsError|= isError!=expectIsError; 1027 } 1028 if(wrongIsError) { 1029 log_err("U8_APPEND did not set isError correctly\n"); 1030 } 1031 if(length!=LENGTHOF(expectSafe) || 0!=memcmp(buffer, expectSafe, length)) { 1032 log_err("U8_APPEND did not generate the expected output\n"); 1033 } 1034 } 1035 1036 static void 1037 TestSurrogates() { 1038 static const uint8_t b[]={ 1039 0xc3, 0x9f, /* 00DF */ 1040 0xed, 0x9f, 0xbf, /* D7FF */ 1041 0xed, 0xa0, 0x81, /* D801 */ 1042 0xed, 0xbf, 0xbe, /* DFFE */ 1043 0xee, 0x80, 0x80, /* E000 */ 1044 0xf0, 0x97, 0xbf, 0xbe /* 17FFE */ 1045 }; 1046 static const UChar32 cp[]={ 1047 0xdf, 0xd7ff, 0xd801, 0xdffe, 0xe000, 0x17ffe 1048 }; 1049 1050 UChar32 cu, cs, cl; 1051 int32_t i, j, k, iu, is, il, length; 1052 1053 k=0; /* index into cp[] */ 1054 length=LENGTHOF(b); 1055 for(i=0; i<length;) { 1056 j=i; 1057 U8_NEXT_UNSAFE(b, j, cu); 1058 iu=j; 1059 1060 j=i; 1061 U8_NEXT(b, j, length, cs); 1062 is=j; 1063 1064 j=i; 1065 L8_NEXT(b, j, length, cl); 1066 il=j; 1067 1068 if(cu!=cp[k]) { 1069 log_err("U8_NEXT_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]); 1070 } 1071 1072 /* U8_NEXT() returns <0 for surrogate code points */ 1073 if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) { 1074 log_err("U8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu); 1075 } 1076 1077 /* L8_NEXT() returns surrogate code points like U8_NEXT_UNSAFE() */ 1078 if(cl!=cu) { 1079 log_err("L8_NEXT(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu); 1080 } 1081 1082 if(is!=iu || il!=iu) { 1083 log_err("U8_NEXT(b[%ld]) or L8_NEXT(b[%ld]) did not advance the index correctly\n", (long)i, (long)i); 1084 } 1085 1086 ++k; /* next code point */ 1087 i=iu; /* advance by one UTF-8 sequence */ 1088 } 1089 1090 while(i>0) { 1091 --k; /* previous code point */ 1092 1093 j=i; 1094 U8_PREV_UNSAFE(b, j, cu); 1095 iu=j; 1096 1097 j=i; 1098 U8_PREV(b, 0, j, cs); 1099 is=j; 1100 1101 j=i; 1102 L8_PREV(b, 0, j, cl); 1103 il=j; 1104 1105 if(cu!=cp[k]) { 1106 log_err("U8_PREV_UNSAFE(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cu, (long)cp[k]); 1107 } 1108 1109 /* U8_PREV() returns <0 for surrogate code points */ 1110 if(U_IS_SURROGATE(cu) ? cs>=0 : cs!=cu) { 1111 log_err("U8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cs, (long)cu); 1112 } 1113 1114 /* L8_PREV() returns surrogate code points like U8_PREV_UNSAFE() */ 1115 if(cl!=cu) { 1116 log_err("L8_PREV(b[%ld])=U+%04lX != U+%04lX\n", (long)i, (long)cl, (long)cu); 1117 } 1118 1119 if(is!=iu || il !=iu) { 1120 log_err("U8_PREV(b[%ld]) or L8_PREV(b[%ld]) did not advance the index correctly\n", (long)i, (long)i); 1121 } 1122 1123 i=iu; /* go back by one UTF-8 sequence */ 1124 } 1125 } 1126 1127 static void printUChars(const uint8_t *uchars, int16_t len){ 1128 int16_t i=0; 1129 for(i=0; i<len; i++){ 1130 log_err("0x%02x ", *(uchars+i)); 1131 } 1132 } 1133
