1 // 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * Copyright (C) 2012-2014, International Business Machines 6 * Corporation and others. All Rights Reserved. 7 ******************************************************************************* 8 * uitercollationiterator.cpp 9 * 10 * created on: 2012sep23 (from utf16collationiterator.cpp) 11 * created by: Markus W. Scherer 12 */ 13 14 #include "unicode/utypes.h" 15 16 #if !UCONFIG_NO_COLLATION 17 18 #include "unicode/uiter.h" 19 #include "charstr.h" 20 #include "cmemory.h" 21 #include "collation.h" 22 #include "collationdata.h" 23 #include "collationfcd.h" 24 #include "collationiterator.h" 25 #include "normalizer2impl.h" 26 #include "uassert.h" 27 #include "uitercollationiterator.h" 28 29 U_NAMESPACE_BEGIN 30 31 UIterCollationIterator::~UIterCollationIterator() {} 32 33 void 34 UIterCollationIterator::resetToOffset(int32_t newOffset) { 35 reset(); 36 iter.move(&iter, newOffset, UITER_START); 37 } 38 39 int32_t 40 UIterCollationIterator::getOffset() const { 41 return iter.getIndex(&iter, UITER_CURRENT); 42 } 43 44 uint32_t 45 UIterCollationIterator::handleNextCE32(UChar32 &c, UErrorCode & /*errorCode*/) { 46 c = iter.next(&iter); 47 if(c < 0) { 48 return Collation::FALLBACK_CE32; 49 } 50 return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c); 51 } 52 53 UChar 54 UIterCollationIterator::handleGetTrailSurrogate() { 55 UChar32 trail = iter.next(&iter); 56 if(!U16_IS_TRAIL(trail) && trail >= 0) { iter.previous(&iter); } 57 return (UChar)trail; 58 } 59 60 UChar32 61 UIterCollationIterator::nextCodePoint(UErrorCode & /*errorCode*/) { 62 return uiter_next32(&iter); 63 } 64 65 UChar32 66 UIterCollationIterator::previousCodePoint(UErrorCode & /*errorCode*/) { 67 return uiter_previous32(&iter); 68 } 69 70 void 71 UIterCollationIterator::forwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { 72 while(num > 0 && (uiter_next32(&iter)) >= 0) { 73 --num; 74 } 75 } 76 77 void 78 UIterCollationIterator::backwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { 79 while(num > 0 && (uiter_previous32(&iter)) >= 0) { 80 --num; 81 } 82 } 83 84 // FCDUIterCollationIterator ----------------------------------------------- *** 85 86 FCDUIterCollationIterator::~FCDUIterCollationIterator() {} 87 88 void 89 FCDUIterCollationIterator::resetToOffset(int32_t newOffset) { 90 UIterCollationIterator::resetToOffset(newOffset); 91 start = newOffset; 92 state = ITER_CHECK_FWD; 93 } 94 95 int32_t 96 FCDUIterCollationIterator::getOffset() const { 97 if(state <= ITER_CHECK_BWD) { 98 return iter.getIndex(&iter, UITER_CURRENT); 99 } else if(state == ITER_IN_FCD_SEGMENT) { 100 return pos; 101 } else if(pos == 0) { 102 return start; 103 } else { 104 return limit; 105 } 106 } 107 108 uint32_t 109 FCDUIterCollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) { 110 for(;;) { 111 if(state == ITER_CHECK_FWD) { 112 c = iter.next(&iter); 113 if(c < 0) { 114 return Collation::FALLBACK_CE32; 115 } 116 if(CollationFCD::hasTccc(c)) { 117 if(CollationFCD::maybeTibetanCompositeVowel(c) || 118 CollationFCD::hasLccc(iter.current(&iter))) { 119 iter.previous(&iter); 120 if(!nextSegment(errorCode)) { 121 c = U_SENTINEL; 122 return Collation::FALLBACK_CE32; 123 } 124 continue; 125 } 126 } 127 break; 128 } else if(state == ITER_IN_FCD_SEGMENT && pos != limit) { 129 c = iter.next(&iter); 130 ++pos; 131 U_ASSERT(c >= 0); 132 break; 133 } else if(state >= IN_NORM_ITER_AT_LIMIT && pos != normalized.length()) { 134 c = normalized[pos++]; 135 break; 136 } else { 137 switchToForward(); 138 } 139 } 140 return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c); 141 } 142 143 UChar 144 FCDUIterCollationIterator::handleGetTrailSurrogate() { 145 if(state <= ITER_IN_FCD_SEGMENT) { 146 UChar32 trail = iter.next(&iter); 147 if(U16_IS_TRAIL(trail)) { 148 if(state == ITER_IN_FCD_SEGMENT) { ++pos; } 149 } else if(trail >= 0) { 150 iter.previous(&iter); 151 } 152 return (UChar)trail; 153 } else { 154 U_ASSERT(pos < normalized.length()); 155 UChar trail; 156 if(U16_IS_TRAIL(trail = normalized[pos])) { ++pos; } 157 return trail; 158 } 159 } 160 161 UChar32 162 FCDUIterCollationIterator::nextCodePoint(UErrorCode &errorCode) { 163 UChar32 c; 164 for(;;) { 165 if(state == ITER_CHECK_FWD) { 166 c = iter.next(&iter); 167 if(c < 0) { 168 return c; 169 } 170 if(CollationFCD::hasTccc(c)) { 171 if(CollationFCD::maybeTibetanCompositeVowel(c) || 172 CollationFCD::hasLccc(iter.current(&iter))) { 173 iter.previous(&iter); 174 if(!nextSegment(errorCode)) { 175 return U_SENTINEL; 176 } 177 continue; 178 } 179 } 180 if(U16_IS_LEAD(c)) { 181 UChar32 trail = iter.next(&iter); 182 if(U16_IS_TRAIL(trail)) { 183 return U16_GET_SUPPLEMENTARY(c, trail); 184 } else if(trail >= 0) { 185 iter.previous(&iter); 186 } 187 } 188 return c; 189 } else if(state == ITER_IN_FCD_SEGMENT && pos != limit) { 190 c = uiter_next32(&iter); 191 pos += U16_LENGTH(c); 192 U_ASSERT(c >= 0); 193 return c; 194 } else if(state >= IN_NORM_ITER_AT_LIMIT && pos != normalized.length()) { 195 c = normalized.char32At(pos); 196 pos += U16_LENGTH(c); 197 return c; 198 } else { 199 switchToForward(); 200 } 201 } 202 } 203 204 UChar32 205 FCDUIterCollationIterator::previousCodePoint(UErrorCode &errorCode) { 206 UChar32 c; 207 for(;;) { 208 if(state == ITER_CHECK_BWD) { 209 c = iter.previous(&iter); 210 if(c < 0) { 211 start = pos = 0; 212 state = ITER_IN_FCD_SEGMENT; 213 return U_SENTINEL; 214 } 215 if(CollationFCD::hasLccc(c)) { 216 UChar32 prev = U_SENTINEL; 217 if(CollationFCD::maybeTibetanCompositeVowel(c) || 218 CollationFCD::hasTccc(prev = iter.previous(&iter))) { 219 iter.next(&iter); 220 if(prev >= 0) { 221 iter.next(&iter); 222 } 223 if(!previousSegment(errorCode)) { 224 return U_SENTINEL; 225 } 226 continue; 227 } 228 // hasLccc(trail)=true for all trail surrogates 229 if(U16_IS_TRAIL(c)) { 230 if(prev < 0) { 231 prev = iter.previous(&iter); 232 } 233 if(U16_IS_LEAD(prev)) { 234 return U16_GET_SUPPLEMENTARY(prev, c); 235 } 236 } 237 if(prev >= 0) { 238 iter.next(&iter); 239 } 240 } 241 return c; 242 } else if(state == ITER_IN_FCD_SEGMENT && pos != start) { 243 c = uiter_previous32(&iter); 244 pos -= U16_LENGTH(c); 245 U_ASSERT(c >= 0); 246 return c; 247 } else if(state >= IN_NORM_ITER_AT_LIMIT && pos != 0) { 248 c = normalized.char32At(pos - 1); 249 pos -= U16_LENGTH(c); 250 return c; 251 } else { 252 switchToBackward(); 253 } 254 } 255 } 256 257 void 258 FCDUIterCollationIterator::forwardNumCodePoints(int32_t num, UErrorCode &errorCode) { 259 // Specify the class to avoid a virtual-function indirection. 260 // In Java, we would declare this class final. 261 while(num > 0 && FCDUIterCollationIterator::nextCodePoint(errorCode) >= 0) { 262 --num; 263 } 264 } 265 266 void 267 FCDUIterCollationIterator::backwardNumCodePoints(int32_t num, UErrorCode &errorCode) { 268 // Specify the class to avoid a virtual-function indirection. 269 // In Java, we would declare this class final. 270 while(num > 0 && FCDUIterCollationIterator::previousCodePoint(errorCode) >= 0) { 271 --num; 272 } 273 } 274 275 void 276 FCDUIterCollationIterator::switchToForward() { 277 U_ASSERT(state == ITER_CHECK_BWD || 278 (state == ITER_IN_FCD_SEGMENT && pos == limit) || 279 (state >= IN_NORM_ITER_AT_LIMIT && pos == normalized.length())); 280 if(state == ITER_CHECK_BWD) { 281 // Turn around from backward checking. 282 start = pos = iter.getIndex(&iter, UITER_CURRENT); 283 if(pos == limit) { 284 state = ITER_CHECK_FWD; // Check forward. 285 } else { // pos < limit 286 state = ITER_IN_FCD_SEGMENT; // Stay in FCD segment. 287 } 288 } else { 289 // Reached the end of the FCD segment. 290 if(state == ITER_IN_FCD_SEGMENT) { 291 // The input text segment is FCD, extend it forward. 292 } else { 293 // The input text segment needed to be normalized. 294 // Switch to checking forward from it. 295 if(state == IN_NORM_ITER_AT_START) { 296 iter.move(&iter, limit - start, UITER_CURRENT); 297 } 298 start = limit; 299 } 300 state = ITER_CHECK_FWD; 301 } 302 } 303 304 UBool 305 FCDUIterCollationIterator::nextSegment(UErrorCode &errorCode) { 306 if(U_FAILURE(errorCode)) { return FALSE; } 307 U_ASSERT(state == ITER_CHECK_FWD); 308 // The input text [start..(iter index)[ passes the FCD check. 309 pos = iter.getIndex(&iter, UITER_CURRENT); 310 // Collect the characters being checked, in case they need to be normalized. 311 UnicodeString s; 312 uint8_t prevCC = 0; 313 for(;;) { 314 // Fetch the next character and its fcd16 value. 315 UChar32 c = uiter_next32(&iter); 316 if(c < 0) { break; } 317 uint16_t fcd16 = nfcImpl.getFCD16(c); 318 uint8_t leadCC = (uint8_t)(fcd16 >> 8); 319 if(leadCC == 0 && !s.isEmpty()) { 320 // FCD boundary before this character. 321 uiter_previous32(&iter); 322 break; 323 } 324 s.append(c); 325 if(leadCC != 0 && (prevCC > leadCC || CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) { 326 // Fails FCD check. Find the next FCD boundary and normalize. 327 for(;;) { 328 c = uiter_next32(&iter); 329 if(c < 0) { break; } 330 if(nfcImpl.getFCD16(c) <= 0xff) { 331 uiter_previous32(&iter); 332 break; 333 } 334 s.append(c); 335 } 336 if(!normalize(s, errorCode)) { return FALSE; } 337 start = pos; 338 limit = pos + s.length(); 339 state = IN_NORM_ITER_AT_LIMIT; 340 pos = 0; 341 return TRUE; 342 } 343 prevCC = (uint8_t)fcd16; 344 if(prevCC == 0) { 345 // FCD boundary after the last character. 346 break; 347 } 348 } 349 limit = pos + s.length(); 350 U_ASSERT(pos != limit); 351 iter.move(&iter, -s.length(), UITER_CURRENT); 352 state = ITER_IN_FCD_SEGMENT; 353 return TRUE; 354 } 355 356 void 357 FCDUIterCollationIterator::switchToBackward() { 358 U_ASSERT(state == ITER_CHECK_FWD || 359 (state == ITER_IN_FCD_SEGMENT && pos == start) || 360 (state >= IN_NORM_ITER_AT_LIMIT && pos == 0)); 361 if(state == ITER_CHECK_FWD) { 362 // Turn around from forward checking. 363 limit = pos = iter.getIndex(&iter, UITER_CURRENT); 364 if(pos == start) { 365 state = ITER_CHECK_BWD; // Check backward. 366 } else { // pos > start 367 state = ITER_IN_FCD_SEGMENT; // Stay in FCD segment. 368 } 369 } else { 370 // Reached the start of the FCD segment. 371 if(state == ITER_IN_FCD_SEGMENT) { 372 // The input text segment is FCD, extend it backward. 373 } else { 374 // The input text segment needed to be normalized. 375 // Switch to checking backward from it. 376 if(state == IN_NORM_ITER_AT_LIMIT) { 377 iter.move(&iter, start - limit, UITER_CURRENT); 378 } 379 limit = start; 380 } 381 state = ITER_CHECK_BWD; 382 } 383 } 384 385 UBool 386 FCDUIterCollationIterator::previousSegment(UErrorCode &errorCode) { 387 if(U_FAILURE(errorCode)) { return FALSE; } 388 U_ASSERT(state == ITER_CHECK_BWD); 389 // The input text [(iter index)..limit[ passes the FCD check. 390 pos = iter.getIndex(&iter, UITER_CURRENT); 391 // Collect the characters being checked, in case they need to be normalized. 392 UnicodeString s; 393 uint8_t nextCC = 0; 394 for(;;) { 395 // Fetch the previous character and its fcd16 value. 396 UChar32 c = uiter_previous32(&iter); 397 if(c < 0) { break; } 398 uint16_t fcd16 = nfcImpl.getFCD16(c); 399 uint8_t trailCC = (uint8_t)fcd16; 400 if(trailCC == 0 && !s.isEmpty()) { 401 // FCD boundary after this character. 402 uiter_next32(&iter); 403 break; 404 } 405 s.append(c); 406 if(trailCC != 0 && ((nextCC != 0 && trailCC > nextCC) || 407 CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) { 408 // Fails FCD check. Find the previous FCD boundary and normalize. 409 while(fcd16 > 0xff) { 410 c = uiter_previous32(&iter); 411 if(c < 0) { break; } 412 fcd16 = nfcImpl.getFCD16(c); 413 if(fcd16 == 0) { 414 (void)uiter_next32(&iter); 415 break; 416 } 417 s.append(c); 418 } 419 s.reverse(); 420 if(!normalize(s, errorCode)) { return FALSE; } 421 limit = pos; 422 start = pos - s.length(); 423 state = IN_NORM_ITER_AT_START; 424 pos = normalized.length(); 425 return TRUE; 426 } 427 nextCC = (uint8_t)(fcd16 >> 8); 428 if(nextCC == 0) { 429 // FCD boundary before the following character. 430 break; 431 } 432 } 433 start = pos - s.length(); 434 U_ASSERT(pos != start); 435 iter.move(&iter, s.length(), UITER_CURRENT); 436 state = ITER_IN_FCD_SEGMENT; 437 return TRUE; 438 } 439 440 UBool 441 FCDUIterCollationIterator::normalize(const UnicodeString &s, UErrorCode &errorCode) { 442 // NFD without argument checking. 443 U_ASSERT(U_SUCCESS(errorCode)); 444 nfcImpl.decompose(s, normalized, errorCode); 445 return U_SUCCESS(errorCode); 446 } 447 448 U_NAMESPACE_END 449 450 #endif // !UCONFIG_NO_COLLATION 451
