1 /* 2 ********************************************************************** 3 * Copyright (C) 2001-2010, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 * Date Name Description 7 * 07/03/01 aliu Creation. 8 ********************************************************************** 9 */ 10 11 #include "unicode/utypes.h" 12 13 #if !UCONFIG_NO_TRANSLITERATION 14 15 #include "unicode/normalizer2.h" 16 #include "cstring.h" 17 #include "nortrans.h" 18 19 U_NAMESPACE_BEGIN 20 21 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(NormalizationTransliterator) 22 23 static inline Transliterator::Token cstrToken(const char *s) { 24 return Transliterator::pointerToken((void *)s); 25 } 26 27 /** 28 * System registration hook. 29 */ 30 void NormalizationTransliterator::registerIDs() { 31 // In the Token, the byte after the NUL is the UNormalization2Mode. 32 Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFC"), 33 _create, cstrToken("nfc\0\0")); 34 Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFKC"), 35 _create, cstrToken("nfkc\0\0")); 36 Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFD"), 37 _create, cstrToken("nfc\0\1")); 38 Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-NFKD"), 39 _create, cstrToken("nfkc\0\1")); 40 Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-FCD"), 41 _create, cstrToken("nfc\0\2")); 42 Transliterator::_registerFactory(UNICODE_STRING_SIMPLE("Any-FCC"), 43 _create, cstrToken("nfc\0\3")); 44 Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("NFC"), 45 UNICODE_STRING_SIMPLE("NFD"), TRUE); 46 Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("NFKC"), 47 UNICODE_STRING_SIMPLE("NFKD"), TRUE); 48 Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("FCC"), 49 UNICODE_STRING_SIMPLE("NFD"), FALSE); 50 Transliterator::_registerSpecialInverse(UNICODE_STRING_SIMPLE("FCD"), 51 UNICODE_STRING_SIMPLE("FCD"), FALSE); 52 } 53 54 /** 55 * Factory methods 56 */ 57 Transliterator* NormalizationTransliterator::_create(const UnicodeString& ID, 58 Token context) { 59 const char *name = (const char *)context.pointer; 60 UNormalization2Mode mode = (UNormalization2Mode)uprv_strchr(name, 0)[1]; 61 UErrorCode errorCode = U_ZERO_ERROR; 62 const Normalizer2 *norm2 = Normalizer2::getInstance(NULL, name, mode, errorCode); 63 if(U_SUCCESS(errorCode)) { 64 return new NormalizationTransliterator(ID, *norm2); 65 } else { 66 return NULL; 67 } 68 } 69 70 /** 71 * Constructs a transliterator. 72 */ 73 NormalizationTransliterator::NormalizationTransliterator(const UnicodeString& id, 74 const Normalizer2 &norm2) : 75 Transliterator(id, 0), fNorm2(norm2) {} 76 77 /** 78 * Destructor. 79 */ 80 NormalizationTransliterator::~NormalizationTransliterator() { 81 } 82 83 /** 84 * Copy constructor. 85 */ 86 NormalizationTransliterator::NormalizationTransliterator(const NormalizationTransliterator& o) : 87 Transliterator(o), fNorm2(o.fNorm2) {} 88 89 /** 90 * Transliterator API. 91 */ 92 Transliterator* NormalizationTransliterator::clone(void) const { 93 return new NormalizationTransliterator(*this); 94 } 95 96 /** 97 * Implements {@link Transliterator#handleTransliterate}. 98 */ 99 void NormalizationTransliterator::handleTransliterate(Replaceable& text, UTransPosition& offsets, 100 UBool isIncremental) const { 101 // start and limit of the input range 102 int32_t start = offsets.start; 103 int32_t limit = offsets.limit; 104 if(start >= limit) { 105 return; 106 } 107 108 /* 109 * Normalize as short chunks at a time as possible even in 110 * bulk mode, so that styled text is minimally disrupted. 111 * In incremental mode, a chunk that ends with offsets.limit 112 * must not be normalized. 113 * 114 * If it was known that the input text is not styled, then 115 * a bulk mode normalization could look like this: 116 117 UnicodeString input, normalized; 118 int32_t length = limit - start; 119 _Replaceable_extractBetween(text, start, limit, input.getBuffer(length)); 120 input.releaseBuffer(length); 121 122 UErrorCode status = U_ZERO_ERROR; 123 fNorm2.normalize(input, normalized, status); 124 125 text.handleReplaceBetween(start, limit, normalized); 126 127 int32_t delta = normalized.length() - length; 128 offsets.contextLimit += delta; 129 offsets.limit += delta; 130 offsets.start = limit + delta; 131 132 */ 133 UErrorCode errorCode = U_ZERO_ERROR; 134 UnicodeString segment; 135 UnicodeString normalized; 136 UChar32 c = text.char32At(start); 137 do { 138 int32_t prev = start; 139 // Skip at least one character so we make progress. 140 // c holds the character at start. 141 segment.remove(); 142 do { 143 segment.append(c); 144 start += U16_LENGTH(c); 145 } while(start < limit && !fNorm2.hasBoundaryBefore(c = text.char32At(start))); 146 if(start == limit && isIncremental && !fNorm2.hasBoundaryAfter(c)) { 147 // stop in incremental mode when we reach the input limit 148 // in case there are additional characters that could change the 149 // normalization result 150 start=prev; 151 break; 152 } 153 fNorm2.normalize(segment, normalized, errorCode); 154 if(U_FAILURE(errorCode)) { 155 break; 156 } 157 if(segment != normalized) { 158 // replace the input chunk with its normalized form 159 text.handleReplaceBetween(prev, start, normalized); 160 161 // update all necessary indexes accordingly 162 int32_t delta = normalized.length() - (start - prev); 163 start += delta; 164 limit += delta; 165 } 166 } while(start < limit); 167 168 offsets.start = start; 169 offsets.contextLimit += limit - offsets.limit; 170 offsets.limit = limit; 171 } 172 173 U_NAMESPACE_END 174 175 #endif /* #if !UCONFIG_NO_TRANSLITERATION */ 176
