1 /* 2 ************************************************************************************ 3 * Copyright (C) 2006-2013, International Business Machines Corporation 4 * and others. All Rights Reserved. 5 ************************************************************************************ 6 */ 7 8 #include "unicode/utypes.h" 9 10 #if !UCONFIG_NO_BREAK_ITERATION 11 12 #include "brkeng.h" 13 #include "dictbe.h" 14 #include "unicode/uchar.h" 15 #include "unicode/uniset.h" 16 #include "unicode/chariter.h" 17 #include "unicode/ures.h" 18 #include "unicode/udata.h" 19 #include "unicode/putil.h" 20 #include "unicode/ustring.h" 21 #include "unicode/uscript.h" 22 #include "unicode/ucharstrie.h" 23 #include "unicode/bytestrie.h" 24 #include "charstr.h" 25 #include "dictionarydata.h" 26 #include "uvector.h" 27 #include "umutex.h" 28 #include "uresimp.h" 29 #include "ubrkimpl.h" 30 31 U_NAMESPACE_BEGIN 32 33 /* 34 ****************************************************************** 35 */ 36 37 LanguageBreakEngine::LanguageBreakEngine() { 38 } 39 40 LanguageBreakEngine::~LanguageBreakEngine() { 41 } 42 43 /* 44 ****************************************************************** 45 */ 46 47 LanguageBreakFactory::LanguageBreakFactory() { 48 } 49 50 LanguageBreakFactory::~LanguageBreakFactory() { 51 } 52 53 /* 54 ****************************************************************** 55 */ 56 57 UnhandledEngine::UnhandledEngine(UErrorCode &/*status*/) { 58 for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) { 59 fHandled[i] = 0; 60 } 61 } 62 63 UnhandledEngine::~UnhandledEngine() { 64 for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) { 65 if (fHandled[i] != 0) { 66 delete fHandled[i]; 67 } 68 } 69 } 70 71 UBool 72 UnhandledEngine::handles(UChar32 c, int32_t breakType) const { 73 return (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])) 74 && fHandled[breakType] != 0 && fHandled[breakType]->contains(c)); 75 } 76 77 int32_t 78 UnhandledEngine::findBreaks( UText *text, 79 int32_t startPos, 80 int32_t endPos, 81 UBool reverse, 82 int32_t breakType, 83 UStack &/*foundBreaks*/ ) const { 84 if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) { 85 UChar32 c = utext_current32(text); 86 if (reverse) { 87 while((int32_t)utext_getNativeIndex(text) > startPos && fHandled[breakType]->contains(c)) { 88 c = utext_previous32(text); 89 } 90 } 91 else { 92 while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) { 93 utext_next32(text); // TODO: recast loop to work with post-increment operations. 94 c = utext_current32(text); 95 } 96 } 97 } 98 return 0; 99 } 100 101 void 102 UnhandledEngine::handleCharacter(UChar32 c, int32_t breakType) { 103 if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) { 104 if (fHandled[breakType] == 0) { 105 fHandled[breakType] = new UnicodeSet(); 106 if (fHandled[breakType] == 0) { 107 return; 108 } 109 } 110 if (!fHandled[breakType]->contains(c)) { 111 UErrorCode status = U_ZERO_ERROR; 112 // Apply the entire script of the character. 113 int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT); 114 fHandled[breakType]->applyIntPropertyValue(UCHAR_SCRIPT, script, status); 115 } 116 } 117 } 118 119 /* 120 ****************************************************************** 121 */ 122 123 ICULanguageBreakFactory::ICULanguageBreakFactory(UErrorCode &/*status*/) { 124 fEngines = 0; 125 } 126 127 ICULanguageBreakFactory::~ICULanguageBreakFactory() { 128 if (fEngines != 0) { 129 delete fEngines; 130 } 131 } 132 133 U_NAMESPACE_END 134 U_CDECL_BEGIN 135 static void U_CALLCONV _deleteEngine(void *obj) { 136 delete (const icu::LanguageBreakEngine *) obj; 137 } 138 U_CDECL_END 139 U_NAMESPACE_BEGIN 140 141 const LanguageBreakEngine * 142 ICULanguageBreakFactory::getEngineFor(UChar32 c, int32_t breakType) { 143 UBool needsInit; 144 int32_t i; 145 const LanguageBreakEngine *lbe = NULL; 146 UErrorCode status = U_ZERO_ERROR; 147 148 // TODO: The global mutex should not be used. 149 // The global mutex should only be used for short periods. 150 // A ICULanguageBreakFactory specific mutex should be used. 151 umtx_lock(NULL); 152 needsInit = (UBool)(fEngines == NULL); 153 if (!needsInit) { 154 i = fEngines->size(); 155 while (--i >= 0) { 156 lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i)); 157 if (lbe != NULL && lbe->handles(c, breakType)) { 158 break; 159 } 160 lbe = NULL; 161 } 162 } 163 umtx_unlock(NULL); 164 165 if (lbe != NULL) { 166 return lbe; 167 } 168 169 if (needsInit) { 170 UStack *engines = new UStack(_deleteEngine, NULL, status); 171 if (U_SUCCESS(status) && engines == NULL) { 172 status = U_MEMORY_ALLOCATION_ERROR; 173 } 174 else if (U_FAILURE(status)) { 175 delete engines; 176 engines = NULL; 177 } 178 else { 179 umtx_lock(NULL); 180 if (fEngines == NULL) { 181 fEngines = engines; 182 engines = NULL; 183 } 184 umtx_unlock(NULL); 185 delete engines; 186 } 187 } 188 189 if (fEngines == NULL) { 190 return NULL; 191 } 192 193 // We didn't find an engine the first time through, or there was no 194 // stack. Create an engine. 195 const LanguageBreakEngine *newlbe = loadEngineFor(c, breakType); 196 197 // Now get the lock, and see if someone else has created it in the 198 // meantime 199 umtx_lock(NULL); 200 i = fEngines->size(); 201 while (--i >= 0) { 202 lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i)); 203 if (lbe != NULL && lbe->handles(c, breakType)) { 204 break; 205 } 206 lbe = NULL; 207 } 208 if (lbe == NULL && newlbe != NULL) { 209 fEngines->push((void *)newlbe, status); 210 lbe = newlbe; 211 newlbe = NULL; 212 } 213 umtx_unlock(NULL); 214 215 delete newlbe; 216 217 return lbe; 218 } 219 220 const LanguageBreakEngine * 221 ICULanguageBreakFactory::loadEngineFor(UChar32 c, int32_t breakType) { 222 UErrorCode status = U_ZERO_ERROR; 223 UScriptCode code = uscript_getScript(c, &status); 224 if (U_SUCCESS(status)) { 225 DictionaryMatcher *m = loadDictionaryMatcherFor(code, breakType); 226 if (m != NULL) { 227 const LanguageBreakEngine *engine = NULL; 228 switch(code) { 229 case USCRIPT_THAI: 230 engine = new ThaiBreakEngine(m, status); 231 break; 232 case USCRIPT_LAO: 233 engine = new LaoBreakEngine(m, status); 234 break; 235 case USCRIPT_KHMER: 236 engine = new KhmerBreakEngine(m, status); 237 break; 238 239 #if !UCONFIG_NO_NORMALIZATION 240 // CJK not available w/o normalization 241 case USCRIPT_HANGUL: 242 engine = new CjkBreakEngine(m, kKorean, status); 243 break; 244 245 // use same BreakEngine and dictionary for both Chinese and Japanese 246 case USCRIPT_HIRAGANA: 247 case USCRIPT_KATAKANA: 248 case USCRIPT_HAN: 249 engine = new CjkBreakEngine(m, kChineseJapanese, status); 250 break; 251 #if 0 252 // TODO: Have to get some characters with script=common handled 253 // by CjkBreakEngine (e.g. U+309B). Simply subjecting 254 // them to CjkBreakEngine does not work. The engine has to 255 // special-case them. 256 case USCRIPT_COMMON: 257 { 258 UBlockCode block = ublock_getCode(code); 259 if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA) 260 engine = new CjkBreakEngine(dict, kChineseJapanese, status); 261 break; 262 } 263 #endif 264 #endif 265 266 default: 267 break; 268 } 269 if (engine == NULL) { 270 delete m; 271 } 272 else if (U_FAILURE(status)) { 273 delete engine; 274 engine = NULL; 275 } 276 return engine; 277 } 278 } 279 return NULL; 280 } 281 282 DictionaryMatcher * 283 ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script, int32_t /* brkType */) { 284 UErrorCode status = U_ZERO_ERROR; 285 // open root from brkitr tree. 286 UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status); 287 b = ures_getByKeyWithFallback(b, "dictionaries", b, &status); 288 int32_t dictnlength = 0; 289 const UChar *dictfname = 290 ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status); 291 if (U_FAILURE(status)) { 292 ures_close(b); 293 return NULL; 294 } 295 CharString dictnbuf; 296 CharString ext; 297 const UChar *extStart = u_memrchr(dictfname, 0x002e, dictnlength); // last dot 298 if (extStart != NULL) { 299 int32_t len = (int32_t)(extStart - dictfname); 300 ext.appendInvariantChars(UnicodeString(FALSE, extStart + 1, dictnlength - len - 1), status); 301 dictnlength = len; 302 } 303 dictnbuf.appendInvariantChars(UnicodeString(FALSE, dictfname, dictnlength), status); 304 ures_close(b); 305 306 UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status); 307 if (U_SUCCESS(status)) { 308 // build trie 309 const uint8_t *data = (const uint8_t *)udata_getMemory(file); 310 const int32_t *indexes = (const int32_t *)data; 311 const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET]; 312 const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK; 313 DictionaryMatcher *m = NULL; 314 if (trieType == DictionaryData::TRIE_TYPE_BYTES) { 315 const int32_t transform = indexes[DictionaryData::IX_TRANSFORM]; 316 const char *characters = (const char *)(data + offset); 317 m = new BytesDictionaryMatcher(characters, transform, file); 318 } 319 else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) { 320 const UChar *characters = (const UChar *)(data + offset); 321 m = new UCharsDictionaryMatcher(characters, file); 322 } 323 if (m == NULL) { 324 // no matcher exists to take ownership - either we are an invalid 325 // type or memory allocation failed 326 udata_close(file); 327 } 328 return m; 329 } else if (dictfname != NULL) { 330 // we don't have a dictionary matcher. 331 // returning NULL here will cause us to fail to find a dictionary break engine, as expected 332 status = U_ZERO_ERROR; 333 return NULL; 334 } 335 return NULL; 336 } 337 338 U_NAMESPACE_END 339 340 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ 341
