Home | History | Annotate | Download | only in common 
      1 /*
      2  ************************************************************************************
      3  * Copyright (C) 2006-2013, International Business Machines Corporation
      4  * and others. All Rights Reserved.
      5  ************************************************************************************
      6  */
      7 
      8 #include "unicode/utypes.h"
      9 
     10 #if !UCONFIG_NO_BREAK_ITERATION
     11 
     12 #include "brkeng.h"
     13 #include "dictbe.h"
     14 #include "unicode/uchar.h"
     15 #include "unicode/uniset.h"
     16 #include "unicode/chariter.h"
     17 #include "unicode/ures.h"
     18 #include "unicode/udata.h"
     19 #include "unicode/putil.h"
     20 #include "unicode/ustring.h"
     21 #include "unicode/uscript.h"
     22 #include "unicode/ucharstrie.h"
     23 #include "unicode/bytestrie.h"
     24 #include "charstr.h"
     25 #include "dictionarydata.h"
     26 #include "uvector.h"
     27 #include "umutex.h"
     28 #include "uresimp.h"
     29 #include "ubrkimpl.h"
     30 
     31 U_NAMESPACE_BEGIN
     32 
     33 /*
     34  ******************************************************************
     35  */
     36 
     37 LanguageBreakEngine::LanguageBreakEngine() {
     38 }
     39 
     40 LanguageBreakEngine::~LanguageBreakEngine() {
     41 }
     42 
     43 /*
     44  ******************************************************************
     45  */
     46 
     47 LanguageBreakFactory::LanguageBreakFactory() {
     48 }
     49 
     50 LanguageBreakFactory::~LanguageBreakFactory() {
     51 }
     52 
     53 /*
     54  ******************************************************************
     55  */
     56 
     57 UnhandledEngine::UnhandledEngine(UErrorCode &/*status*/) {
     58     for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) {
     59         fHandled[i] = 0;
     60     }
     61 }
     62 
     63 UnhandledEngine::~UnhandledEngine() {
     64     for (int32_t i = 0; i < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0])); ++i) {
     65         if (fHandled[i] != 0) {
     66             delete fHandled[i];
     67         }
     68     }
     69 }
     70 
     71 UBool
     72 UnhandledEngine::handles(UChar32 c, int32_t breakType) const {
     73     return (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))
     74         && fHandled[breakType] != 0 && fHandled[breakType]->contains(c));
     75 }
     76 
     77 int32_t
     78 UnhandledEngine::findBreaks( UText *text,
     79                                  int32_t startPos,
     80                                  int32_t endPos,
     81                                  UBool reverse,
     82                                  int32_t breakType,
     83                                  UStack &/*foundBreaks*/ ) const {
     84     if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) {
     85         UChar32 c = utext_current32(text);
     86         if (reverse) {
     87             while((int32_t)utext_getNativeIndex(text) > startPos && fHandled[breakType]->contains(c)) {
     88                 c = utext_previous32(text);
     89             }
     90         }
     91         else {
     92             while((int32_t)utext_getNativeIndex(text) < endPos && fHandled[breakType]->contains(c)) {
     93                 utext_next32(text);            // TODO:  recast loop to work with post-increment operations.
     94                 c = utext_current32(text);
     95             }
     96         }
     97     }
     98     return 0;
     99 }
    100 
    101 void
    102 UnhandledEngine::handleCharacter(UChar32 c, int32_t breakType) {
    103     if (breakType >= 0 && breakType < (int32_t)(sizeof(fHandled)/sizeof(fHandled[0]))) {
    104         if (fHandled[breakType] == 0) {
    105             fHandled[breakType] = new UnicodeSet();
    106             if (fHandled[breakType] == 0) {
    107                 return;
    108             }
    109         }
    110         if (!fHandled[breakType]->contains(c)) {
    111             UErrorCode status = U_ZERO_ERROR;
    112             // Apply the entire script of the character.
    113             int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT);
    114             fHandled[breakType]->applyIntPropertyValue(UCHAR_SCRIPT, script, status);
    115         }
    116     }
    117 }
    118 
    119 /*
    120  ******************************************************************
    121  */
    122 
    123 ICULanguageBreakFactory::ICULanguageBreakFactory(UErrorCode &/*status*/) {
    124     fEngines = 0;
    125 }
    126 
    127 ICULanguageBreakFactory::~ICULanguageBreakFactory() {
    128     if (fEngines != 0) {
    129         delete fEngines;
    130     }
    131 }
    132 
    133 U_NAMESPACE_END
    134 U_CDECL_BEGIN
    135 static void U_CALLCONV _deleteEngine(void *obj) {
    136     delete (const icu::LanguageBreakEngine *) obj;
    137 }
    138 U_CDECL_END
    139 U_NAMESPACE_BEGIN
    140 
    141 const LanguageBreakEngine *
    142 ICULanguageBreakFactory::getEngineFor(UChar32 c, int32_t breakType) {
    143     UBool       needsInit;
    144     int32_t     i;
    145     const LanguageBreakEngine *lbe = NULL;
    146     UErrorCode  status = U_ZERO_ERROR;
    147 
    148     // TODO: The global mutex should not be used.
    149     // The global mutex should only be used for short periods.
    150     // A ICULanguageBreakFactory specific mutex should be used.
    151     umtx_lock(NULL);
    152     needsInit = (UBool)(fEngines == NULL);
    153     if (!needsInit) {
    154         i = fEngines->size();
    155         while (--i >= 0) {
    156             lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
    157             if (lbe != NULL && lbe->handles(c, breakType)) {
    158                 break;
    159             }
    160             lbe = NULL;
    161         }
    162     }
    163     umtx_unlock(NULL);
    164 
    165     if (lbe != NULL) {
    166         return lbe;
    167     }
    168 
    169     if (needsInit) {
    170         UStack  *engines = new UStack(_deleteEngine, NULL, status);
    171         if (U_SUCCESS(status) && engines == NULL) {
    172             status = U_MEMORY_ALLOCATION_ERROR;
    173         }
    174         else if (U_FAILURE(status)) {
    175             delete engines;
    176             engines = NULL;
    177         }
    178         else {
    179             umtx_lock(NULL);
    180             if (fEngines == NULL) {
    181                 fEngines = engines;
    182                 engines = NULL;
    183             }
    184             umtx_unlock(NULL);
    185             delete engines;
    186         }
    187     }
    188 
    189     if (fEngines == NULL) {
    190         return NULL;
    191     }
    192 
    193     // We didn't find an engine the first time through, or there was no
    194     // stack. Create an engine.
    195     const LanguageBreakEngine *newlbe = loadEngineFor(c, breakType);
    196 
    197     // Now get the lock, and see if someone else has created it in the
    198     // meantime
    199     umtx_lock(NULL);
    200     i = fEngines->size();
    201     while (--i >= 0) {
    202         lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
    203         if (lbe != NULL && lbe->handles(c, breakType)) {
    204             break;
    205         }
    206         lbe = NULL;
    207     }
    208     if (lbe == NULL && newlbe != NULL) {
    209         fEngines->push((void *)newlbe, status);
    210         lbe = newlbe;
    211         newlbe = NULL;
    212     }
    213     umtx_unlock(NULL);
    214 
    215     delete newlbe;
    216 
    217     return lbe;
    218 }
    219 
    220 const LanguageBreakEngine *
    221 ICULanguageBreakFactory::loadEngineFor(UChar32 c, int32_t breakType) {
    222     UErrorCode status = U_ZERO_ERROR;
    223     UScriptCode code = uscript_getScript(c, &status);
    224     if (U_SUCCESS(status)) {
    225         DictionaryMatcher *m = loadDictionaryMatcherFor(code, breakType);
    226         if (m != NULL) {
    227             const LanguageBreakEngine *engine = NULL;
    228             switch(code) {
    229             case USCRIPT_THAI:
    230                 engine = new ThaiBreakEngine(m, status);
    231                 break;
    232             case USCRIPT_LAO:
    233                 engine = new LaoBreakEngine(m, status);
    234                 break;
    235             case USCRIPT_KHMER:
    236                 engine = new KhmerBreakEngine(m, status);
    237                 break;
    238 
    239 #if !UCONFIG_NO_NORMALIZATION
    240                 // CJK not available w/o normalization
    241             case USCRIPT_HANGUL:
    242                 engine = new CjkBreakEngine(m, kKorean, status);
    243                 break;
    244 
    245             // use same BreakEngine and dictionary for both Chinese and Japanese
    246             case USCRIPT_HIRAGANA:
    247             case USCRIPT_KATAKANA:
    248             case USCRIPT_HAN:
    249                 engine = new CjkBreakEngine(m, kChineseJapanese, status);
    250                 break;
    251 #if 0
    252             // TODO: Have to get some characters with script=common handled
    253             // by CjkBreakEngine (e.g. U+309B). Simply subjecting
    254             // them to CjkBreakEngine does not work. The engine has to
    255             // special-case them.
    256             case USCRIPT_COMMON:
    257             {
    258                 UBlockCode block = ublock_getCode(code);
    259                 if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
    260                    engine = new CjkBreakEngine(dict, kChineseJapanese, status);
    261                 break;
    262             }
    263 #endif
    264 #endif
    265 
    266             default:
    267                 break;
    268             }
    269             if (engine == NULL) {
    270                 delete m;
    271             }
    272             else if (U_FAILURE(status)) {
    273                 delete engine;
    274                 engine = NULL;
    275             }
    276             return engine;
    277         }
    278     }
    279     return NULL;
    280 }
    281 
    282 DictionaryMatcher *
    283 ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script, int32_t /* brkType */) {
    284     UErrorCode status = U_ZERO_ERROR;
    285     // open root from brkitr tree.
    286     UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
    287     b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);
    288     int32_t dictnlength = 0;
    289     const UChar *dictfname =
    290         ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status);
    291     if (U_FAILURE(status)) {
    292         ures_close(b);
    293         return NULL;
    294     }
    295     CharString dictnbuf;
    296     CharString ext;
    297     const UChar *extStart = u_memrchr(dictfname, 0x002e, dictnlength);  // last dot
    298     if (extStart != NULL) {
    299         int32_t len = (int32_t)(extStart - dictfname);
    300         ext.appendInvariantChars(UnicodeString(FALSE, extStart + 1, dictnlength - len - 1), status);
    301         dictnlength = len;
    302     }
    303     dictnbuf.appendInvariantChars(UnicodeString(FALSE, dictfname, dictnlength), status);
    304     ures_close(b);
    305 
    306     UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status);
    307     if (U_SUCCESS(status)) {
    308         // build trie
    309         const uint8_t *data = (const uint8_t *)udata_getMemory(file);
    310         const int32_t *indexes = (const int32_t *)data;
    311         const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
    312         const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK;
    313         DictionaryMatcher *m = NULL;
    314         if (trieType == DictionaryData::TRIE_TYPE_BYTES) {
    315             const int32_t transform = indexes[DictionaryData::IX_TRANSFORM];
    316             const char *characters = (const char *)(data + offset);
    317             m = new BytesDictionaryMatcher(characters, transform, file);
    318         }
    319         else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) {
    320             const UChar *characters = (const UChar *)(data + offset);
    321             m = new UCharsDictionaryMatcher(characters, file);
    322         }
    323         if (m == NULL) {
    324             // no matcher exists to take ownership - either we are an invalid
    325             // type or memory allocation failed
    326             udata_close(file);
    327         }
    328         return m;
    329     } else if (dictfname != NULL) {
    330         // we don't have a dictionary matcher.
    331         // returning NULL here will cause us to fail to find a dictionary break engine, as expected
    332         status = U_ZERO_ERROR;
    333         return NULL;
    334     }
    335     return NULL;
    336 }
    337 
    338 U_NAMESPACE_END
    339 
    340 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
    341