1 /* 2 ********************************************************************** 3 * Copyright (C) 2005-2012, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 */ 7 8 #include "unicode/utypes.h" 9 10 #if !UCONFIG_NO_CONVERSION 11 12 #include "unicode/ucsdet.h" 13 14 #include "csdetect.h" 15 #include "csmatch.h" 16 #include "uenumimp.h" 17 18 #include "cmemory.h" 19 #include "cstring.h" 20 #include "umutex.h" 21 #include "ucln_in.h" 22 #include "uarrsort.h" 23 #include "inputext.h" 24 #include "csrsbcs.h" 25 #include "csrmbcs.h" 26 #include "csrutf8.h" 27 #include "csrucode.h" 28 #include "csr2022.h" 29 30 #define ARRAY_SIZE(array) (sizeof array / sizeof array[0]) 31 32 #define NEW_ARRAY(type,count) (type *) uprv_malloc((count) * sizeof(type)) 33 #define DELETE_ARRAY(array) uprv_free((void *) (array)) 34 35 U_CDECL_BEGIN 36 static icu::CharsetRecognizer **fCSRecognizers = NULL; 37 38 static int32_t fCSRecognizers_size = 0; 39 40 static UBool U_CALLCONV csdet_cleanup(void) 41 { 42 if (fCSRecognizers != NULL) { 43 for(int32_t r = 0; r < fCSRecognizers_size; r += 1) { 44 delete fCSRecognizers[r]; 45 fCSRecognizers[r] = NULL; 46 } 47 48 DELETE_ARRAY(fCSRecognizers); 49 fCSRecognizers = NULL; 50 fCSRecognizers_size = 0; 51 } 52 53 return TRUE; 54 } 55 56 static int32_t U_CALLCONV 57 charsetMatchComparator(const void * /*context*/, const void *left, const void *right) 58 { 59 U_NAMESPACE_USE 60 61 const CharsetMatch **csm_l = (const CharsetMatch **) left; 62 const CharsetMatch **csm_r = (const CharsetMatch **) right; 63 64 // NOTE: compare is backwards to sort from highest to lowest. 65 return (*csm_r)->getConfidence() - (*csm_l)->getConfidence(); 66 } 67 68 U_CDECL_END 69 70 U_NAMESPACE_BEGIN 71 72 void CharsetDetector::setRecognizers(UErrorCode &status) 73 { 74 UBool needsInit; 75 CharsetRecognizer **recognizers; 76 77 if (U_FAILURE(status)) { 78 return; 79 } 80 81 UMTX_CHECK(NULL, (UBool) (fCSRecognizers == NULL), needsInit); 82 83 if (needsInit) { 84 CharsetRecognizer *tempArray[] = { 85 new CharsetRecog_UTF8(), 86 87 new CharsetRecog_UTF_16_BE(), 88 new CharsetRecog_UTF_16_LE(), 89 new CharsetRecog_UTF_32_BE(), 90 new CharsetRecog_UTF_32_LE(), 91 92 new CharsetRecog_8859_1(), 93 new CharsetRecog_8859_2(), 94 new CharsetRecog_8859_5_ru(), 95 new CharsetRecog_8859_6_ar(), 96 new CharsetRecog_8859_7_el(), 97 new CharsetRecog_8859_8_I_he(), 98 new CharsetRecog_8859_8_he(), 99 new CharsetRecog_windows_1251(), 100 new CharsetRecog_windows_1256(), 101 new CharsetRecog_KOI8_R(), 102 new CharsetRecog_8859_9_tr(), 103 new CharsetRecog_sjis(), 104 new CharsetRecog_gb_18030(), 105 new CharsetRecog_euc_jp(), 106 new CharsetRecog_euc_kr(), 107 new CharsetRecog_big5(), 108 109 new CharsetRecog_2022JP(), 110 new CharsetRecog_2022KR(), 111 new CharsetRecog_2022CN(), 112 113 new CharsetRecog_IBM424_he_rtl(), 114 new CharsetRecog_IBM424_he_ltr(), 115 new CharsetRecog_IBM420_ar_rtl(), 116 new CharsetRecog_IBM420_ar_ltr() 117 }; 118 int32_t rCount = ARRAY_SIZE(tempArray); 119 int32_t r; 120 121 recognizers = NEW_ARRAY(CharsetRecognizer *, rCount); 122 123 if (recognizers == NULL) { 124 status = U_MEMORY_ALLOCATION_ERROR; 125 return; 126 } else { 127 for (r = 0; r < rCount; r += 1) { 128 recognizers[r] = tempArray[r]; 129 130 if (recognizers[r] == NULL) { 131 status = U_MEMORY_ALLOCATION_ERROR; 132 break; 133 } 134 } 135 } 136 137 if (U_SUCCESS(status)) { 138 umtx_lock(NULL); 139 if (fCSRecognizers == NULL) { 140 fCSRecognizers_size = rCount; 141 fCSRecognizers = recognizers; 142 } 143 umtx_unlock(NULL); 144 } 145 146 if (fCSRecognizers != recognizers) { 147 for (r = 0; r < rCount; r += 1) { 148 delete recognizers[r]; 149 recognizers[r] = NULL; 150 } 151 152 DELETE_ARRAY(recognizers); 153 } 154 155 recognizers = NULL; 156 ucln_i18n_registerCleanup(UCLN_I18N_CSDET, csdet_cleanup); 157 } 158 } 159 160 CharsetDetector::CharsetDetector(UErrorCode &status) 161 : textIn(new InputText(status)), resultArray(NULL), 162 resultCount(0), fStripTags(FALSE), fFreshTextSet(FALSE) 163 { 164 if (U_FAILURE(status)) { 165 return; 166 } 167 168 setRecognizers(status); 169 170 if (U_FAILURE(status)) { 171 return; 172 } 173 174 resultArray = (CharsetMatch **)uprv_malloc(sizeof(CharsetMatch *)*fCSRecognizers_size); 175 176 if (resultArray == NULL) { 177 status = U_MEMORY_ALLOCATION_ERROR; 178 return; 179 } 180 181 for(int32_t i = 0; i < fCSRecognizers_size; i += 1) { 182 resultArray[i] = new CharsetMatch(); 183 184 if (resultArray[i] == NULL) { 185 status = U_MEMORY_ALLOCATION_ERROR; 186 break; 187 } 188 } 189 } 190 191 CharsetDetector::~CharsetDetector() 192 { 193 delete textIn; 194 195 for(int32_t i = 0; i < fCSRecognizers_size; i += 1) { 196 delete resultArray[i]; 197 } 198 199 uprv_free(resultArray); 200 } 201 202 void CharsetDetector::setText(const char *in, int32_t len) 203 { 204 textIn->setText(in, len); 205 fFreshTextSet = TRUE; 206 } 207 208 UBool CharsetDetector::setStripTagsFlag(UBool flag) 209 { 210 UBool temp = fStripTags; 211 fStripTags = flag; 212 fFreshTextSet = TRUE; 213 return temp; 214 } 215 216 UBool CharsetDetector::getStripTagsFlag() const 217 { 218 return fStripTags; 219 } 220 221 void CharsetDetector::setDeclaredEncoding(const char *encoding, int32_t len) const 222 { 223 textIn->setDeclaredEncoding(encoding,len); 224 } 225 226 int32_t CharsetDetector::getDetectableCount() 227 { 228 UErrorCode status = U_ZERO_ERROR; 229 230 setRecognizers(status); 231 232 return fCSRecognizers_size; 233 } 234 235 const CharsetMatch *CharsetDetector::detect(UErrorCode &status) 236 { 237 int32_t maxMatchesFound = 0; 238 239 detectAll(maxMatchesFound, status); 240 241 if(maxMatchesFound > 0) { 242 return resultArray[0]; 243 } else { 244 return NULL; 245 } 246 } 247 248 const CharsetMatch * const *CharsetDetector::detectAll(int32_t &maxMatchesFound, UErrorCode &status) 249 { 250 if(!textIn->isSet()) { 251 status = U_MISSING_RESOURCE_ERROR;// TODO: Need to set proper status code for input text not set 252 253 return NULL; 254 } else if (fFreshTextSet) { 255 CharsetRecognizer *csr; 256 int32_t i; 257 258 textIn->MungeInput(fStripTags); 259 260 // Iterate over all possible charsets, remember all that 261 // give a match quality > 0. 262 resultCount = 0; 263 for (i = 0; i < fCSRecognizers_size; i += 1) { 264 csr = fCSRecognizers[i]; 265 if (csr->match(textIn, resultArray[resultCount])) { 266 resultCount++; 267 } 268 } 269 270 if (resultCount > 1) { 271 uprv_sortArray(resultArray, resultCount, sizeof resultArray[0], charsetMatchComparator, NULL, TRUE, &status); 272 } 273 fFreshTextSet = FALSE; 274 } 275 276 maxMatchesFound = resultCount; 277 278 return resultArray; 279 } 280 281 /*const char *CharsetDetector::getCharsetName(int32_t index, UErrorCode &status) const 282 { 283 if( index > fCSRecognizers_size-1 || index < 0) { 284 status = U_INDEX_OUTOFBOUNDS_ERROR; 285 286 return 0; 287 } else { 288 return fCSRecognizers[index]->getName(); 289 } 290 }*/ 291 292 U_NAMESPACE_END 293 294 U_CDECL_BEGIN 295 typedef struct { 296 int32_t currIndex; 297 } Context; 298 299 300 301 static void U_CALLCONV 302 enumClose(UEnumeration *en) { 303 if(en->context != NULL) { 304 DELETE_ARRAY(en->context); 305 } 306 307 DELETE_ARRAY(en); 308 } 309 310 static int32_t U_CALLCONV 311 enumCount(UEnumeration *, UErrorCode *) { 312 return fCSRecognizers_size; 313 } 314 315 static const char* U_CALLCONV 316 enumNext(UEnumeration *en, int32_t *resultLength, UErrorCode * /*status*/) { 317 if(((Context *)en->context)->currIndex >= fCSRecognizers_size) { 318 if(resultLength != NULL) { 319 *resultLength = 0; 320 } 321 return NULL; 322 } 323 const char *currName = fCSRecognizers[((Context *)en->context)->currIndex]->getName(); 324 if(resultLength != NULL) { 325 *resultLength = (int32_t)uprv_strlen(currName); 326 } 327 ((Context *)en->context)->currIndex++; 328 329 return currName; 330 } 331 332 static void U_CALLCONV 333 enumReset(UEnumeration *en, UErrorCode *) { 334 ((Context *)en->context)->currIndex = 0; 335 } 336 337 static const UEnumeration gCSDetEnumeration = { 338 NULL, 339 NULL, 340 enumClose, 341 enumCount, 342 uenum_unextDefault, 343 enumNext, 344 enumReset 345 }; 346 347 U_CAPI UEnumeration * U_EXPORT2 348 ucsdet_getAllDetectableCharsets(const UCharsetDetector * /*ucsd*/, UErrorCode *status) 349 { 350 U_NAMESPACE_USE 351 352 if(U_FAILURE(*status)) { 353 return 0; 354 } 355 356 /* Initialize recognized charsets. */ 357 CharsetDetector::getDetectableCount(); 358 359 UEnumeration *en = NEW_ARRAY(UEnumeration, 1); 360 memcpy(en, &gCSDetEnumeration, sizeof(UEnumeration)); 361 en->context = (void*)NEW_ARRAY(Context, 1); 362 uprv_memset(en->context, 0, sizeof(Context)); 363 return en; 364 } 365 U_CDECL_END 366 367 #endif 368 369
