1 /* 2 ********************************************************************** 3 * Copyright (C) 2005-2010, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 */ 7 8 #include "unicode/utypes.h" 9 10 #include "cmemory.h" 11 12 #if !UCONFIG_NO_CONVERSION 13 #include "csrsbcs.h" 14 15 #define N_GRAM_SIZE 3 16 #define N_GRAM_MASK 0xFFFFFF 17 18 U_NAMESPACE_BEGIN 19 20 NGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap) 21 :byteIndex(0), ngram(0) 22 { 23 ngramList = theNgramList; 24 charMap = theCharMap; 25 26 ngramCount = hitCount = 0; 27 } 28 29 /* 30 * Binary search for value in table, which must have exactly 64 entries. 31 */ 32 33 int32_t NGramParser::search(const int32_t *table, int32_t value) 34 { 35 int32_t index = 0; 36 37 if (table[index + 32] <= value) { 38 index += 32; 39 } 40 41 if (table[index + 16] <= value) { 42 index += 16; 43 } 44 45 if (table[index + 8] <= value) { 46 index += 8; 47 } 48 49 if (table[index + 4] <= value) { 50 index += 4; 51 } 52 53 if (table[index + 2] <= value) { 54 index += 2; 55 } 56 57 if (table[index + 1] <= value) { 58 index += 1; 59 } 60 61 if (table[index] > value) { 62 index -= 1; 63 } 64 65 if (index < 0 || table[index] != value) { 66 return -1; 67 } 68 69 return index; 70 } 71 72 void NGramParser::lookup(int32_t thisNgram) 73 { 74 ngramCount += 1; 75 76 if (search(ngramList, thisNgram) >= 0) { 77 hitCount += 1; 78 } 79 80 } 81 82 void NGramParser::addByte(int32_t b) 83 { 84 ngram = ((ngram << 8) + b) & N_GRAM_MASK; 85 lookup(ngram); 86 } 87 88 int32_t NGramParser::nextByte(InputText *det) 89 { 90 if (byteIndex >= det->fInputLen) { 91 return -1; 92 } 93 94 return det->fInputBytes[byteIndex++]; 95 } 96 97 int32_t NGramParser::parse(InputText *det) 98 { 99 int32_t b; 100 bool ignoreSpace = FALSE; 101 102 while ((b = nextByte(det)) >= 0) { 103 uint8_t mb = charMap[b]; 104 105 // TODO: 0x20 might not be a space in all character sets... 106 if (mb != 0) { 107 if (!(mb == 0x20 && ignoreSpace)) { 108 addByte(mb); 109 } 110 111 ignoreSpace = (mb == 0x20); 112 } 113 } 114 115 // TODO: Is this OK? The buffer could have ended in the middle of a word... 116 addByte(0x20); 117 118 double rawPercent = (double) hitCount / (double) ngramCount; 119 120 // if (rawPercent <= 2.0) { 121 // return 0; 122 // } 123 124 // TODO - This is a bit of a hack to take care of a case 125 // were we were getting a confidence of 135... 126 if (rawPercent > 0.33) { 127 return 98; 128 } 129 130 return (int32_t) (rawPercent * 300.0); 131 } 132 133 CharsetRecog_sbcs::CharsetRecog_sbcs() 134 : haveC1Bytes(FALSE) 135 { 136 // nothing else to do 137 } 138 139 CharsetRecog_sbcs::~CharsetRecog_sbcs() 140 { 141 // nothing to do 142 } 143 144 int32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) 145 { 146 NGramParser parser(ngrams, byteMap); 147 int32_t result; 148 149 haveC1Bytes = det->fC1Bytes; 150 result = parser.parse(det); 151 152 return result; 153 } 154 155 static const uint8_t charMap_8859_1[] = { 156 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 157 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 158 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 159 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 160 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 161 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 162 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 163 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 164 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 165 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 166 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 167 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 168 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 169 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 170 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 171 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 172 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 173 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 174 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 175 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 176 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 177 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, 178 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 179 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, 180 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 181 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 182 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 183 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, 184 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 185 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 186 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 187 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 188 }; 189 190 static const uint8_t charMap_8859_2[] = { 191 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 192 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 193 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 194 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 195 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 196 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 197 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 198 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 199 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 200 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 201 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 202 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 203 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 204 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 205 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 206 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 207 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 208 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 209 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 210 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 211 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20, 212 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, 213 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7, 214 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, 215 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 216 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 217 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 218 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, 219 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 220 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 221 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 222 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, 223 }; 224 225 static const uint8_t charMap_8859_5[] = { 226 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 227 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 228 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 229 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 230 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 231 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 232 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 233 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 234 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 235 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 236 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 237 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 238 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 239 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 240 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 241 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 242 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 243 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 244 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 245 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 246 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 247 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, 248 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 249 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 250 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 251 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 252 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 253 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 254 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 255 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 256 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 257 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, 258 }; 259 260 static const uint8_t charMap_8859_6[] = { 261 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 262 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 263 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 264 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 265 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 266 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 267 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 268 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 269 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 270 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 271 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 272 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 273 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 274 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 275 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 276 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 277 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 278 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 279 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 280 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 281 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 282 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 283 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 284 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 285 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 286 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 287 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 288 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20, 289 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 290 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20, 291 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 292 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 293 }; 294 295 static const uint8_t charMap_8859_7[] = { 296 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 297 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 298 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 299 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 300 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 301 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 302 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 303 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 304 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 305 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 306 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 307 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 308 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 309 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 310 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 311 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 312 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 313 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 314 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 315 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 316 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20, 317 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 318 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20, 319 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE, 320 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 321 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 322 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 323 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF, 324 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 325 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 326 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 327 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, 328 }; 329 330 static const uint8_t charMap_8859_8[] = { 331 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 332 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 333 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 334 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 335 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 336 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 337 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 338 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 339 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 340 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 341 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 342 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 343 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 344 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 345 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 346 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 347 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 348 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 349 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 350 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 351 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 352 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 353 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 354 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 355 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 356 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 357 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 358 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 359 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 360 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 361 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 362 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20, 363 }; 364 365 static const uint8_t charMap_8859_9[] = { 366 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 367 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 368 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 369 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 370 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 371 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 372 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 373 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 374 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 375 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 376 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 377 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 378 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 379 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 380 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 381 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 382 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 383 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 384 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 385 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 386 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 387 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, 388 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 389 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, 390 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 391 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 392 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 393 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF, 394 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 395 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 396 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 397 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 398 }; 399 400 static const int32_t ngrams_windows_1251[] = { 401 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE, 402 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED, 403 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, 404 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520, 405 }; 406 407 static const uint8_t charMap_windows_1251[] = { 408 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 409 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 410 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 411 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 412 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 413 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 414 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 415 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 416 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 417 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 418 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 419 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 420 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 421 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 422 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 423 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 424 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, 425 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, 426 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 427 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, 428 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20, 429 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF, 430 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20, 431 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF, 432 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 433 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 434 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 435 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 436 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 437 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 438 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 439 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 440 }; 441 442 static const int32_t ngrams_windows_1256[] = { 443 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8, 444 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD, 445 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20, 446 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420, 447 }; 448 449 static const uint8_t charMap_windows_1256[] = { 450 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 451 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 452 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 453 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 454 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 455 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 456 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 457 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 458 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 459 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 460 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 461 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 462 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 463 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 464 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 465 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 466 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, 467 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F, 468 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 469 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F, 470 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 471 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, 472 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 473 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 474 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 475 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 476 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20, 477 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 478 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 479 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 480 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20, 481 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF, 482 }; 483 484 static const int32_t ngrams_KOI8_R[] = { 485 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1, 486 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE, 487 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1, 488 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF, 489 }; 490 491 static const uint8_t charMap_KOI8_R[] = { 492 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 493 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 494 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 495 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 496 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 497 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 498 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 499 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 500 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 501 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 502 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 503 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 504 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 505 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 506 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 507 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 508 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 509 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 510 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 511 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 512 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, 513 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 514 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, 515 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 516 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 517 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 518 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 519 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 520 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 521 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 522 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 523 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 524 }; 525 526 static const int32_t ngrams_IBM424_he_rtl[] = { 527 0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641, 528 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045, 529 0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, 530 0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069, 531 }; 532 533 static const int32_t ngrams_IBM424_he_ltr[] = { 534 0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141, 535 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054, 536 0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940, 537 0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651, 538 }; 539 540 static const uint8_t charMap_IBM424_he[] = { 541 /* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ 542 /* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 543 /* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 544 /* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 545 /* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 546 /* 4- */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 547 /* 5- */ 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 548 /* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 549 /* 7- */ 0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40, 550 /* 8- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 551 /* 9- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 552 /* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 553 /* B- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 554 /* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 555 /* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 556 /* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 557 /* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 558 }; 559 560 static const int32_t ngrams_IBM420_ar_rtl[] = { 561 0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158, 562 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB, 563 0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40, 564 0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40, 565 }; 566 567 static const int32_t ngrams_IBM420_ar_ltr[] = { 568 0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF, 569 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD, 570 0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156, 571 0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156 572 }; 573 574 static const uint8_t charMap_IBM420_ar[]= { 575 /* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ 576 /* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 577 /* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 578 /* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 579 /* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 580 /* 4- */ 0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 581 /* 5- */ 0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 582 /* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 583 /* 7- */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 584 /* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 585 /* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, 586 /* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 587 /* B- */ 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 588 /* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF, 589 /* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 590 /* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF, 591 /* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40, 592 }; 593 594 //ISO-8859-1,2,5,6,7,8,9 Ngrams 595 static const int32_t ngrams_8859_1_en[] = { 596 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F, 597 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74, 598 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420, 599 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320, 600 }; 601 602 static const int32_t ngrams_8859_1_da[] = { 603 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620, 604 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320, 605 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520, 606 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572, 607 }; 608 609 static const int32_t ngrams_8859_1_de[] = { 610 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F, 611 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220, 612 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465, 613 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572, 614 }; 615 616 static const int32_t ngrams_8859_1_es[] = { 617 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, 618 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C, 619 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064, 620 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20, 621 }; 622 623 static const int32_t ngrams_8859_1_fr[] = { 624 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E, 625 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20, 626 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420, 627 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220, 628 }; 629 630 static const int32_t ngrams_8859_1_it[] = { 631 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073, 632 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220, 633 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20, 634 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F, 635 }; 636 637 static const int32_t ngrams_8859_1_nl[] = { 638 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665, 639 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E, 640 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F, 641 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F, 642 }; 643 644 static const int32_t ngrams_8859_1_no[] = { 645 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469, 646 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474, 647 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65, 648 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572, 649 }; 650 651 static const int32_t ngrams_8859_1_pt[] = { 652 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, 653 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20, 654 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065, 655 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F, 656 }; 657 658 static const int32_t ngrams_8859_1_sv[] = { 659 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469, 660 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220, 661 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20, 662 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220, 663 }; 664 665 static const int32_t ngrams_8859_2_cs[] = { 666 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F, 667 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465, 668 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865, 669 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564, 670 }; 671 672 static const int32_t ngrams_8859_2_hu[] = { 673 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69, 674 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20, 675 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061, 676 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320, 677 }; 678 679 static const int32_t ngrams_8859_2_pl[] = { 680 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779, 681 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20, 682 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769, 683 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720, 684 }; 685 686 static const int32_t ngrams_8859_2_ro[] = { 687 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69, 688 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070, 689 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72, 690 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20, 691 }; 692 693 static const int32_t ngrams_8859_5_ru[] = { 694 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE, 695 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD, 696 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2, 697 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520, 698 }; 699 700 static const int32_t ngrams_8859_6_ar[] = { 701 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8, 702 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1, 703 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20, 704 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620, 705 }; 706 707 static const int32_t ngrams_8859_7_el[] = { 708 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7, 709 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120, 710 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5, 711 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20, 712 }; 713 714 static const int32_t ngrams_8859_8_I_he[] = { 715 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0, 716 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4, 717 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE, 718 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9, 719 }; 720 721 static const int32_t ngrams_8859_8_he[] = { 722 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0, 723 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC, 724 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920, 725 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9, 726 }; 727 728 static const int32_t ngrams_8859_9_tr[] = { 729 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961, 730 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062, 731 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062, 732 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD, 733 }; 734 735 CharsetRecog_8859_1::~CharsetRecog_8859_1() 736 { 737 // nothing to do 738 } 739 740 const char *CharsetRecog_8859_1::getName() const 741 { 742 return haveC1Bytes? "windows-1252" : "ISO-8859-1"; 743 } 744 745 const char *CharsetRecog_8859_1_en::getLanguage() const 746 { 747 return "en"; 748 } 749 750 CharsetRecog_8859_1_en::~CharsetRecog_8859_1_en() 751 { 752 // nothing to do 753 } 754 755 int32_t CharsetRecog_8859_1_en::match(InputText *textIn) 756 { 757 int32_t result = match_sbcs(textIn, ngrams_8859_1_en, charMap_8859_1); 758 759 // printf("8859_1_en: result = %d\n", result); 760 return result; //match_sbcs(textIn, ngrams, charMap); 761 } 762 763 CharsetRecog_8859_1_da::~CharsetRecog_8859_1_da() 764 { 765 // nothing to do 766 } 767 768 const char *CharsetRecog_8859_1_da::getLanguage() const 769 { 770 return "da"; 771 } 772 773 int32_t CharsetRecog_8859_1_da::match(InputText *textIn) 774 { 775 return match_sbcs(textIn, ngrams_8859_1_da, charMap_8859_1); 776 } 777 778 CharsetRecog_8859_1_de::~CharsetRecog_8859_1_de() {} 779 780 const char *CharsetRecog_8859_1_de::getLanguage() const 781 { 782 return "de"; 783 } 784 785 int32_t CharsetRecog_8859_1_de::match(InputText *textIn) 786 { 787 return match_sbcs(textIn, ngrams_8859_1_de, charMap_8859_1); 788 } 789 790 CharsetRecog_8859_1_es::~CharsetRecog_8859_1_es() 791 { 792 // nothing to do 793 } 794 795 const char *CharsetRecog_8859_1_es::getLanguage() const 796 { 797 return "es"; 798 } 799 800 int32_t CharsetRecog_8859_1_es::match(InputText *textIn) 801 { 802 return match_sbcs(textIn, ngrams_8859_1_es, charMap_8859_1); 803 } 804 805 CharsetRecog_8859_1_fr::~CharsetRecog_8859_1_fr() 806 { 807 // nothing to do 808 } 809 810 const char *CharsetRecog_8859_1_fr::getLanguage() const 811 { 812 return "fr"; 813 } 814 815 int32_t CharsetRecog_8859_1_fr::match(InputText *textIn) 816 { 817 return match_sbcs(textIn, ngrams_8859_1_fr, charMap_8859_1); 818 } 819 820 CharsetRecog_8859_1_it::~CharsetRecog_8859_1_it() 821 { 822 // nothing to do 823 } 824 825 const char *CharsetRecog_8859_1_it::getLanguage() const 826 { 827 return "it"; 828 } 829 830 int32_t CharsetRecog_8859_1_it::match(InputText *textIn) 831 { 832 return match_sbcs(textIn, ngrams_8859_1_it, charMap_8859_1); 833 } 834 835 CharsetRecog_8859_1_nl::~CharsetRecog_8859_1_nl() 836 { 837 // nothing to do 838 } 839 840 const char *CharsetRecog_8859_1_nl::getLanguage() const 841 { 842 return "nl"; 843 } 844 845 int32_t CharsetRecog_8859_1_nl::match(InputText *textIn) 846 { 847 return match_sbcs(textIn, ngrams_8859_1_nl, charMap_8859_1); 848 } 849 850 CharsetRecog_8859_1_no::~CharsetRecog_8859_1_no() {} 851 852 const char *CharsetRecog_8859_1_no::getLanguage() const 853 { 854 return "no"; 855 } 856 857 int32_t CharsetRecog_8859_1_no::match(InputText *textIn) 858 { 859 return match_sbcs(textIn, ngrams_8859_1_no, charMap_8859_1); 860 } 861 862 CharsetRecog_8859_1_pt::~CharsetRecog_8859_1_pt() 863 { 864 // nothing to do 865 } 866 867 const char *CharsetRecog_8859_1_pt::getLanguage() const 868 { 869 return "pt"; 870 } 871 872 int32_t CharsetRecog_8859_1_pt::match(InputText *textIn) 873 { 874 return match_sbcs(textIn, ngrams_8859_1_pt, charMap_8859_1); 875 } 876 877 CharsetRecog_8859_1_sv::~CharsetRecog_8859_1_sv() {} 878 879 const char *CharsetRecog_8859_1_sv::getLanguage() const 880 { 881 return "sv"; 882 } 883 884 int32_t CharsetRecog_8859_1_sv::match(InputText *textIn) 885 { 886 return match_sbcs(textIn, ngrams_8859_1_sv, charMap_8859_1); 887 } 888 889 CharsetRecog_8859_2::~CharsetRecog_8859_2() 890 { 891 // nothing to do 892 } 893 894 const char *CharsetRecog_8859_2::getName() const 895 { 896 return haveC1Bytes? "windows-1250" : "ISO-8859-2"; 897 } 898 899 CharsetRecog_8859_2_cs::~CharsetRecog_8859_2_cs() 900 { 901 // nothing to do 902 } 903 904 const char *CharsetRecog_8859_2_cs::getLanguage() const 905 { 906 return "cs"; 907 } 908 909 int32_t CharsetRecog_8859_2_cs::match(InputText *textIn) 910 { 911 return match_sbcs(textIn, ngrams_8859_2_cs, charMap_8859_2); 912 } 913 914 CharsetRecog_8859_2_hu::~CharsetRecog_8859_2_hu() 915 { 916 // nothing to do 917 } 918 919 const char *CharsetRecog_8859_2_hu::getLanguage() const 920 { 921 return "hu"; 922 } 923 924 int32_t CharsetRecog_8859_2_hu::match(InputText *textIn) 925 { 926 return match_sbcs(textIn, ngrams_8859_2_hu, charMap_8859_2); 927 } 928 929 CharsetRecog_8859_2_pl::~CharsetRecog_8859_2_pl() 930 { 931 // nothing to do 932 } 933 934 const char *CharsetRecog_8859_2_pl::getLanguage() const 935 { 936 return "pl"; 937 } 938 939 int32_t CharsetRecog_8859_2_pl::match(InputText *textIn) 940 { 941 return match_sbcs(textIn, ngrams_8859_2_pl, charMap_8859_2); 942 } 943 944 CharsetRecog_8859_2_ro::~CharsetRecog_8859_2_ro() 945 { 946 // nothing to do 947 } 948 949 const char *CharsetRecog_8859_2_ro::getLanguage() const 950 { 951 return "ro"; 952 } 953 954 int32_t CharsetRecog_8859_2_ro::match(InputText *textIn) 955 { 956 return match_sbcs(textIn, ngrams_8859_2_ro, charMap_8859_2); 957 } 958 959 CharsetRecog_8859_5::~CharsetRecog_8859_5() 960 { 961 // nothing to do 962 } 963 964 const char *CharsetRecog_8859_5::getName() const 965 { 966 return "ISO-8859-5"; 967 } 968 969 CharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru() 970 { 971 // nothing to do 972 } 973 974 const char *CharsetRecog_8859_5_ru::getLanguage() const 975 { 976 return "ru"; 977 } 978 979 int32_t CharsetRecog_8859_5_ru::match(InputText *textIn) 980 { 981 return match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5); 982 } 983 984 CharsetRecog_8859_6::~CharsetRecog_8859_6() 985 { 986 // nothing to do 987 } 988 989 const char *CharsetRecog_8859_6::getName() const 990 { 991 return "ISO-8859-6"; 992 } 993 994 CharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar() 995 { 996 // nothing to do 997 } 998 999 const char *CharsetRecog_8859_6_ar::getLanguage() const 1000 { 1001 return "ar"; 1002 } 1003 1004 int32_t CharsetRecog_8859_6_ar::match(InputText *textIn) 1005 { 1006 return match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6); 1007 } 1008 1009 CharsetRecog_8859_7::~CharsetRecog_8859_7() 1010 { 1011 // nothing to do 1012 } 1013 1014 const char *CharsetRecog_8859_7::getName() const 1015 { 1016 return haveC1Bytes? "windows-1253" : "ISO-8859-7"; 1017 } 1018 1019 CharsetRecog_8859_7_el::~CharsetRecog_8859_7_el() 1020 { 1021 // nothing to do 1022 } 1023 1024 const char *CharsetRecog_8859_7_el::getLanguage() const 1025 { 1026 return "el"; 1027 } 1028 1029 int32_t CharsetRecog_8859_7_el::match(InputText *textIn) 1030 { 1031 return match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7); 1032 } 1033 1034 CharsetRecog_8859_8::~CharsetRecog_8859_8() 1035 { 1036 // nothing to do 1037 } 1038 1039 const char *CharsetRecog_8859_8::getName() const 1040 { 1041 return haveC1Bytes? "windows-1255" : "ISO-8859-8"; 1042 } 1043 1044 CharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he () 1045 { 1046 // nothing to do 1047 } 1048 1049 const char *CharsetRecog_8859_8_I_he::getName() const 1050 { 1051 return haveC1Bytes? "windows-1255" : "ISO-8859-8-I"; 1052 } 1053 1054 const char *CharsetRecog_8859_8_I_he::getLanguage() const 1055 { 1056 return "he"; 1057 } 1058 1059 int32_t CharsetRecog_8859_8_I_he::match(InputText *textIn) 1060 { 1061 return match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8); 1062 } 1063 1064 CharsetRecog_8859_8_he::~CharsetRecog_8859_8_he() 1065 { 1066 // od ot gnihton 1067 } 1068 1069 const char *CharsetRecog_8859_8_he::getLanguage() const 1070 { 1071 return "he"; 1072 } 1073 1074 int32_t CharsetRecog_8859_8_he::match(InputText *textIn) 1075 { 1076 return match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8); 1077 } 1078 1079 CharsetRecog_8859_9::~CharsetRecog_8859_9() 1080 { 1081 // nothing to do 1082 } 1083 1084 const char *CharsetRecog_8859_9::getName() const 1085 { 1086 return haveC1Bytes? "windows-1254" : "ISO-8859-9"; 1087 } 1088 1089 CharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr () 1090 { 1091 // nothing to do 1092 } 1093 1094 const char *CharsetRecog_8859_9_tr::getLanguage() const 1095 { 1096 return "tr"; 1097 } 1098 1099 int32_t CharsetRecog_8859_9_tr::match(InputText *textIn) 1100 { 1101 return match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9); 1102 } 1103 1104 CharsetRecog_windows_1256::~CharsetRecog_windows_1256() 1105 { 1106 // nothing to do 1107 } 1108 1109 const char *CharsetRecog_windows_1256::getName() const 1110 { 1111 return "windows-1256"; 1112 } 1113 1114 const char *CharsetRecog_windows_1256::getLanguage() const 1115 { 1116 return "ar"; 1117 } 1118 1119 int32_t CharsetRecog_windows_1256::match(InputText *textIn) 1120 { 1121 return match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256); 1122 } 1123 1124 CharsetRecog_windows_1251::~CharsetRecog_windows_1251() 1125 { 1126 // nothing to do 1127 } 1128 1129 const char *CharsetRecog_windows_1251::getName() const 1130 { 1131 return "windows-1251"; 1132 } 1133 1134 const char *CharsetRecog_windows_1251::getLanguage() const 1135 { 1136 return "ru"; 1137 } 1138 1139 int32_t CharsetRecog_windows_1251::match(InputText *textIn) 1140 { 1141 return match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251); 1142 } 1143 1144 CharsetRecog_KOI8_R::~CharsetRecog_KOI8_R() 1145 { 1146 // nothing to do 1147 } 1148 1149 const char *CharsetRecog_KOI8_R::getName() const 1150 { 1151 return "KOI8-R"; 1152 } 1153 1154 const char *CharsetRecog_KOI8_R::getLanguage() const 1155 { 1156 return "ru"; 1157 } 1158 1159 int32_t CharsetRecog_KOI8_R::match(InputText *textIn) 1160 { 1161 return match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R); 1162 } 1163 1164 CharsetRecog_IBM424_he::~CharsetRecog_IBM424_he() 1165 { 1166 // nothing to do 1167 } 1168 1169 const char *CharsetRecog_IBM424_he::getLanguage() const 1170 { 1171 return "he"; 1172 } 1173 1174 CharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl() 1175 { 1176 // nothing to do 1177 } 1178 1179 const char *CharsetRecog_IBM424_he_rtl::getName() const 1180 { 1181 return "IBM424_rtl"; 1182 } 1183 1184 int32_t CharsetRecog_IBM424_he_rtl::match(InputText *textIn) 1185 { 1186 return match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he); 1187 } 1188 1189 CharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr() 1190 { 1191 // nothing to do 1192 } 1193 1194 const char *CharsetRecog_IBM424_he_ltr::getName() const 1195 { 1196 return "IBM424_ltr"; 1197 } 1198 1199 int32_t CharsetRecog_IBM424_he_ltr::match(InputText *textIn) 1200 { 1201 return match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he); 1202 } 1203 1204 static const uint8_t unshapeMap_IBM420[] = { 1205 /* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ 1206 /* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1207 /* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1208 /* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1209 /* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1210 /* 4- */ 0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 1211 /* 5- */ 0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 1212 /* 6- */ 0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 1213 /* 7- */ 0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 1214 /* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F, 1215 /* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E, 1216 /* A- */ 0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF, 1217 /* B- */ 0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF, 1218 /* C- */ 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF, 1219 /* D- */ 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF, 1220 /* E- */ 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 1221 /* F- */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 1222 }; 1223 1224 CharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar() 1225 { 1226 // nothing to do 1227 } 1228 1229 const char *CharsetRecog_IBM420_ar::getLanguage() const 1230 { 1231 return "ar"; 1232 } 1233 1234 void CharsetRecog_IBM420_ar::matchInit(InputText *textIn) { 1235 prev_fInputBytesLength = textIn->fInputLen; 1236 prev_fInputBytes = textIn->fInputBytes; 1237 1238 int32_t length = 0; 1239 uint8_t *bb = unshape(prev_fInputBytes, prev_fInputBytesLength, length); 1240 1241 if (bb != NULL) { 1242 textIn->fInputBytes = bb; 1243 textIn->fInputLen = length; 1244 1245 deleteBuffer = TRUE; 1246 } else { 1247 deleteBuffer = FALSE; 1248 } 1249 } 1250 1251 uint8_t *CharsetRecog_IBM420_ar::unshape(const uint8_t *inputBytes, int32_t inputBytesLength, int32_t &length) { 1252 uint8_t *resultArray = unshapeLamAlef(inputBytes, inputBytesLength, length); 1253 1254 if (resultArray != NULL) { 1255 for (int32_t i = 0; i < inputBytesLength; i++) { 1256 resultArray[i] = unshapeMap_IBM420[resultArray[i]]; 1257 } 1258 } 1259 1260 return resultArray; 1261 } 1262 1263 uint8_t *CharsetRecog_IBM420_ar::unshapeLamAlef(const uint8_t *inputBytes, int32_t inputBytesLength, int32_t &length) { 1264 int32_t bigBufferLength = inputBytesLength * 2; 1265 uint8_t *bigBuffer = (uint8_t *)uprv_malloc(bigBufferLength); 1266 uint8_t *resultBuffer = NULL; 1267 1268 if (bigBuffer != NULL) { 1269 int32_t bufferIndex; 1270 uint8_t unshapedLamAlef[] = { 0xb1, 0x56 }; 1271 1272 for (int32_t i = bufferIndex = 0; i < inputBytesLength; i++) { 1273 if (isLamAlef(inputBytes[i])) { 1274 bigBuffer[bufferIndex++] = unshapedLamAlef[0]; 1275 bigBuffer[bufferIndex++] = unshapedLamAlef[1]; 1276 } else { 1277 bigBuffer[bufferIndex++] = inputBytes[i]; 1278 } 1279 } 1280 1281 length = bufferIndex; 1282 resultBuffer = (uint8_t *)uprv_malloc(length); 1283 if (resultBuffer != NULL) { 1284 uprv_memcpy(resultBuffer, bigBuffer, length); 1285 } 1286 } 1287 1288 if (bigBuffer != NULL) { 1289 uprv_free(bigBuffer); 1290 } 1291 1292 return resultBuffer; 1293 } 1294 1295 void CharsetRecog_IBM420_ar::matchFinish(InputText *textIn) { 1296 if (deleteBuffer) { 1297 uprv_free(textIn->fInputBytes); 1298 1299 textIn->fInputBytes = prev_fInputBytes; 1300 textIn->fInputLen = prev_fInputBytesLength; 1301 } 1302 } 1303 1304 UBool CharsetRecog_IBM420_ar::isLamAlef(uint8_t b) { 1305 uint8_t shapedLamAlef[] = { 1306 0xb2, 0xb3, 0xb4, 0xb5, 0xb7, 0xb8 1307 }; 1308 1309 for (uint32_t i = 0; i < sizeof(shapedLamAlef); i++) { 1310 if (b == shapedLamAlef[i]) { 1311 return TRUE; 1312 } 1313 } 1314 1315 return FALSE; 1316 } 1317 1318 CharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl() 1319 { 1320 // nothing to do 1321 } 1322 1323 const char *CharsetRecog_IBM420_ar_rtl::getName() const 1324 { 1325 return "IBM420_rtl"; 1326 } 1327 1328 int32_t CharsetRecog_IBM420_ar_rtl::match(InputText *textIn) 1329 { 1330 return match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar); 1331 } 1332 1333 CharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr() 1334 { 1335 // nothing to do 1336 } 1337 1338 const char *CharsetRecog_IBM420_ar_ltr::getName() const 1339 { 1340 return "IBM420_ltr"; 1341 } 1342 1343 int32_t CharsetRecog_IBM420_ar_ltr::match(InputText *textIn) 1344 { 1345 return match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar); 1346 } 1347 1348 U_NAMESPACE_END 1349 #endif 1350 1351
