1 /* 2 * Copyright (C) 2009 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <stdio.h> 18 #include <stdlib.h> 19 20 #include "PhoneticStringUtils.h" 21 #include <utils/Unicode.h> 22 23 // We'd like 0 length string last of sorted list. So when input string is NULL 24 // or 0 length string, we use these instead. 25 #define CODEPOINT_FOR_NULL_STR 0xFFFD 26 #define STR_FOR_NULL_STR "\xEF\xBF\xBD" 27 28 // We assume that users will not notice strings not sorted properly when the 29 // first 128 characters are the same. 30 #define MAX_CODEPOINTS 128 31 32 namespace android { 33 34 // Get hiragana from halfwidth katakana. 35 static int GetHiraganaFromHalfwidthKatakana(char32_t codepoint, 36 char32_t next_codepoint, 37 bool *next_is_consumed) { 38 if (codepoint < 0xFF66 || 0xFF9F < codepoint) { 39 return codepoint; 40 } 41 42 switch (codepoint) { 43 case 0xFF66: // wo 44 return 0x3092; 45 case 0xFF67: // xa 46 return 0x3041; 47 case 0xFF68: // xi 48 return 0x3043; 49 case 0xFF69: // xu 50 return 0x3045; 51 case 0xFF6A: // xe 52 return 0x3047; 53 case 0xFF6B: // xo 54 return 0x3049; 55 case 0xFF6C: // xya 56 return 0x3083; 57 case 0xFF6D: // xyu 58 return 0x3085; 59 case 0xFF6E: // xyo 60 return 0x3087; 61 case 0xFF6F: // xtsu 62 return 0x3063; 63 case 0xFF70: // - 64 return 0x30FC; 65 case 0xFF9C: // wa 66 return 0x308F; 67 case 0xFF9D: // n 68 return 0x3093; 69 break; 70 default: { 71 if (0xFF71 <= codepoint && codepoint <= 0xFF75) { 72 // a, i, u, e, o 73 if (codepoint == 0xFF73 && next_codepoint == 0xFF9E) { 74 if (next_is_consumed != NULL) { 75 *next_is_consumed = true; 76 } 77 return 0x3094; // vu 78 } else { 79 return 0x3042 + (codepoint - 0xFF71) * 2; 80 } 81 } else if (0xFF76 <= codepoint && codepoint <= 0xFF81) { 82 // ka - chi 83 if (next_codepoint == 0xFF9E) { 84 // "dakuten" (voiced mark) 85 if (next_is_consumed != NULL) { 86 *next_is_consumed = true; 87 } 88 return 0x304B + (codepoint - 0xFF76) * 2 + 1; 89 } else { 90 return 0x304B + (codepoint - 0xFF76) * 2; 91 } 92 } else if (0xFF82 <= codepoint && codepoint <= 0xFF84) { 93 // tsu, te, to (skip xtsu) 94 if (next_codepoint == 0xFF9E) { 95 // "dakuten" (voiced mark) 96 if (next_is_consumed != NULL) { 97 *next_is_consumed = true; 98 } 99 return 0x3064 + (codepoint - 0xFF82) * 2 + 1; 100 } else { 101 return 0x3064 + (codepoint - 0xFF82) * 2; 102 } 103 } else if (0xFF85 <= codepoint && codepoint <= 0xFF89) { 104 // na, ni, nu, ne, no 105 return 0x306A + (codepoint - 0xFF85); 106 } else if (0xFF8A <= codepoint && codepoint <= 0xFF8E) { 107 // ha, hi, hu, he, ho 108 if (next_codepoint == 0xFF9E) { 109 // "dakuten" (voiced mark) 110 if (next_is_consumed != NULL) { 111 *next_is_consumed = true; 112 } 113 return 0x306F + (codepoint - 0xFF8A) * 3 + 1; 114 } else if (next_codepoint == 0xFF9F) { 115 // "han-dakuten" (half voiced mark) 116 if (next_is_consumed != NULL) { 117 *next_is_consumed = true; 118 } 119 return 0x306F + (codepoint - 0xFF8A) * 3 + 2; 120 } else { 121 return 0x306F + (codepoint - 0xFF8A) * 3; 122 } 123 } else if (0xFF8F <= codepoint && codepoint <= 0xFF93) { 124 // ma, mi, mu, me, mo 125 return 0x307E + (codepoint - 0xFF8F); 126 } else if (0xFF94 <= codepoint && codepoint <= 0xFF96) { 127 // ya, yu, yo 128 return 0x3084 + (codepoint - 0xFF94) * 2; 129 } else if (0xFF97 <= codepoint && codepoint <= 0xFF9B) { 130 // ra, ri, ru, re, ro 131 return 0x3089 + (codepoint - 0xFF97); 132 } 133 // Note: 0xFF9C, 0xFF9D are handled above 134 } // end of default 135 } 136 137 return codepoint; 138 } 139 140 // Assuming input is hiragana, convert the hiragana to "normalized" hiragana. 141 static int GetNormalizedHiragana(int codepoint) { 142 if (codepoint < 0x3040 || 0x309F < codepoint) { 143 return codepoint; 144 } 145 146 // TODO: should care (semi-)voiced mark (0x3099, 0x309A). 147 148 // Trivial kana conversions. 149 // e.g. xa => a 150 switch (codepoint) { 151 case 0x3041: 152 case 0x3043: 153 case 0x3045: 154 case 0x3047: 155 case 0x3049: 156 case 0x3063: 157 case 0x3083: 158 case 0x3085: 159 case 0x3087: 160 case 0x308E: // xwa 161 return codepoint + 1; 162 case 0x3095: // xka 163 return 0x304B; 164 case 0x3096: // xke 165 return 0x3051; 166 case 0x31F0: // xku 167 return 0x304F; 168 case 0x31F1: // xsi 169 return 0x3057; 170 case 0x31F2: // xsu 171 return 0x3059; 172 case 0x31F3: // xto 173 return 0x3068; 174 case 0x31F4: // xnu 175 return 0x306C; 176 case 0x31F5: // xha 177 return 0x306F; 178 case 0x31F6: // xhi 179 return 0x3072; 180 case 0x31F7: // xhu 181 return 0x3075; 182 case 0x31F8: // xhe 183 return 0x3078; 184 case 0x31F9: // xho 185 return 0x307B; 186 case 0x31FA: // xmu 187 return 0x3080; 188 case 0x31FB: // xra 189 case 0x31FC: // xri 190 case 0x31FD: // xru 191 case 0x31FE: // xre 192 case 0x31FF: // xro 193 // ra: 0x3089 194 return 0x3089 + (codepoint - 0x31FB); 195 default: 196 return codepoint; 197 } 198 } 199 200 static int GetNormalizedKana(char32_t codepoint, 201 char32_t next_codepoint, 202 bool *next_is_consumed) { 203 // First, convert fullwidth katakana and halfwidth katakana to hiragana. 204 if (0x30A1 <= codepoint && codepoint <= 0x30F6) { 205 // Make fullwidth katakana same as hiragana. 206 // 96 == 0x30A1 - 0x3041c 207 codepoint = codepoint - 96; 208 } else if (codepoint == 0x309F) { 209 // Digraph YORI; Yo 210 codepoint = 0x3088; 211 } else if (codepoint == 0x30FF) { 212 // Digraph KOTO; Ko 213 codepoint = 0x3053; 214 } else { 215 codepoint = GetHiraganaFromHalfwidthKatakana( 216 codepoint, next_codepoint, next_is_consumed); 217 } 218 219 // Normalize Hiragana. 220 return GetNormalizedHiragana(codepoint); 221 } 222 223 int GetNormalizedCodePoint(char32_t codepoint, 224 char32_t next_codepoint, 225 bool *next_is_consumed) { 226 if (next_is_consumed != NULL) { 227 *next_is_consumed = false; 228 } 229 230 if (codepoint <= 0x0020 || codepoint == 0x3000) { 231 // Whitespaces. Keep it as is. 232 return codepoint; 233 } else if ((0x0021 <= codepoint && codepoint <= 0x007E) || 234 (0xFF01 <= codepoint && codepoint <= 0xFF5E)) { 235 // Ascii and fullwidth ascii. Keep it as is 236 return codepoint; 237 } else if (codepoint == 0x02DC || codepoint == 0x223C) { 238 // tilde 239 return 0xFF5E; 240 } else if (codepoint <= 0x3040 || 241 (0x3100 <= codepoint && codepoint < 0xFF00) || 242 codepoint == CODEPOINT_FOR_NULL_STR) { 243 // Keep it as is. 244 return codepoint; 245 } 246 247 // Below is Kana-related handling. 248 249 return GetNormalizedKana(codepoint, next_codepoint, next_is_consumed); 250 } 251 252 static bool GetExpectedString( 253 const char *src, char **dst, size_t *dst_len, 254 int (*get_codepoint_function)(char32_t, char32_t, bool*)) { 255 if (dst == NULL || dst_len == NULL) { 256 return false; 257 } 258 259 if (src == NULL || *src == '\0') { 260 src = STR_FOR_NULL_STR; 261 } 262 263 char32_t codepoints[MAX_CODEPOINTS]; // if array size is changed the for loop needs to be changed 264 265 ssize_t src_len = utf8_length(src); 266 if (src_len <= 0) { 267 return false; 268 } 269 270 bool next_is_consumed; 271 size_t j = 0; 272 for (size_t i = 0; i < (size_t)src_len && j < MAX_CODEPOINTS;) { 273 int32_t ret = utf32_from_utf8_at(src, src_len, i, &i); 274 if (ret < 0) { 275 // failed to parse UTF-8 276 return false; 277 } 278 ret = get_codepoint_function( 279 static_cast<char32_t>(ret), 280 i + 1 < (size_t)src_len ? src[i + 1] : 0, 281 &next_is_consumed); 282 if (ret > 0) { 283 codepoints[j] = static_cast<char32_t>(ret); 284 j++; 285 } 286 if (next_is_consumed) { 287 i++; 288 } 289 } 290 size_t length = j; 291 292 if (length == 0) { 293 // If all of codepoints are invalid, we place the string at the end of 294 // the list. 295 codepoints[0] = 0x10000 + CODEPOINT_FOR_NULL_STR; 296 length = 1; 297 } 298 299 ssize_t new_len = utf32_to_utf8_length(codepoints, length); 300 if (new_len < 0) { 301 return false; 302 } 303 304 *dst = static_cast<char *>(malloc(new_len + 1)); 305 if (*dst == NULL) { 306 return false; 307 } 308 309 utf32_to_utf8(codepoints, length, *dst); 310 311 *dst_len = new_len; 312 return true; 313 } 314 315 bool GetNormalizedString(const char *src, char **dst, size_t *len) { 316 return GetExpectedString(src, dst, len, GetNormalizedCodePoint); 317 } 318 319 } // namespace android 320
