1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.inputmethod.latin.utils; 18 19 import static com.android.inputmethod.latin.Constants.CODE_UNSPECIFIED; 20 21 import android.text.Spanned; 22 import android.text.TextUtils; 23 24 import com.android.inputmethod.annotations.UsedForTesting; 25 import com.android.inputmethod.latin.Constants; 26 27 import java.util.ArrayList; 28 import java.util.Arrays; 29 import java.util.Locale; 30 import java.util.regex.Matcher; 31 import java.util.regex.Pattern; 32 33 public final class StringUtils { 34 public static final int CAPITALIZE_NONE = 0; // No caps, or mixed case 35 public static final int CAPITALIZE_FIRST = 1; // First only 36 public static final int CAPITALIZE_ALL = 2; // All caps 37 38 private static final String EMPTY_STRING = ""; 39 40 private static final char CHAR_LINE_FEED = 0X000A; 41 private static final char CHAR_VERTICAL_TAB = 0X000B; 42 private static final char CHAR_FORM_FEED = 0X000C; 43 private static final char CHAR_CARRIAGE_RETURN = 0X000D; 44 private static final char CHAR_NEXT_LINE = 0X0085; 45 private static final char CHAR_LINE_SEPARATOR = 0X2028; 46 private static final char CHAR_PARAGRAPH_SEPARATOR = 0X2029; 47 48 private StringUtils() { 49 // This utility class is not publicly instantiable. 50 } 51 52 public static int codePointCount(final String text) { 53 if (TextUtils.isEmpty(text)) return 0; 54 return text.codePointCount(0, text.length()); 55 } 56 57 public static String newSingleCodePointString(int codePoint) { 58 if (Character.charCount(codePoint) == 1) { 59 // Optimization: avoid creating a temporary array for characters that are 60 // represented by a single char value 61 return String.valueOf((char) codePoint); 62 } 63 // For surrogate pair 64 return new String(Character.toChars(codePoint)); 65 } 66 67 public static boolean containsInArray(final String text, final String[] array) { 68 for (final String element : array) { 69 if (text.equals(element)) return true; 70 } 71 return false; 72 } 73 74 /** 75 * Comma-Splittable Text is similar to Comma-Separated Values (CSV) but has much simpler syntax. 76 * Unlike CSV, Comma-Splittable Text has no escaping mechanism, so that the text can't contain 77 * a comma character in it. 78 */ 79 private static final String SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT = ","; 80 81 public static boolean containsInCommaSplittableText(final String text, 82 final String extraValues) { 83 if (TextUtils.isEmpty(extraValues)) { 84 return false; 85 } 86 return containsInArray(text, extraValues.split(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT)); 87 } 88 89 public static String removeFromCommaSplittableTextIfExists(final String text, 90 final String extraValues) { 91 if (TextUtils.isEmpty(extraValues)) { 92 return EMPTY_STRING; 93 } 94 final String[] elements = extraValues.split(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT); 95 if (!containsInArray(text, elements)) { 96 return extraValues; 97 } 98 final ArrayList<String> result = new ArrayList<>(elements.length - 1); 99 for (final String element : elements) { 100 if (!text.equals(element)) { 101 result.add(element); 102 } 103 } 104 return TextUtils.join(SEPARATOR_FOR_COMMA_SPLITTABLE_TEXT, result); 105 } 106 107 /** 108 * Remove duplicates from an array of strings. 109 * 110 * This method will always keep the first occurrence of all strings at their position 111 * in the array, removing the subsequent ones. 112 */ 113 public static void removeDupes(final ArrayList<String> suggestions) { 114 if (suggestions.size() < 2) return; 115 int i = 1; 116 // Don't cache suggestions.size(), since we may be removing items 117 while (i < suggestions.size()) { 118 final String cur = suggestions.get(i); 119 // Compare each suggestion with each previous suggestion 120 for (int j = 0; j < i; j++) { 121 final String previous = suggestions.get(j); 122 if (TextUtils.equals(cur, previous)) { 123 suggestions.remove(i); 124 i--; 125 break; 126 } 127 } 128 i++; 129 } 130 } 131 132 public static String capitalizeFirstCodePoint(final String s, final Locale locale) { 133 if (s.length() <= 1) { 134 return s.toUpperCase(locale); 135 } 136 // Please refer to the comment below in 137 // {@link #capitalizeFirstAndDowncaseRest(String,Locale)} as this has the same shortcomings 138 final int cutoff = s.offsetByCodePoints(0, 1); 139 return s.substring(0, cutoff).toUpperCase(locale) + s.substring(cutoff); 140 } 141 142 public static String capitalizeFirstAndDowncaseRest(final String s, final Locale locale) { 143 if (s.length() <= 1) { 144 return s.toUpperCase(locale); 145 } 146 // TODO: fix the bugs below 147 // - This does not work for Greek, because it returns upper case instead of title case. 148 // - It does not work for Serbian, because it fails to account for the "lj" character, 149 // which should be "Lj" in title case and "LJ" in upper case. 150 // - It does not work for Dutch, because it fails to account for the "ij" digraph when it's 151 // written as two separate code points. They are two different characters but both should 152 // be capitalized as "IJ" as if they were a single letter in most words (not all). If the 153 // unicode char for the ligature is used however, it works. 154 final int cutoff = s.offsetByCodePoints(0, 1); 155 return s.substring(0, cutoff).toUpperCase(locale) + s.substring(cutoff).toLowerCase(locale); 156 } 157 158 private static final int[] EMPTY_CODEPOINTS = {}; 159 160 public static int[] toCodePointArray(final CharSequence charSequence) { 161 return toCodePointArray(charSequence, 0, charSequence.length()); 162 } 163 164 /** 165 * Converts a range of a string to an array of code points. 166 * @param charSequence the source string. 167 * @param startIndex the start index inside the string in java chars, inclusive. 168 * @param endIndex the end index inside the string in java chars, exclusive. 169 * @return a new array of code points. At most endIndex - startIndex, but possibly less. 170 */ 171 public static int[] toCodePointArray(final CharSequence charSequence, 172 final int startIndex, final int endIndex) { 173 final int length = charSequence.length(); 174 if (length <= 0) { 175 return EMPTY_CODEPOINTS; 176 } 177 final int[] codePoints = 178 new int[Character.codePointCount(charSequence, startIndex, endIndex)]; 179 copyCodePointsAndReturnCodePointCount(codePoints, charSequence, startIndex, endIndex, 180 false /* downCase */); 181 return codePoints; 182 } 183 184 /** 185 * Copies the codepoints in a CharSequence to an int array. 186 * 187 * This method assumes there is enough space in the array to store the code points. The size 188 * can be measured with Character#codePointCount(CharSequence, int, int) before passing to this 189 * method. If the int array is too small, an ArrayIndexOutOfBoundsException will be thrown. 190 * Also, this method makes no effort to be thread-safe. Do not modify the CharSequence while 191 * this method is running, or the behavior is undefined. 192 * This method can optionally downcase code points before copying them, but it pays no attention 193 * to locale while doing so. 194 * 195 * @param destination the int array. 196 * @param charSequence the CharSequence. 197 * @param startIndex the start index inside the string in java chars, inclusive. 198 * @param endIndex the end index inside the string in java chars, exclusive. 199 * @param downCase if this is true, code points will be downcased before being copied. 200 * @return the number of copied code points. 201 */ 202 public static int copyCodePointsAndReturnCodePointCount(final int[] destination, 203 final CharSequence charSequence, final int startIndex, final int endIndex, 204 final boolean downCase) { 205 int destIndex = 0; 206 for (int index = startIndex; index < endIndex; 207 index = Character.offsetByCodePoints(charSequence, index, 1)) { 208 final int codePoint = Character.codePointAt(charSequence, index); 209 // TODO: stop using this, as it's not aware of the locale and does not always do 210 // the right thing. 211 destination[destIndex] = downCase ? Character.toLowerCase(codePoint) : codePoint; 212 destIndex++; 213 } 214 return destIndex; 215 } 216 217 public static int[] toSortedCodePointArray(final String string) { 218 final int[] codePoints = toCodePointArray(string); 219 Arrays.sort(codePoints); 220 return codePoints; 221 } 222 223 /** 224 * Construct a String from a code point array 225 * 226 * @param codePoints a code point array that is null terminated when its logical length is 227 * shorter than the array length. 228 * @return a string constructed from the code point array. 229 */ 230 public static String getStringFromNullTerminatedCodePointArray(final int[] codePoints) { 231 int stringLength = codePoints.length; 232 for (int i = 0; i < codePoints.length; i++) { 233 if (codePoints[i] == 0) { 234 stringLength = i; 235 break; 236 } 237 } 238 return new String(codePoints, 0 /* offset */, stringLength); 239 } 240 241 // This method assumes the text is not null. For the empty string, it returns CAPITALIZE_NONE. 242 public static int getCapitalizationType(final String text) { 243 // If the first char is not uppercase, then the word is either all lower case or 244 // camel case, and in either case we return CAPITALIZE_NONE. 245 final int len = text.length(); 246 int index = 0; 247 for (; index < len; index = text.offsetByCodePoints(index, 1)) { 248 if (Character.isLetter(text.codePointAt(index))) { 249 break; 250 } 251 } 252 if (index == len) return CAPITALIZE_NONE; 253 if (!Character.isUpperCase(text.codePointAt(index))) { 254 return CAPITALIZE_NONE; 255 } 256 int capsCount = 1; 257 int letterCount = 1; 258 for (index = text.offsetByCodePoints(index, 1); index < len; 259 index = text.offsetByCodePoints(index, 1)) { 260 if (1 != capsCount && letterCount != capsCount) break; 261 final int codePoint = text.codePointAt(index); 262 if (Character.isUpperCase(codePoint)) { 263 ++capsCount; 264 ++letterCount; 265 } else if (Character.isLetter(codePoint)) { 266 // We need to discount non-letters since they may not be upper-case, but may 267 // still be part of a word (e.g. single quote or dash, as in "IT'S" or "FULL-TIME") 268 ++letterCount; 269 } 270 } 271 // We know the first char is upper case. So we want to test if either every letter other 272 // than the first is lower case, or if they are all upper case. If the string is exactly 273 // one char long, then we will arrive here with letterCount 1, and this is correct, too. 274 if (1 == capsCount) return CAPITALIZE_FIRST; 275 return (letterCount == capsCount ? CAPITALIZE_ALL : CAPITALIZE_NONE); 276 } 277 278 public static boolean isIdenticalAfterUpcase(final String text) { 279 final int length = text.length(); 280 int i = 0; 281 while (i < length) { 282 final int codePoint = text.codePointAt(i); 283 if (Character.isLetter(codePoint) && !Character.isUpperCase(codePoint)) { 284 return false; 285 } 286 i += Character.charCount(codePoint); 287 } 288 return true; 289 } 290 291 public static boolean isIdenticalAfterDowncase(final String text) { 292 final int length = text.length(); 293 int i = 0; 294 while (i < length) { 295 final int codePoint = text.codePointAt(i); 296 if (Character.isLetter(codePoint) && !Character.isLowerCase(codePoint)) { 297 return false; 298 } 299 i += Character.charCount(codePoint); 300 } 301 return true; 302 } 303 304 public static boolean isIdenticalAfterCapitalizeEachWord(final String text, 305 final int[] sortedSeparators) { 306 boolean needsCapsNext = true; 307 final int len = text.length(); 308 for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) { 309 final int codePoint = text.codePointAt(i); 310 if (Character.isLetter(codePoint)) { 311 if ((needsCapsNext && !Character.isUpperCase(codePoint)) 312 || (!needsCapsNext && !Character.isLowerCase(codePoint))) { 313 return false; 314 } 315 } 316 // We need a capital letter next if this is a separator. 317 needsCapsNext = (Arrays.binarySearch(sortedSeparators, codePoint) >= 0); 318 } 319 return true; 320 } 321 322 // TODO: like capitalizeFirst*, this does not work perfectly for Dutch because of the IJ digraph 323 // which should be capitalized together in *some* cases. 324 public static String capitalizeEachWord(final String text, final int[] sortedSeparators, 325 final Locale locale) { 326 final StringBuilder builder = new StringBuilder(); 327 boolean needsCapsNext = true; 328 final int len = text.length(); 329 for (int i = 0; i < len; i = text.offsetByCodePoints(i, 1)) { 330 final String nextChar = text.substring(i, text.offsetByCodePoints(i, 1)); 331 if (needsCapsNext) { 332 builder.append(nextChar.toUpperCase(locale)); 333 } else { 334 builder.append(nextChar.toLowerCase(locale)); 335 } 336 // We need a capital letter next if this is a separator. 337 needsCapsNext = (Arrays.binarySearch(sortedSeparators, nextChar.codePointAt(0)) >= 0); 338 } 339 return builder.toString(); 340 } 341 342 /** 343 * Approximates whether the text before the cursor looks like a URL. 344 * 345 * This is not foolproof, but it should work well in the practice. 346 * Essentially it walks backward from the cursor until it finds something that's not a letter, 347 * digit, or common URL symbol like underscore. If it hasn't found a period yet, then it 348 * does not look like a URL. 349 * If the text: 350 * - starts with www and contains a period 351 * - starts with a slash preceded by either a slash, whitespace, or start-of-string 352 * Then it looks like a URL and we return true. Otherwise, we return false. 353 * 354 * Note: this method is called quite often, and should be fast. 355 * 356 * TODO: This will return that "abc./def" and ".abc/def" look like URLs to keep down the 357 * code complexity, but ideally it should not. It's acceptable for now. 358 */ 359 public static boolean lastPartLooksLikeURL(final CharSequence text) { 360 int i = text.length(); 361 if (0 == i) return false; 362 int wCount = 0; 363 int slashCount = 0; 364 boolean hasSlash = false; 365 boolean hasPeriod = false; 366 int codePoint = 0; 367 while (i > 0) { 368 codePoint = Character.codePointBefore(text, i); 369 if (codePoint < Constants.CODE_PERIOD || codePoint > 'z') { 370 // Handwavy heuristic to see if that's a URL character. Anything between period 371 // and z. This includes all lower- and upper-case ascii letters, period, 372 // underscore, arrobase, question mark, equal sign. It excludes spaces, exclamation 373 // marks, double quotes... 374 // Anything that's not a URL-like character causes us to break from here and 375 // evaluate normally. 376 break; 377 } 378 if (Constants.CODE_PERIOD == codePoint) { 379 hasPeriod = true; 380 } 381 if (Constants.CODE_SLASH == codePoint) { 382 hasSlash = true; 383 if (2 == ++slashCount) { 384 return true; 385 } 386 } else { 387 slashCount = 0; 388 } 389 if ('w' == codePoint) { 390 ++wCount; 391 } else { 392 wCount = 0; 393 } 394 i = Character.offsetByCodePoints(text, i, -1); 395 } 396 // End of the text run. 397 // If it starts with www and includes a period, then it looks like a URL. 398 if (wCount >= 3 && hasPeriod) return true; 399 // If it starts with a slash, and the code point before is whitespace, it looks like an URL. 400 if (1 == slashCount && (0 == i || Character.isWhitespace(codePoint))) return true; 401 // If it has both a period and a slash, it looks like an URL. 402 if (hasPeriod && hasSlash) return true; 403 // Otherwise, it doesn't look like an URL. 404 return false; 405 } 406 407 /** 408 * Examines the string and returns whether we're inside a double quote. 409 * 410 * This is used to decide whether we should put an automatic space before or after a double 411 * quote character. If we're inside a quotation, then we want to close it, so we want a space 412 * after and not before. Otherwise, we want to open the quotation, so we want a space before 413 * and not after. Exception: after a digit, we never want a space because the "inch" or 414 * "minutes" use cases is dominant after digits. 415 * In the practice, we determine whether we are in a quotation or not by finding the previous 416 * double quote character, and looking at whether it's followed by whitespace. If so, that 417 * was a closing quotation mark, so we're not inside a double quote. If it's not followed 418 * by whitespace, then it was an opening quotation mark, and we're inside a quotation. 419 * 420 * @param text the text to examine. 421 * @return whether we're inside a double quote. 422 */ 423 public static boolean isInsideDoubleQuoteOrAfterDigit(final CharSequence text) { 424 int i = text.length(); 425 if (0 == i) return false; 426 int codePoint = Character.codePointBefore(text, i); 427 if (Character.isDigit(codePoint)) return true; 428 int prevCodePoint = 0; 429 while (i > 0) { 430 codePoint = Character.codePointBefore(text, i); 431 if (Constants.CODE_DOUBLE_QUOTE == codePoint) { 432 // If we see a double quote followed by whitespace, then that 433 // was a closing quote. 434 if (Character.isWhitespace(prevCodePoint)) return false; 435 } 436 if (Character.isWhitespace(codePoint) && Constants.CODE_DOUBLE_QUOTE == prevCodePoint) { 437 // If we see a double quote preceded by whitespace, then that 438 // was an opening quote. No need to continue seeking. 439 return true; 440 } 441 i -= Character.charCount(codePoint); 442 prevCodePoint = codePoint; 443 } 444 // We reached the start of text. If the first char is a double quote, then we're inside 445 // a double quote. Otherwise we're not. 446 return Constants.CODE_DOUBLE_QUOTE == codePoint; 447 } 448 449 public static boolean isEmptyStringOrWhiteSpaces(final String s) { 450 final int N = codePointCount(s); 451 for (int i = 0; i < N; ++i) { 452 if (!Character.isWhitespace(s.codePointAt(i))) { 453 return false; 454 } 455 } 456 return true; 457 } 458 459 @UsedForTesting 460 public static String byteArrayToHexString(final byte[] bytes) { 461 if (bytes == null || bytes.length == 0) { 462 return EMPTY_STRING; 463 } 464 final StringBuilder sb = new StringBuilder(); 465 for (byte b : bytes) { 466 sb.append(String.format("%02x", b & 0xff)); 467 } 468 return sb.toString(); 469 } 470 471 /** 472 * Convert hex string to byte array. The string length must be an even number. 473 */ 474 @UsedForTesting 475 public static byte[] hexStringToByteArray(final String hexString) { 476 if (TextUtils.isEmpty(hexString)) { 477 return null; 478 } 479 final int N = hexString.length(); 480 if (N % 2 != 0) { 481 throw new NumberFormatException("Input hex string length must be an even number." 482 + " Length = " + N); 483 } 484 final byte[] bytes = new byte[N / 2]; 485 for (int i = 0; i < N; i += 2) { 486 bytes[i / 2] = (byte) ((Character.digit(hexString.charAt(i), 16) << 4) 487 + Character.digit(hexString.charAt(i + 1), 16)); 488 } 489 return bytes; 490 } 491 492 public static String toUpperCaseOfStringForLocale(final String text, 493 final boolean needsToUpperCase, final Locale locale) { 494 if (text == null || !needsToUpperCase) return text; 495 return text.toUpperCase(locale); 496 } 497 498 public static int toUpperCaseOfCodeForLocale(final int code, final boolean needsToUpperCase, 499 final Locale locale) { 500 if (!Constants.isLetterCode(code) || !needsToUpperCase) return code; 501 final String text = newSingleCodePointString(code); 502 final String casedText = toUpperCaseOfStringForLocale( 503 text, needsToUpperCase, locale); 504 return codePointCount(casedText) == 1 505 ? casedText.codePointAt(0) : CODE_UNSPECIFIED; 506 } 507 508 public static int getTrailingSingleQuotesCount(final CharSequence charSequence) { 509 final int lastIndex = charSequence.length() - 1; 510 int i = lastIndex; 511 while (i >= 0 && charSequence.charAt(i) == Constants.CODE_SINGLE_QUOTE) { 512 --i; 513 } 514 return lastIndex - i; 515 } 516 517 /** 518 * Splits the given {@code charSequence} with at occurrences of the given {@code regex}. 519 * <p> 520 * This is equivalent to 521 * {@code charSequence.toString().split(regex, preserveTrailingEmptySegments ? -1 : 0)} 522 * except that the spans are preserved in the result array. 523 * </p> 524 * @param input the character sequence to be split. 525 * @param regex the regex pattern to be used as the separator. 526 * @param preserveTrailingEmptySegments {@code true} to preserve the trailing empty 527 * segments. Otherwise, trailing empty segments will be removed before being returned. 528 * @return the array which contains the result. All the spans in the {@param input} is 529 * preserved. 530 */ 531 @UsedForTesting 532 public static CharSequence[] split(final CharSequence charSequence, final String regex, 533 final boolean preserveTrailingEmptySegments) { 534 // A short-cut for non-spanned strings. 535 if (!(charSequence instanceof Spanned)) { 536 // -1 means that trailing empty segments will be preserved. 537 return charSequence.toString().split(regex, preserveTrailingEmptySegments ? -1 : 0); 538 } 539 540 // Hereafter, emulate String.split for CharSequence. 541 final ArrayList<CharSequence> sequences = new ArrayList<>(); 542 final Matcher matcher = Pattern.compile(regex).matcher(charSequence); 543 int nextStart = 0; 544 boolean matched = false; 545 while (matcher.find()) { 546 sequences.add(charSequence.subSequence(nextStart, matcher.start())); 547 nextStart = matcher.end(); 548 matched = true; 549 } 550 if (!matched) { 551 // never matched. preserveTrailingEmptySegments is ignored in this case. 552 return new CharSequence[] { charSequence }; 553 } 554 sequences.add(charSequence.subSequence(nextStart, charSequence.length())); 555 if (!preserveTrailingEmptySegments) { 556 for (int i = sequences.size() - 1; i >= 0; --i) { 557 if (!TextUtils.isEmpty(sequences.get(i))) { 558 break; 559 } 560 sequences.remove(i); 561 } 562 } 563 return sequences.toArray(new CharSequence[sequences.size()]); 564 } 565 566 @UsedForTesting 567 public static class Stringizer<E> { 568 public String stringize(final E element) { 569 return element != null ? element.toString() : "null"; 570 } 571 572 @UsedForTesting 573 public final String join(final E[] array) { 574 return joinStringArray(toStringArray(array), null /* delimiter */); 575 } 576 577 @UsedForTesting 578 public final String join(final E[] array, final String delimiter) { 579 return joinStringArray(toStringArray(array), delimiter); 580 } 581 582 protected String[] toStringArray(final E[] array) { 583 final String[] stringArray = new String[array.length]; 584 for (int index = 0; index < array.length; index++) { 585 stringArray[index] = stringize(array[index]); 586 } 587 return stringArray; 588 } 589 590 protected String joinStringArray(final String[] stringArray, final String delimiter) { 591 if (stringArray == null) { 592 return "null"; 593 } 594 if (delimiter == null) { 595 return Arrays.toString(stringArray); 596 } 597 final StringBuilder sb = new StringBuilder(); 598 for (int index = 0; index < stringArray.length; index++) { 599 sb.append(index == 0 ? "[" : delimiter); 600 sb.append(stringArray[index]); 601 } 602 return sb + "]"; 603 } 604 } 605 606 /** 607 * Returns whether the last composed word contains line-breaking character (e.g. CR or LF). 608 * @param text the text to be examined. 609 * @return {@code true} if the last composed word contains line-breaking separator. 610 */ 611 @UsedForTesting 612 public static boolean hasLineBreakCharacter(final String text) { 613 if (TextUtils.isEmpty(text)) { 614 return false; 615 } 616 for (int i = text.length() - 1; i >= 0; --i) { 617 final char c = text.charAt(i); 618 switch (c) { 619 case CHAR_LINE_FEED: 620 case CHAR_VERTICAL_TAB: 621 case CHAR_FORM_FEED: 622 case CHAR_CARRIAGE_RETURN: 623 case CHAR_NEXT_LINE: 624 case CHAR_LINE_SEPARATOR: 625 case CHAR_PARAGRAPH_SEPARATOR: 626 return true; 627 } 628 } 629 return false; 630 } 631 } 632