1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef LIBTEXTCLASSIFIER_UTIL_STRINGS_UTF8_H_ 18 #define LIBTEXTCLASSIFIER_UTIL_STRINGS_UTF8_H_ 19 20 namespace libtextclassifier2 { 21 22 // Returns the length (number of bytes) of the Unicode code point starting at 23 // src, based on inspecting just that one byte. Preconditions: src != NULL, 24 // *src can be read, and *src is not '\0', and src points to a well-formed UTF-8 25 // std::string. 26 static inline int GetNumBytesForNonZeroUTF8Char(const char *src) { 27 // On most platforms, char is unsigned by default, but iOS is an exception. 28 // The cast below makes sure we always interpret *src as an unsigned char. 29 return "\1\1\1\1\1\1\1\1\1\1\1\1\2\2\3\4" 30 [(*(reinterpret_cast<const unsigned char *>(src)) & 0xFF) >> 4]; 31 } 32 33 // Like GetNumBytesForNonZeroUTF8Char, but *src may be '\0'; returns 0 in that 34 // case. 35 static inline int GetNumBytesForUTF8Char(const char *src) { 36 if (*src == '\0') return 0; 37 return GetNumBytesForNonZeroUTF8Char(src); 38 } 39 40 // Returns true if this byte is a trailing UTF-8 byte (10xx xxxx) 41 static inline bool IsTrailByte(char x) { 42 // return (x & 0xC0) == 0x80; 43 // Since trail bytes are always in [0x80, 0xBF], we can optimize: 44 return static_cast<signed char>(x) < -0x40; 45 } 46 47 // Returns true iff src points to a well-formed UTF-8 string. 48 bool IsValidUTF8(const char *src, int size); 49 50 } // namespace libtextclassifier2 51 52 #endif // LIBTEXTCLASSIFIER_UTIL_STRINGS_UTF8_H_ 53
