1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 1999-2011, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * file name: utf.h 11 * encoding: US-ASCII 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 1999sep09 16 * created by: Markus W. Scherer 17 */ 18 19 /** 20 * \file 21 * \brief C API: Code point macros 22 * 23 * This file defines macros for checking whether a code point is 24 * a surrogate or a non-character etc. 25 * 26 * The UChar and UChar32 data types for Unicode code units and code points 27 * are defined in umachine.h because they can be machine-dependent. 28 * 29 * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 0 then utf.h is included by utypes.h 30 * and itself includes utf8.h and utf16.h after some 31 * common definitions. 32 * If U_NO_DEFAULT_INCLUDE_UTF_HEADERS is 1 then each of these headers must be 33 * included explicitly if their definitions are used. 34 * 35 * utf8.h and utf16.h define macros for efficiently getting code points 36 * in and out of UTF-8/16 strings. 37 * utf16.h macros have "U16_" prefixes. 38 * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling. 39 * 40 * ICU mostly processes 16-bit Unicode strings. 41 * Most of the time, such strings are well-formed UTF-16. 42 * Single, unpaired surrogates must be handled as well, and are treated in ICU 43 * like regular code points where possible. 44 * (Pairs of surrogate code points are indistinguishable from supplementary 45 * code points encoded as pairs of supplementary code units.) 46 * 47 * In fact, almost all Unicode code points in normal text (>99%) 48 * are on the BMP (<=U+ffff) and even <=U+d7ff. 49 * ICU functions handle supplementary code points (U+10000..U+10ffff) 50 * but are optimized for the much more frequently occurring BMP code points. 51 * 52 * umachine.h defines UChar to be an unsigned 16-bit integer. 53 * Where available, UChar is defined to be a char16_t 54 * or a wchar_t (if that is an unsigned 16-bit type), otherwise uint16_t. 55 * 56 * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit 57 * Unicode code point (Unicode scalar value, 0..0x10ffff). 58 * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as 59 * the definition of UChar. For details see the documentation for UChar32 itself. 60 * 61 * utf.h defines a small number of C macros for single Unicode code points. 62 * These are simple checks for surrogates and non-characters. 63 * For actual Unicode character properties see uchar.h. 64 * 65 * By default, string operations must be done with error checking in case 66 * a string is not well-formed UTF-16. 67 * The macros will detect if a surrogate code unit is unpaired 68 * (lead unit without trail unit or vice versa) and just return the unit itself 69 * as the code point. 70 * 71 * The regular "safe" macros require that the initial, passed-in string index 72 * is within bounds. They only check the index when they read more than one 73 * code unit. This is usually done with code similar to the following loop: 74 * <pre>while(i<length) { 75 * U16_NEXT(s, i, length, c); 76 * // use c 77 * }</pre> 78 * 79 * When it is safe to assume that text is well-formed UTF-16 80 * (does not contain single, unpaired surrogates), then one can use 81 * U16_..._UNSAFE macros. 82 * These do not check for proper code unit sequences or truncated text and may 83 * yield wrong results or even cause a crash if they are used with "malformed" 84 * text. 85 * In practice, U16_..._UNSAFE macros will produce slightly less code but 86 * should not be faster because the processing is only different when a 87 * surrogate code unit is detected, which will be rare. 88 * 89 * Similarly for UTF-8, there are "safe" macros without a suffix, 90 * and U8_..._UNSAFE versions. 91 * The performance differences are much larger here because UTF-8 provides so 92 * many opportunities for malformed sequences. 93 * The unsafe UTF-8 macros are entirely implemented inside the macro definitions 94 * and are fast, while the safe UTF-8 macros call functions for all but the 95 * trivial (ASCII) cases. 96 * (ICU 3.6 optimizes U8_NEXT() and U8_APPEND() to handle most other common 97 * characters inline as well.) 98 * 99 * Unlike with UTF-16, malformed sequences cannot be expressed with distinct 100 * code point values (0..U+10ffff). They are indicated with negative values instead. 101 * 102 * For more information see the ICU User Guide Strings chapter 103 * (http://userguide.icu-project.org/strings). 104 * 105 * <em>Usage:</em> 106 * ICU coding guidelines for if() statements should be followed when using these macros. 107 * Compound statements (curly braces {}) must be used for if-else-while... 108 * bodies and all macro statements should be terminated with semicolon. 109 * 110 * @stable ICU 2.4 111 */ 112 113 #ifndef __UTF_H__ 114 #define __UTF_H__ 115 116 #include "unicode/umachine.h" 117 /* include the utfXX.h after the following definitions */ 118 119 /* single-code point definitions -------------------------------------------- */ 120 121 /** 122 * Is this code point a Unicode noncharacter? 123 * @param c 32-bit code point 124 * @return TRUE or FALSE 125 * @stable ICU 2.4 126 */ 127 #define U_IS_UNICODE_NONCHAR(c) \ 128 ((c)>=0xfdd0 && \ 129 ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \ 130 (uint32_t)(c)<=0x10ffff) 131 132 /** 133 * Is c a Unicode code point value (0..U+10ffff) 134 * that can be assigned a character? 135 * 136 * Code points that are not characters include: 137 * - single surrogate code points (U+d800..U+dfff, 2048 code points) 138 * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points) 139 * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points) 140 * - the highest Unicode code point value is U+10ffff 141 * 142 * This means that all code points below U+d800 are character code points, 143 * and that boundary is tested first for performance. 144 * 145 * @param c 32-bit code point 146 * @return TRUE or FALSE 147 * @stable ICU 2.4 148 */ 149 #define U_IS_UNICODE_CHAR(c) \ 150 ((uint32_t)(c)<0xd800 || \ 151 ((uint32_t)(c)>0xdfff && \ 152 (uint32_t)(c)<=0x10ffff && \ 153 !U_IS_UNICODE_NONCHAR(c))) 154 155 /** 156 * Is this code point a BMP code point (U+0000..U+ffff)? 157 * @param c 32-bit code point 158 * @return TRUE or FALSE 159 * @stable ICU 2.8 160 */ 161 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff) 162 163 /** 164 * Is this code point a supplementary code point (U+10000..U+10ffff)? 165 * @param c 32-bit code point 166 * @return TRUE or FALSE 167 * @stable ICU 2.8 168 */ 169 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff) 170 171 /** 172 * Is this code point a lead surrogate (U+d800..U+dbff)? 173 * @param c 32-bit code point 174 * @return TRUE or FALSE 175 * @stable ICU 2.4 176 */ 177 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800) 178 179 /** 180 * Is this code point a trail surrogate (U+dc00..U+dfff)? 181 * @param c 32-bit code point 182 * @return TRUE or FALSE 183 * @stable ICU 2.4 184 */ 185 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00) 186 187 /** 188 * Is this code point a surrogate (U+d800..U+dfff)? 189 * @param c 32-bit code point 190 * @return TRUE or FALSE 191 * @stable ICU 2.4 192 */ 193 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800) 194 195 /** 196 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)), 197 * is it a lead surrogate? 198 * @param c 32-bit code point 199 * @return TRUE or FALSE 200 * @stable ICU 2.4 201 */ 202 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0) 203 204 /** 205 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)), 206 * is it a trail surrogate? 207 * @param c 32-bit code point 208 * @return TRUE or FALSE 209 * @stable ICU 4.2 210 */ 211 #define U_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0) 212 213 /* include the utfXX.h ------------------------------------------------------ */ 214 215 #if !U_NO_DEFAULT_INCLUDE_UTF_HEADERS 216 217 #include "unicode/utf8.h" 218 #include "unicode/utf16.h" 219 220 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */ 221 #include "unicode/utf_old.h" 222 223 #endif /* !U_NO_DEFAULT_INCLUDE_UTF_HEADERS */ 224 225 #endif /* __UTF_H__ */ 226
