1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 1999-2004, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: utf.h 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 1999sep09 14 * created by: Markus W. Scherer 15 */ 16 17 /** 18 * \file 19 * \brief C API: Code point macros 20 * 21 * This file defines macros for checking whether a code point is 22 * a surrogate or a non-character etc. 23 * 24 * The UChar and UChar32 data types for Unicode code units and code points 25 * are defined in umachines.h because they can be machine-dependent. 26 * 27 * utf.h is included by utypes.h and itself includes utf8.h and utf16.h after some 28 * common definitions. Those files define macros for efficiently getting code points 29 * in and out of UTF-8/16 strings. 30 * utf16.h macros have "U16_" prefixes. 31 * utf8.h defines similar macros with "U8_" prefixes for UTF-8 string handling. 32 * 33 * ICU processes 16-bit Unicode strings. 34 * Most of the time, such strings are well-formed UTF-16. 35 * Single, unpaired surrogates must be handled as well, and are treated in ICU 36 * like regular code points where possible. 37 * (Pairs of surrogate code points are indistinguishable from supplementary 38 * code points encoded as pairs of supplementary code units.) 39 * 40 * In fact, almost all Unicode code points in normal text (>99%) 41 * are on the BMP (<=U+ffff) and even <=U+d7ff. 42 * ICU functions handle supplementary code points (U+10000..U+10ffff) 43 * but are optimized for the much more frequently occurring BMP code points. 44 * 45 * utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then 46 * UChar is defined to be exactly wchar_t, otherwise uint16_t. 47 * 48 * UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit 49 * Unicode code point (Unicode scalar value, 0..0x10ffff). 50 * Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as 51 * the definition of UChar. For details see the documentation for UChar32 itself. 52 * 53 * utf.h also defines a small number of C macros for single Unicode code points. 54 * These are simple checks for surrogates and non-characters. 55 * For actual Unicode character properties see uchar.h. 56 * 57 * By default, string operations must be done with error checking in case 58 * a string is not well-formed UTF-16. 59 * The macros will detect if a surrogate code unit is unpaired 60 * (lead unit without trail unit or vice versa) and just return the unit itself 61 * as the code point. 62 * (It is an accidental property of Unicode and UTF-16 that all 63 * malformed sequences can be expressed unambiguously with a distinct subrange 64 * of Unicode code points.) 65 * 66 * When it is safe to assume that text is well-formed UTF-16 67 * (does not contain single, unpaired surrogates), then one can use 68 * U16_..._UNSAFE macros. 69 * These do not check for proper code unit sequences or truncated text and may 70 * yield wrong results or even cause a crash if they are used with "malformed" 71 * text. 72 * In practice, U16_..._UNSAFE macros will produce slightly less code but 73 * should not be faster because the processing is only different when a 74 * surrogate code unit is detected, which will be rare. 75 * 76 * Similarly for UTF-8, there are "safe" macros without a suffix, 77 * and U8_..._UNSAFE versions. 78 * The performance differences are much larger here because UTF-8 provides so 79 * many opportunities for malformed sequences. 80 * The unsafe UTF-8 macros are entirely implemented inside the macro definitions 81 * and are fast, while the safe UTF-8 macros call functions for all but the 82 * trivial (ASCII) cases. 83 * 84 * Unlike with UTF-16, malformed sequences cannot be expressed with distinct 85 * code point values (0..U+10ffff). They are indicated with negative values instead. 86 * 87 * For more information see the ICU User Guide Strings chapter 88 * (http://oss.software.ibm.com/icu/userguide/). 89 * 90 * <em>Usage:</em> 91 * ICU coding guidelines for if() statements should be followed when using these macros. 92 * Compound statements (curly braces {}) must be used for if-else-while... 93 * bodies and all macro statements should be terminated with semicolon. 94 * 95 * @stable ICU 2.4 96 */ 97 98 #ifndef __UTF_H__ 99 #define __UTF_H__ 100 101 #include "unicode/utypes.h" 102 /* include the utfXX.h after the following definitions */ 103 104 /* single-code point definitions -------------------------------------------- */ 105 106 /** 107 * This value is intended for sentinel values for APIs that 108 * (take or) return single code points (UChar32). 109 * It is outside of the Unicode code point range 0..0x10ffff. 110 * 111 * For example, a "done" or "error" value in a new API 112 * could be indicated with U_SENTINEL. 113 * 114 * ICU APIs designed before ICU 2.4 usually define service-specific "done" 115 * values, mostly 0xffff. 116 * Those may need to be distinguished from 117 * actual U+ffff text contents by calling functions like 118 * CharacterIterator::hasNext() or UnicodeString::length(). 119 * 120 * @return -1 121 * @see UChar32 122 * @stable ICU 2.4 123 */ 124 #define U_SENTINEL (-1) 125 126 /** 127 * Is this code point a Unicode noncharacter? 128 * @param c 32-bit code point 129 * @return TRUE or FALSE 130 * @stable ICU 2.4 131 */ 132 #define U_IS_UNICODE_NONCHAR(c) \ 133 ((c)>=0xfdd0 && \ 134 ((uint32_t)(c)<=0xfdef || ((c)&0xfffe)==0xfffe) && \ 135 (uint32_t)(c)<=0x10ffff) 136 137 /** 138 * Is c a Unicode code point value (0..U+10ffff) 139 * that can be assigned a character? 140 * 141 * Code points that are not characters include: 142 * - single surrogate code points (U+d800..U+dfff, 2048 code points) 143 * - the last two code points on each plane (U+__fffe and U+__ffff, 34 code points) 144 * - U+fdd0..U+fdef (new with Unicode 3.1, 32 code points) 145 * - the highest Unicode code point value is U+10ffff 146 * 147 * This means that all code points below U+d800 are character code points, 148 * and that boundary is tested first for performance. 149 * 150 * @param c 32-bit code point 151 * @return TRUE or FALSE 152 * @stable ICU 2.4 153 */ 154 #define U_IS_UNICODE_CHAR(c) \ 155 ((uint32_t)(c)<0xd800 || \ 156 ((uint32_t)(c)>0xdfff && \ 157 (uint32_t)(c)<=0x10ffff && \ 158 !U_IS_UNICODE_NONCHAR(c))) 159 160 #ifndef U_HIDE_DRAFT_API 161 162 /** 163 * Is this code point a BMP code point (U+0000..U+ffff)? 164 * @param c 32-bit code point 165 * @return TRUE or FALSE 166 * @draft ICU 2.8 167 */ 168 #define U_IS_BMP(c) ((uint32_t)(c)<=0xffff) 169 170 /** 171 * Is this code point a supplementary code point (U+10000..U+10ffff)? 172 * @param c 32-bit code point 173 * @return TRUE or FALSE 174 * @draft ICU 2.8 175 */ 176 #define U_IS_SUPPLEMENTARY(c) ((uint32_t)((c)-0x10000)<=0xfffff) 177 178 #endif /*U_HIDE_DRAFT_API*/ 179 180 /** 181 * Is this code point a lead surrogate (U+d800..U+dbff)? 182 * @param c 32-bit code point 183 * @return TRUE or FALSE 184 * @stable ICU 2.4 185 */ 186 #define U_IS_LEAD(c) (((c)&0xfffffc00)==0xd800) 187 188 /** 189 * Is this code point a trail surrogate (U+dc00..U+dfff)? 190 * @param c 32-bit code point 191 * @return TRUE or FALSE 192 * @stable ICU 2.4 193 */ 194 #define U_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00) 195 196 /** 197 * Is this code point a surrogate (U+d800..U+dfff)? 198 * @param c 32-bit code point 199 * @return TRUE or FALSE 200 * @stable ICU 2.4 201 */ 202 #define U_IS_SURROGATE(c) (((c)&0xfffff800)==0xd800) 203 204 /** 205 * Assuming c is a surrogate code point (U_IS_SURROGATE(c)), 206 * is it a lead surrogate? 207 * @param c 32-bit code point 208 * @return TRUE or FALSE 209 * @stable ICU 2.4 210 */ 211 #define U_IS_SURROGATE_LEAD(c) (((c)&0x400)==0) 212 213 /* include the utfXX.h ------------------------------------------------------ */ 214 215 #include "unicode/utf8.h" 216 #include "unicode/utf16.h" 217 218 /* utf_old.h contains deprecated, pre-ICU 2.4 definitions */ 219 #include "unicode/utf_old.h" 220 221 #endif 222
