1 /* 2 *************************************************************************** 3 * Copyright (C) 2008-2009, International Business Machines Corporation 4 * and others. All Rights Reserved. 5 *************************************************************************** 6 * 7 * uspoof_impl.h 8 * 9 * Implemenation header for spoof detection 10 * 11 */ 12 13 #ifndef USPOOFIM_H 14 #define USPOOFIM_H 15 16 #include "unicode/utypes.h" 17 #include "unicode/uspoof.h" 18 #include "utrie2.h" 19 #include "unicode/uscript.h" 20 #include "unicode/udata.h" 21 22 23 #if !UCONFIG_NO_NORMALIZATION 24 25 U_NAMESPACE_BEGIN 26 27 // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from 28 // a single input code point. This is function of the unicode.org data. 29 #define USPOOF_MAX_SKELETON_EXPANSION 20 30 31 // The default stack buffer size for copies or conversions or normalizations 32 // of input strings being checked. (Used in multiple places.) 33 #define USPOOF_STACK_BUFFER_SIZE 100 34 35 // Magic number for sanity checking spoof data. 36 #define USPOOF_MAGIC 0x3845fdef 37 38 class SpoofData; 39 struct SpoofDataHeader; 40 struct SpoofStringLengthsElement; 41 class ScriptSet; 42 43 /** 44 * Class SpoofImpl corresponds directly to the plain C API opaque type 45 * USpoofChecker. One can be cast to the other. 46 */ 47 class SpoofImpl : public UObject { 48 public: 49 SpoofImpl(SpoofData *data, UErrorCode &status); 50 SpoofImpl(); 51 virtual ~SpoofImpl(); 52 53 /** Copy constructor, used by the user level uspoof_clone() function. 54 */ 55 SpoofImpl(const SpoofImpl &src, UErrorCode &status); 56 57 static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status); 58 static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status); 59 60 /** Get the confusable skeleton transform for a single code point. 61 * The result is a string with a length between 1 and 18. 62 * @param tableMask bit flag specifying which confusable table to use. 63 * One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc. 64 * @return The length in UTF-16 code units of the substition string. 65 */ 66 int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UChar *destBuf) const; 67 68 /** Set and Get AllowedLocales, implementations of the corresponding API */ 69 void setAllowedLocales(const char *localesList, UErrorCode &status); 70 const char * getAllowedLocales(UErrorCode &status); 71 72 // Add (union) to the UnicodeSet all of the characters for the scripts used for 73 // the specified locale. Part of the implementation of setAllowedLocales. 74 void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status); 75 76 77 /** parse a hex number. Untility used by the builders. */ 78 static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status); 79 80 // Implementation for Whole Script tests. 81 // Return the test bit flag to be ORed into the eventual user return value 82 // if a Spoof opportunity is detected. 83 void wholeScriptCheck( 84 const UChar *text, int32_t length, ScriptSet *result, UErrorCode &status) const; 85 86 /** Scan a string to determine how many scripts it includes. 87 * Ignore characters with script=Common and scirpt=Inherited. 88 * @param text The UChar text to be scanned 89 * @param length The length of the input text, -1 for nul termintated. 90 * @param pos An out parameter, set to the first input postion at which 91 * a second script was encountered, ignoring Common and Inherited. 92 * @param status For errors. 93 * @return the number of (non-common,inherited) scripts encountered, 94 * clipped to a max of two. 95 */ 96 int32_t scriptScan(const UChar *text, int32_t length, int32_t &pos, UErrorCode &status) const; 97 98 99 // WholeScript and MixedScript check implementation. 100 // 101 ScriptSet *WholeScriptCheck(const UChar *text, int32_t length, UErrorCode &status) const; 102 103 static UClassID U_EXPORT2 getStaticClassID(void); 104 virtual UClassID getDynamicClassID(void) const; 105 106 // 107 // Data Members 108 // 109 110 int32_t fMagic; // Internal sanity check. 111 int32_t fChecks; // Bit vector of checks to perform. 112 113 SpoofData *fSpoofData; 114 115 int32_t fCheckMask; // Spoof table selector. f(Check Type) 116 117 const UnicodeSet *fAllowedCharsSet; // The UnicodeSet of allowed characters. 118 // for this Spoof Checker. Defaults to all chars. 119 120 const char *fAllowedLocales; // The list of allowed locales. 121 }; 122 123 124 125 // 126 // Confusable Mappings Data Structures 127 // 128 // For the confusable data, we are essentially implementing a map, 129 // key: a code point 130 // value: a string. Most commonly one char in length, but can be more. 131 // 132 // The keys are stored as a sorted array of 32 bit ints. 133 // bits 0-23 a code point value 134 // bits 24-31 flags 135 // 24: 1 if entry applies to SL table 136 // 25: 1 if entry applies to SA table 137 // 26: 1 if entry applies to ML table 138 // 27: 1 if entry applies to MA table 139 // 28: 1 if there are multiple entries for this code point. 140 // 29-30: length of value string, in UChars. 141 // values are (1, 2, 3, other) 142 // The key table is sorted in ascending code point order. (not on the 143 // 32 bit int value, the flag bits do not participate in the sorting.) 144 // 145 // Lookup is done by means of a binary search in the key table. 146 // 147 // The corresponding values are kept in a parallel array of 16 bit ints. 148 // If the value string is of length 1, it is literally in the value array. 149 // For longer strings, the value array contains an index into the strings table. 150 // 151 // String Table: 152 // The strings table contains all of the value strings (those of length two or greater) 153 // concatentated together into one long UChar (UTF-16) array. 154 // 155 // The array is arranged by length of the strings - all strings of the same length 156 // are stored together. The sections are ordered by length of the strings - 157 // all two char strings first, followed by all of the three Char strings, etc. 158 // 159 // There is no nul character or other mark between adjacent strings. 160 // 161 // String Lengths table 162 // The length of strings from 1 to 3 is flagged in the key table. 163 // For strings of length 4 or longer, the string length table provides a 164 // mapping between an index into the string table and the corresponding length. 165 // Strings of these lengths are rare, so lookup time is not an issue. 166 // Each entry consists of 167 // uint16_t index of the _last_ string with this length 168 // uint16_t the length 169 // 170 171 // Flag bits in the Key entries 172 #define USPOOF_SL_TABLE_FLAG (1<<24) 173 #define USPOOF_SA_TABLE_FLAG (1<<25) 174 #define USPOOF_ML_TABLE_FLAG (1<<26) 175 #define USPOOF_MA_TABLE_FLAG (1<<27) 176 #define USPOOF_KEY_MULTIPLE_VALUES (1<<28) 177 #define USPOOF_KEY_LENGTH_SHIFT 29 178 #define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3) 179 180 181 struct SpoofStringLengthsElement { 182 uint16_t fLastString; // index in string table of last string with this length 183 uint16_t fStrLength; // Length of strings 184 }; 185 186 187 //------------------------------------------------------------------------------- 188 // 189 // ScriptSet - Wrapper class for the Script code bit sets that are part of the 190 // whole script confusable data. 191 // 192 // This class is used both at data build and at run time. 193 // The constructor is only used at build time. 194 // At run time, just point at the prebuilt data and go. 195 // 196 //------------------------------------------------------------------------------- 197 class ScriptSet: public UMemory { 198 public: 199 ScriptSet(); 200 ~ScriptSet(); 201 202 UBool operator == (const ScriptSet &other); 203 ScriptSet & operator = (const ScriptSet &other); 204 205 void Union(const ScriptSet &other); 206 void Union(UScriptCode script); 207 void intersect(const ScriptSet &other); 208 void intersect(UScriptCode script); 209 void setAll(); 210 void resetAll(); 211 int32_t countMembers(); 212 213 private: 214 uint32_t bits[6]; 215 }; 216 217 218 219 220 //------------------------------------------------------------------------------- 221 // 222 // NFKDBuffer A little class to handle the NFKD normalization that is 223 // needed on incoming identifiers to be checked. 224 // Takes care of buffer handling and normalization 225 // 226 // Instances of this class are intended to be stack-allocated. 227 // 228 // TODO: how to map position offsets back to user values? 229 // 230 //-------------------------------------------------------------------------------- 231 class NFKDBuffer: public UMemory { 232 public: 233 NFKDBuffer(const UChar *text, int32_t length, UErrorCode &status); 234 ~NFKDBuffer(); 235 const UChar *getBuffer(); 236 int32_t getLength(); 237 238 private: 239 const UChar *fOriginalText; 240 UChar *fNormalizedText; 241 int32_t fNormalizedTextLength; 242 UChar fSmallBuf[USPOOF_STACK_BUFFER_SIZE]; 243 }; 244 245 246 247 248 249 //------------------------------------------------------------------------------------- 250 // 251 // SpoofData 252 // 253 // A small class that wraps the raw (usually memory mapped) spoof data. 254 // Serves two primary functions: 255 // 1. Convenience. Contains real pointers to the data, to avoid dealing with 256 // the offsets in the raw data. 257 // 2. Reference counting. When a spoof checker is cloned, the raw data is shared 258 // and must be retained until all checkers using the data are closed. 259 // Nothing in this struct includes state that is specific to any particular 260 // USpoofDetector object. 261 // 262 //--------------------------------------------------------------------------------------- 263 class SpoofData: public UMemory { 264 public: 265 static SpoofData *getDefault(UErrorCode &status); // Load standard ICU spoof data. 266 SpoofData(UErrorCode &status); // Create new spoof data wrapper. 267 // Only used when building new data from rules. 268 269 // Constructor for use when creating from prebuilt default data. 270 // A UDataMemory is what the ICU internal data loading functions provide. 271 // The udm is adopted by the SpoofData. 272 SpoofData(UDataMemory *udm, UErrorCode &status); 273 274 // Constructor for use when creating from serialized data. 275 // 276 SpoofData(const void *serializedData, int32_t length, UErrorCode &status); 277 278 // Check raw Spoof Data Version compatibility. 279 // Return TRUE it looks good. 280 static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status); 281 ~SpoofData(); // Destructor not normally used. 282 // Use removeReference() instead. 283 // Reference Counting functions. 284 // Clone of a user-level spoof detector increments the ref count on the data. 285 // Close of a user-level spoof detector decrements the ref count. 286 // If the data is owned by us, it will be deleted when count goes to zero. 287 SpoofData *addReference(); 288 void removeReference(); 289 290 // Reserve space in the raw data. For use by builder when putting together a 291 // new set of data. Init the new storage to zero, to prevent inconsistent 292 // results if it is not all otherwise set by the requester. 293 // Return: 294 // pointer to the new space that was added by this function. 295 void *reserveSpace(int32_t numBytes, UErrorCode &status); 296 297 // initialize the pointers from this object to the raw data. 298 void initPtrs(UErrorCode &status); 299 300 // Reset all fields to an initial state. 301 // Called from the top of all constructors. 302 void reset(); 303 304 SpoofDataHeader *fRawData; // Ptr to the raw memory-mapped data 305 UBool fDataOwned; // True if the raw data is owned, and needs 306 // to be deleted when refcount goes to zero. 307 UDataMemory *fUDM; // If not NULL, our data came from a 308 // UDataMemory, which we must close when 309 // we're done. 310 311 uint32_t fMemLimit; // Limit of available raw data space 312 int32_t fRefCount; 313 314 // Confusable data 315 int32_t *fCFUKeys; 316 uint16_t *fCFUValues; 317 SpoofStringLengthsElement *fCFUStringLengths; 318 UChar *fCFUStrings; 319 320 // Whole Script Confusable Data 321 UTrie2 *fAnyCaseTrie; 322 UTrie2 *fLowerCaseTrie; 323 ScriptSet *fScriptSets; 324 }; 325 326 327 //--------------------------------------------------------------------------------------- 328 // 329 // Raw Binary Data Formats, as loaded from the ICU data file, 330 // or as built by the builder. 331 // 332 //--------------------------------------------------------------------------------------- 333 struct SpoofDataHeader { 334 int32_t fMagic; // (0x8345fdef) 335 uint8_t fFormatVersion[4]; // Data Format. Same as the value in struct UDataInfo 336 // if there is one associated with this data. 337 int32_t fLength; // Total lenght in bytes of this spoof data, 338 // including all sections, not just the header. 339 340 // The following four sections refer to data representing the confusable data 341 // from the Unicode.org data from "confusables.txt" 342 343 int32_t fCFUKeys; // byte offset to Keys table (from SpoofDataHeader *) 344 int32_t fCFUKeysSize; // number of entries in keys table (32 bits each) 345 346 // TODO: change name to fCFUValues, for consistency. 347 int32_t fCFUStringIndex; // byte offset to String Indexes table 348 int32_t fCFUStringIndexSize; // number of entries in String Indexes table (16 bits each) 349 // (number of entries must be same as in Keys table 350 351 int32_t fCFUStringTable; // byte offset of String table 352 int32_t fCFUStringTableLen; // length of string table (in 16 bit UChars) 353 354 int32_t fCFUStringLengths; // byte offset to String Lengths table 355 int32_t fCFUStringLengthsSize; // number of entries in lengths table. (2 x 16 bits each) 356 357 358 // The following sections are for data from confusablesWholeScript.txt 359 360 int32_t fAnyCaseTrie; // byte offset to the serialized Any Case Trie 361 int32_t fAnyCaseTrieLength; // Length (bytes) of the serialized Any Case Trie 362 363 int32_t fLowerCaseTrie; // byte offset to the serialized Lower Case Trie 364 int32_t fLowerCaseTrieLength; // Length (bytes) of the serialized Lower Case Trie 365 366 int32_t fScriptSets; // byte offset to array of ScriptSets 367 int32_t fScriptSetsLength; // Number of ScriptSets (24 bytes each) 368 369 370 // The following sections are for data from xidmodifications.txt 371 372 373 int32_t unused[15]; // Padding, Room for Expansion 374 375 }; 376 377 378 379 380 // 381 // Structure for the Whole Script Confusable Data 382 // See Unicode UAX-39, Unicode Security Mechanisms, for a description of the 383 // Whole Script confusable data 384 // 385 // The data provides mappings from code points to a set of scripts 386 // that contain characters that might be confused with the code point. 387 // There are two mappings, one for lower case only, and one for characters 388 // of any case. 389 // 390 // The actual data consists of a utrie2 to map from a code point to an offset, 391 // and an array of UScriptSets (essentially bit maps) that is indexed 392 // by the offsets obtained from the Trie. 393 // 394 // 395 396 397 U_NAMESPACE_END 398 399 /** 400 * Endianness swap function for binary spoof data. 401 * @internal 402 */ 403 U_CAPI int32_t U_EXPORT2 404 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, 405 UErrorCode *status); 406 407 408 #endif 409 410 #endif /* USPOOFIM_H */ 411 412
