Home | History | Annotate | Download | only in i18n
      1 /*
      2 ***************************************************************************
      3 * Copyright (C) 2008-2010, International Business Machines Corporation
      4 * and others. All Rights Reserved.
      5 ***************************************************************************
      6 *
      7 *  uspoof_impl.h
      8 *
      9 *    Implemenation header for spoof detection
     10 *
     11 */
     12 
     13 #ifndef USPOOFIM_H
     14 #define USPOOFIM_H
     15 
     16 #include "unicode/utypes.h"
     17 #include "unicode/uspoof.h"
     18 #include "utrie2.h"
     19 #include "unicode/uscript.h"
     20 #include "unicode/udata.h"
     21 
     22 
     23 #if !UCONFIG_NO_NORMALIZATION
     24 
     25 #ifdef XP_CPLUSPLUS
     26 
     27 U_NAMESPACE_BEGIN
     28 
     29 // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
     30 //   a single input code point.  This is function of the unicode.org data.
     31 #define USPOOF_MAX_SKELETON_EXPANSION 20
     32 
     33 // The default stack buffer size for copies or conversions or normalizations
     34 // of input strings being checked.  (Used in multiple places.)
     35 #define USPOOF_STACK_BUFFER_SIZE 100
     36 
     37 // Magic number for sanity checking spoof data.
     38 #define USPOOF_MAGIC 0x3845fdef
     39 
     40 class SpoofData;
     41 struct SpoofDataHeader;
     42 struct SpoofStringLengthsElement;
     43 class ScriptSet;
     44 
     45 /**
     46   *  Class SpoofImpl corresponds directly to the plain C API opaque type
     47   *  USpoofChecker.  One can be cast to the other.
     48   */
     49 class SpoofImpl : public UObject  {
     50 public:
     51 	SpoofImpl(SpoofData *data, UErrorCode &status);
     52 	SpoofImpl();
     53 	virtual ~SpoofImpl();
     54 
     55     /** Copy constructor, used by the user level uspoof_clone() function.
     56      */
     57     SpoofImpl(const SpoofImpl &src, UErrorCode &status);
     58 
     59     static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
     60     static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
     61 
     62     /** Get the confusable skeleton transform for a single code point.
     63      *  The result is a string with a length between 1 and 18.
     64      *  @param    tableMask  bit flag specifying which confusable table to use.
     65      *                       One of USPOOF_SL_TABLE_FLAG, USPOOF_MA_TABLE_FLAG, etc.
     66      *  @return   The length in UTF-16 code units of the substition string.
     67      */
     68     int32_t confusableLookup(UChar32 inChar, int32_t tableMask, UChar *destBuf) const;
     69 
     70     /** Set and Get AllowedLocales, implementations of the corresponding API */
     71     void setAllowedLocales(const char *localesList, UErrorCode &status);
     72     const char * getAllowedLocales(UErrorCode &status);
     73 
     74     // Add (union) to the UnicodeSet all of the characters for the scripts used for
     75     // the specified locale.  Part of the implementation of setAllowedLocales.
     76     void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
     77 
     78 
     79     /** parse a hex number.  Untility used by the builders.   */
     80     static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);
     81 
     82     // Implementation for Whole Script tests.
     83     // Return the test bit flag to be ORed into the eventual user return value
     84     //    if a Spoof opportunity is detected.
     85     void wholeScriptCheck(
     86         const UChar *text, int32_t length, ScriptSet *result, UErrorCode &status) const;
     87 
     88     /** Scan a string to determine how many scripts it includes.
     89      * Ignore characters with script=Common and scirpt=Inherited.
     90      * @param    text     The UChar text to be scanned
     91      * @param    length   The length of the input text, -1 for nul termintated.
     92      * @param    pos      An out parameter, set to the first input postion at which
     93      *                    a second script was encountered, ignoring Common and Inherited.
     94      * @param    status   For errors.
     95      * @return            the number of (non-common,inherited) scripts encountered,
     96      *                    clipped to a max of two.
     97      */
     98     int32_t scriptScan(const UChar *text, int32_t length, int32_t &pos, UErrorCode &status) const;
     99 
    100 
    101     // WholeScript and MixedScript check implementation.
    102     //
    103     ScriptSet *WholeScriptCheck(const UChar *text, int32_t length, UErrorCode &status) const;
    104 
    105     static UClassID U_EXPORT2 getStaticClassID(void);
    106     virtual UClassID getDynamicClassID(void) const;
    107 
    108     //
    109     // Data Members
    110     //
    111 
    112     int32_t           fMagic;             // Internal sanity check.
    113     int32_t           fChecks;            // Bit vector of checks to perform.
    114 
    115     SpoofData        *fSpoofData;
    116 
    117     int32_t           fCheckMask;         // Spoof table selector.  f(Check Type)
    118 
    119     const UnicodeSet *fAllowedCharsSet;   // The UnicodeSet of allowed characters.
    120                                           //   for this Spoof Checker.  Defaults to all chars.
    121 
    122     const char       *fAllowedLocales;    // The list of allowed locales.
    123 };
    124 
    125 
    126 
    127 //
    128 //  Confusable Mappings Data Structures
    129 //
    130 //    For the confusable data, we are essentially implementing a map,
    131 //       key:    a code point
    132 //       value:  a string.  Most commonly one char in length, but can be more.
    133 //
    134 //    The keys are stored as a sorted array of 32 bit ints.
    135 //             bits 0-23    a code point value
    136 //             bits 24-31   flags
    137 //                24:  1 if entry applies to SL table
    138 //                25:  1 if entry applies to SA table
    139 //                26:  1 if entry applies to ML table
    140 //                27:  1 if entry applies to MA table
    141 //                28:  1 if there are multiple entries for this code point.
    142 //                29-30:  length of value string, in UChars.
    143 //                         values are (1, 2, 3, other)
    144 //        The key table is sorted in ascending code point order.  (not on the
    145 //        32 bit int value, the flag bits do not participate in the sorting.)
    146 //
    147 //        Lookup is done by means of a binary search in the key table.
    148 //
    149 //    The corresponding values are kept in a parallel array of 16 bit ints.
    150 //        If the value string is of length 1, it is literally in the value array.
    151 //        For longer strings, the value array contains an index into the strings table.
    152 //
    153 //    String Table:
    154 //       The strings table contains all of the value strings (those of length two or greater)
    155 //       concatentated together into one long UChar (UTF-16) array.
    156 //
    157 //       The array is arranged by length of the strings - all strings of the same length
    158 //       are stored together.  The sections are ordered by length of the strings -
    159 //       all two char strings first, followed by all of the three Char strings, etc.
    160 //
    161 //       There is no nul character or other mark between adjacent strings.
    162 //
    163 //    String Lengths table
    164 //       The length of strings from 1 to 3 is flagged in the key table.
    165 //       For strings of length 4 or longer, the string length table provides a
    166 //       mapping between an index into the string table and the corresponding length.
    167 //       Strings of these lengths are rare, so lookup time is not an issue.
    168 //       Each entry consists of
    169 //            uint16_t      index of the _last_ string with this length
    170 //            uint16_t      the length
    171 //
    172 
    173 // Flag bits in the Key entries
    174 #define USPOOF_SL_TABLE_FLAG (1<<24)
    175 #define USPOOF_SA_TABLE_FLAG (1<<25)
    176 #define USPOOF_ML_TABLE_FLAG (1<<26)
    177 #define USPOOF_MA_TABLE_FLAG (1<<27)
    178 #define USPOOF_KEY_MULTIPLE_VALUES (1<<28)
    179 #define USPOOF_KEY_LENGTH_SHIFT 29
    180 #define USPOOF_KEY_LENGTH_FIELD(x) (((x)>>29) & 3)
    181 
    182 
    183 struct SpoofStringLengthsElement {
    184     uint16_t      fLastString;         // index in string table of last string with this length
    185     uint16_t      fStrLength;           // Length of strings
    186 };
    187 
    188 
    189 //-------------------------------------------------------------------------------
    190 //
    191 //  ScriptSet - Wrapper class for the Script code bit sets that are part of the
    192 //              whole script confusable data.
    193 //
    194 //              This class is used both at data build and at run time.
    195 //              The constructor is only used at build time.
    196 //              At run time, just point at the prebuilt data and go.
    197 //
    198 //-------------------------------------------------------------------------------
    199 class ScriptSet: public UMemory {
    200   public:
    201     ScriptSet();
    202     ~ScriptSet();
    203 
    204     UBool operator == (const ScriptSet &other);
    205     ScriptSet & operator = (const ScriptSet &other);
    206 
    207     void Union(const ScriptSet &other);
    208     void Union(UScriptCode script);
    209     void intersect(const ScriptSet &other);
    210     void intersect(UScriptCode script);
    211     void setAll();
    212     void resetAll();
    213     int32_t countMembers();
    214 
    215   private:
    216     uint32_t  bits[6];
    217 };
    218 
    219 
    220 
    221 
    222 //-------------------------------------------------------------------------------
    223 //
    224 //  NFKDBuffer   A little class to handle the NFKD normalization that is
    225 //               needed on incoming identifiers to be checked.
    226 //               Takes care of buffer handling and normalization
    227 //
    228 //               Instances of this class are intended to be stack-allocated.
    229 //
    230 //               TODO:  how to map position offsets back to user values?
    231 //
    232 //--------------------------------------------------------------------------------
    233 class NFKDBuffer: public UMemory {
    234 public:
    235     NFKDBuffer(const UChar *text, int32_t length, UErrorCode &status);
    236     ~NFKDBuffer();
    237     const UChar *getBuffer();
    238     int32_t getLength();
    239 
    240   private:
    241     const UChar *fOriginalText;
    242     UChar       *fNormalizedText;
    243     int32_t      fNormalizedTextLength;
    244     UChar        fSmallBuf[USPOOF_STACK_BUFFER_SIZE];
    245 };
    246 
    247 
    248 
    249 
    250 
    251 //-------------------------------------------------------------------------------------
    252 //
    253 //  SpoofData
    254 //
    255 //    A small class that wraps the raw (usually memory mapped) spoof data.
    256 //    Serves two primary functions:
    257 //      1.  Convenience.  Contains real pointers to the data, to avoid dealing with
    258 //          the offsets in the raw data.
    259 //      2.  Reference counting.  When a spoof checker is cloned, the raw data is shared
    260 //          and must be retained until all checkers using the data are closed.
    261 //    Nothing in this struct includes state that is specific to any particular
    262 //    USpoofDetector object.
    263 //
    264 //---------------------------------------------------------------------------------------
    265 class SpoofData: public UMemory {
    266   public:
    267     static SpoofData *getDefault(UErrorCode &status);   // Load standard ICU spoof data.
    268     SpoofData(UErrorCode &status);   // Create new spoof data wrapper.
    269                                      // Only used when building new data from rules.
    270 
    271     // Constructor for use when creating from prebuilt default data.
    272     //   A UDataMemory is what the ICU internal data loading functions provide.
    273     //   The udm is adopted by the SpoofData.
    274     SpoofData(UDataMemory *udm, UErrorCode &status);
    275 
    276     // Constructor for use when creating from serialized data.
    277     //
    278     SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
    279 
    280     //  Check raw Spoof Data Version compatibility.
    281     //  Return TRUE it looks good.
    282     static UBool validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status);
    283     ~SpoofData();                    // Destructor not normally used.
    284                                      // Use removeReference() instead.
    285     // Reference Counting functions.
    286     //    Clone of a user-level spoof detector increments the ref count on the data.
    287     //    Close of a user-level spoof detector decrements the ref count.
    288     //    If the data is owned by us, it will be deleted when count goes to zero.
    289     SpoofData *addReference();
    290     void removeReference();
    291 
    292     // Reserve space in the raw data.  For use by builder when putting together a
    293     //   new set of data.  Init the new storage to zero, to prevent inconsistent
    294     //   results if it is not all otherwise set by the requester.
    295     //  Return:
    296     //    pointer to the new space that was added by this function.
    297     void *reserveSpace(int32_t numBytes, UErrorCode &status);
    298 
    299     // initialize the pointers from this object to the raw data.
    300     void initPtrs(UErrorCode &status);
    301 
    302     // Reset all fields to an initial state.
    303     // Called from the top of all constructors.
    304     void reset();
    305 
    306     SpoofDataHeader             *fRawData;          // Ptr to the raw memory-mapped data
    307     UBool                       fDataOwned;         // True if the raw data is owned, and needs
    308                                                     //  to be deleted when refcount goes to zero.
    309     UDataMemory                 *fUDM;              // If not NULL, our data came from a
    310                                                     //   UDataMemory, which we must close when
    311                                                     //   we're done.
    312 
    313     uint32_t                    fMemLimit;          // Limit of available raw data space
    314     int32_t                     fRefCount;
    315 
    316     // Confusable data
    317     int32_t                     *fCFUKeys;
    318     uint16_t                    *fCFUValues;
    319     SpoofStringLengthsElement   *fCFUStringLengths;
    320     UChar                       *fCFUStrings;
    321 
    322     // Whole Script Confusable Data
    323     UTrie2                      *fAnyCaseTrie;
    324     UTrie2                      *fLowerCaseTrie;
    325     ScriptSet                   *fScriptSets;
    326     };
    327 
    328 
    329 //---------------------------------------------------------------------------------------
    330 //
    331 //  Raw Binary Data Formats, as loaded from the ICU data file,
    332 //    or as built by the builder.
    333 //
    334 //---------------------------------------------------------------------------------------
    335 struct SpoofDataHeader {
    336     int32_t       fMagic;                // (0x3845fdef)
    337     uint8_t       fFormatVersion[4];     // Data Format. Same as the value in struct UDataInfo
    338                                          //   if there is one associated with this data.
    339     int32_t       fLength;               // Total lenght in bytes of this spoof data,
    340                                          //   including all sections, not just the header.
    341 
    342     // The following four sections refer to data representing the confusable data
    343     //   from the Unicode.org data from "confusables.txt"
    344 
    345     int32_t       fCFUKeys;               // byte offset to Keys table (from SpoofDataHeader *)
    346     int32_t       fCFUKeysSize;           // number of entries in keys table  (32 bits each)
    347 
    348     // TODO: change name to fCFUValues, for consistency.
    349     int32_t       fCFUStringIndex;        // byte offset to String Indexes table
    350     int32_t       fCFUStringIndexSize;    // number of entries in String Indexes table (16 bits each)
    351                                           //     (number of entries must be same as in Keys table
    352 
    353     int32_t       fCFUStringTable;        // byte offset of String table
    354     int32_t       fCFUStringTableLen;     // length of string table (in 16 bit UChars)
    355 
    356     int32_t       fCFUStringLengths;      // byte offset to String Lengths table
    357     int32_t       fCFUStringLengthsSize;  // number of entries in lengths table. (2 x 16 bits each)
    358 
    359 
    360     // The following sections are for data from confusablesWholeScript.txt
    361 
    362     int32_t       fAnyCaseTrie;           // byte offset to the serialized Any Case Trie
    363     int32_t       fAnyCaseTrieLength;     // Length (bytes) of the serialized Any Case Trie
    364 
    365     int32_t       fLowerCaseTrie;         // byte offset to the serialized Lower Case Trie
    366     int32_t       fLowerCaseTrieLength;   // Length (bytes) of the serialized Lower Case Trie
    367 
    368     int32_t       fScriptSets;            // byte offset to array of ScriptSets
    369     int32_t       fScriptSetsLength;      // Number of ScriptSets (24 bytes each)
    370 
    371 
    372     // The following sections are for data from xidmodifications.txt
    373 
    374 
    375     int32_t       unused[15];              // Padding, Room for Expansion
    376 
    377  };
    378 
    379 
    380 
    381 
    382 //
    383 //  Structure for the Whole Script Confusable Data
    384 //    See Unicode UAX-39, Unicode Security Mechanisms, for a description of the
    385 //    Whole Script confusable data
    386 //
    387 //  The data provides mappings from code points to a set of scripts
    388 //    that contain characters that might be confused with the code point.
    389 //  There are two mappings, one for lower case only, and one for characters
    390 //    of any case.
    391 //
    392 //  The actual data consists of a utrie2 to map from a code point to an offset,
    393 //  and an array of UScriptSets (essentially bit maps) that is indexed
    394 //  by the offsets obtained from the Trie.
    395 //
    396 //
    397 
    398 
    399 U_NAMESPACE_END
    400 #endif /* XP_CPLUSPLUS */
    401 
    402 /**
    403   * Endianness swap function for binary spoof data.
    404   * @internal
    405   */
    406 U_CAPI int32_t U_EXPORT2
    407 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
    408             UErrorCode *status);
    409 
    410 
    411 #endif
    412 
    413 #endif  /* USPOOFIM_H */
    414 
    415