Home | History | Annotate | Download | only in i18n
      1 // Copyright (C) 2016 and later: Unicode, Inc. and others.
      2 // License & terms of use: http://www.unicode.org/copyright.html
      3 /*
      4 ***************************************************************************
      5 * Copyright (C) 2008-2013, International Business Machines Corporation
      6 * and others. All Rights Reserved.
      7 ***************************************************************************
      8 *
      9 *  uspoof_impl.h
     10 *
     11 *    Implemenation header for spoof detection
     12 *
     13 */
     14 
     15 #ifndef USPOOFIM_H
     16 #define USPOOFIM_H
     17 
     18 #include "uassert.h"
     19 #include "unicode/utypes.h"
     20 #include "unicode/uspoof.h"
     21 #include "unicode/uscript.h"
     22 #include "unicode/udata.h"
     23 
     24 #include "utrie2.h"
     25 
     26 #if !UCONFIG_NO_NORMALIZATION
     27 
     28 #ifdef __cplusplus
     29 
     30 U_NAMESPACE_BEGIN
     31 
     32 // The maximium length (in UTF-16 UChars) of the skeleton replacement string resulting from
     33 //   a single input code point.  This is function of the unicode.org data.
     34 #define USPOOF_MAX_SKELETON_EXPANSION 20
     35 
     36 // The default stack buffer size for copies or conversions or normalizations
     37 // of input strings being checked.  (Used in multiple places.)
     38 #define USPOOF_STACK_BUFFER_SIZE 100
     39 
     40 // Magic number for sanity checking spoof data.
     41 #define USPOOF_MAGIC 0x3845fdef
     42 
     43 // Magic number for sanity checking spoof checkers.
     44 #define USPOOF_CHECK_MAGIC 0x2734ecde
     45 
     46 class ScriptSet;
     47 class SpoofData;
     48 struct SpoofDataHeader;
     49 class ConfusableDataUtils;
     50 
     51 /**
     52   *  Class SpoofImpl corresponds directly to the plain C API opaque type
     53   *  USpoofChecker.  One can be cast to the other.
     54   */
     55 class SpoofImpl : public UObject  {
     56 public:
     57     SpoofImpl(SpoofData *data, UErrorCode& status);
     58     SpoofImpl(UErrorCode& status);
     59     SpoofImpl();
     60     void construct(UErrorCode& status);
     61     virtual ~SpoofImpl();
     62 
     63     /** Copy constructor, used by the user level uspoof_clone() function.
     64      */
     65     SpoofImpl(const SpoofImpl &src, UErrorCode &status);
     66 
     67     USpoofChecker *asUSpoofChecker();
     68     static SpoofImpl *validateThis(USpoofChecker *sc, UErrorCode &status);
     69     static const SpoofImpl *validateThis(const USpoofChecker *sc, UErrorCode &status);
     70 
     71     /** Set and Get AllowedLocales, implementations of the corresponding API */
     72     void setAllowedLocales(const char *localesList, UErrorCode &status);
     73     const char * getAllowedLocales(UErrorCode &status);
     74 
     75     // Add (union) to the UnicodeSet all of the characters for the scripts used for
     76     // the specified locale.  Part of the implementation of setAllowedLocales.
     77     void addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status);
     78 
     79     // Functions implementing the features of UTS 39 section 5.
     80     static void getAugmentedScriptSet(UChar32 codePoint, ScriptSet& result, UErrorCode& status);
     81     void getResolvedScriptSet(const UnicodeString& input, ScriptSet& result, UErrorCode& status) const;
     82     void getResolvedScriptSetWithout(const UnicodeString& input, UScriptCode script, ScriptSet& result, UErrorCode& status) const;
     83     void getNumerics(const UnicodeString& input, UnicodeSet& result, UErrorCode& status) const;
     84     URestrictionLevel getRestrictionLevel(const UnicodeString& input, UErrorCode& status) const;
     85 
     86     /** parse a hex number.  Untility used by the builders.   */
     87     static UChar32 ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status);
     88 
     89     static UClassID U_EXPORT2 getStaticClassID(void);
     90     virtual UClassID getDynamicClassID(void) const;
     91 
     92     //
     93     // Data Members
     94     //
     95 
     96     int32_t           fMagic;             // Internal sanity check.
     97     int32_t           fChecks;            // Bit vector of checks to perform.
     98 
     99     SpoofData        *fSpoofData;
    100 
    101     const UnicodeSet *fAllowedCharsSet;   // The UnicodeSet of allowed characters.
    102                                           //   for this Spoof Checker.  Defaults to all chars.
    103 
    104     const char       *fAllowedLocales;    // The list of allowed locales.
    105     URestrictionLevel fRestrictionLevel;  // The maximum restriction level for an acceptable identifier.
    106 };
    107 
    108 /**
    109  *  Class CheckResult corresponds directly to the plain C API opaque type
    110  *  USpoofCheckResult.  One can be cast to the other.
    111  */
    112 class CheckResult : public UObject {
    113 public:
    114     CheckResult();
    115     virtual ~CheckResult();
    116 
    117     USpoofCheckResult *asUSpoofCheckResult();
    118     static CheckResult *validateThis(USpoofCheckResult *ptr, UErrorCode &status);
    119     static const CheckResult *validateThis(const USpoofCheckResult *ptr, UErrorCode &status);
    120 
    121     void clear();
    122 
    123     // Used to convert this CheckResult to the older int32_t return value API
    124     int32_t toCombinedBitmask(int32_t expectedChecks);
    125 
    126     // Data Members
    127     int32_t fMagic;                        // Internal sanity check.
    128     int32_t fChecks;                       // Bit vector of checks that were failed.
    129     UnicodeSet fNumerics;                  // Set of numerics found in the string.
    130     URestrictionLevel fRestrictionLevel;   // The restriction level of the string.
    131 };
    132 
    133 
    134 //
    135 //  Confusable Mappings Data Structures, version 2.0
    136 //
    137 //    For the confusable data, we are essentially implementing a map,
    138 //       key:    a code point
    139 //       value:  a string.  Most commonly one char in length, but can be more.
    140 //
    141 //    The keys are stored as a sorted array of 32 bit ints.
    142 //             bits 0-23    a code point value
    143 //             bits 24-31   length of value string, in UChars (between 1 and 256 UChars).
    144 //        The key table is sorted in ascending code point order.  (not on the
    145 //        32 bit int value, the flag bits do not participate in the sorting.)
    146 //
    147 //        Lookup is done by means of a binary search in the key table.
    148 //
    149 //    The corresponding values are kept in a parallel array of 16 bit ints.
    150 //        If the value string is of length 1, it is literally in the value array.
    151 //        For longer strings, the value array contains an index into the strings table.
    152 //
    153 //    String Table:
    154 //       The strings table contains all of the value strings (those of length two or greater)
    155 //       concatentated together into one long UChar (UTF-16) array.
    156 //
    157 //       There is no nul character or other mark between adjacent strings.
    158 //
    159 //----------------------------------------------------------------------------
    160 //
    161 //  Changes from format version 1 to format version 2:
    162 //      1) Removal of the whole-script confusable data tables.
    163 //      2) Removal of the SL/SA/ML/MA and multi-table flags in the key bitmask.
    164 //      3) Expansion of string length value in the key bitmask from 2 bits to 8 bits.
    165 //      4) Removal of the string lengths table since 8 bits is sufficient for the
    166 //         lengths of all entries in confusables.txt.
    167 
    168 
    169 
    170 // Internal functions for manipulating confusable data table keys
    171 #define USPOOF_CONFUSABLE_DATA_FORMAT_VERSION 2  // version for ICU 58
    172 class ConfusableDataUtils {
    173 public:
    174     inline static UChar32 keyToCodePoint(int32_t key) {
    175         return key & 0x00ffffff;
    176     }
    177     inline static int32_t keyToLength(int32_t key) {
    178         return ((key & 0xff000000) >> 24) + 1;
    179     }
    180     inline static int32_t codePointAndLengthToKey(UChar32 codePoint, int32_t length) {
    181         U_ASSERT((codePoint & 0x00ffffff) == codePoint);
    182         U_ASSERT(length <= 256);
    183         return codePoint | ((length - 1) << 24);
    184     }
    185 };
    186 
    187 
    188 //-------------------------------------------------------------------------------------
    189 //
    190 //  SpoofData
    191 //
    192 //    A small class that wraps the raw (usually memory mapped) spoof data.
    193 //    Serves two primary functions:
    194 //      1.  Convenience.  Contains real pointers to the data, to avoid dealing with
    195 //          the offsets in the raw data.
    196 //      2.  Reference counting.  When a spoof checker is cloned, the raw data is shared
    197 //          and must be retained until all checkers using the data are closed.
    198 //    Nothing in this struct includes state that is specific to any particular
    199 //    USpoofDetector object.
    200 //
    201 //---------------------------------------------------------------------------------------
    202 class SpoofData: public UMemory {
    203   public:
    204     static SpoofData* getDefault(UErrorCode &status);   // Get standard ICU spoof data.
    205     static void releaseDefault();   // Cleanup reference to default spoof data.
    206 
    207     SpoofData(UErrorCode &status);   // Create new spoof data wrapper.
    208                                      // Only used when building new data from rules.
    209 
    210     // Constructor for use when creating from prebuilt default data.
    211     //   A UDataMemory is what the ICU internal data loading functions provide.
    212     //   The udm is adopted by the SpoofData.
    213     SpoofData(UDataMemory *udm, UErrorCode &status);
    214 
    215     // Constructor for use when creating from serialized data.
    216     //
    217     SpoofData(const void *serializedData, int32_t length, UErrorCode &status);
    218 
    219     //  Check raw Spoof Data Version compatibility.
    220     //  Return TRUE it looks good.
    221     UBool validateDataVersion(UErrorCode &status) const;
    222 
    223     ~SpoofData();                    // Destructor not normally used.
    224                                      // Use removeReference() instead.
    225     // Reference Counting functions.
    226     //    Clone of a user-level spoof detector increments the ref count on the data.
    227     //    Close of a user-level spoof detector decrements the ref count.
    228     //    If the data is owned by us, it will be deleted when count goes to zero.
    229     SpoofData *addReference();
    230     void removeReference();
    231 
    232     // Reset all fields to an initial state.
    233     // Called from the top of all constructors.
    234     void reset();
    235 
    236     // Copy this instance's raw data buffer to the specified address.
    237     int32_t serialize(void *buf, int32_t capacity, UErrorCode &status) const;
    238 
    239     // Get the total number of bytes of data backed by this SpoofData.
    240     // Not to be confused with length, which returns the number of confusable entries.
    241     int32_t size() const;
    242 
    243     // Get the confusable skeleton transform for a single code point.
    244     // The result is a string with a length between 1 and 18 as of Unicode 9.
    245     // This is the main public endpoint for this class.
    246     // @return   The length in UTF-16 code units of the substition string.
    247     int32_t confusableLookup(UChar32 inChar, UnicodeString &dest) const;
    248 
    249     // Get the number of confusable entries in this SpoofData.
    250     int32_t length() const;
    251 
    252     // Get the code point (key) at the specified index.
    253     UChar32 codePointAt(int32_t index) const;
    254 
    255     // Get the confusable skeleton (value) at the specified index.
    256     // Append it to the specified UnicodeString&.
    257     // @return   The length in UTF-16 code units of the skeleton string.
    258     int32_t appendValueTo(int32_t index, UnicodeString& dest) const;
    259 
    260   private:
    261     // Reserve space in the raw data.  For use by builder when putting together a
    262     //   new set of data.  Init the new storage to zero, to prevent inconsistent
    263     //   results if it is not all otherwise set by the requester.
    264     //  Return:
    265     //    pointer to the new space that was added by this function.
    266     void *reserveSpace(int32_t numBytes, UErrorCode &status);
    267 
    268     // initialize the pointers from this object to the raw data.
    269     void initPtrs(UErrorCode &status);
    270 
    271     SpoofDataHeader             *fRawData;          // Ptr to the raw memory-mapped data
    272     UBool                       fDataOwned;         // True if the raw data is owned, and needs
    273                                                     //  to be deleted when refcount goes to zero.
    274     UDataMemory                 *fUDM;              // If not NULL, our data came from a
    275                                                     //   UDataMemory, which we must close when
    276                                                     //   we are done.
    277 
    278     uint32_t                    fMemLimit;          // Limit of available raw data space
    279     u_atomic_int32_t            fRefCount;
    280 
    281     // Confusable data
    282     int32_t                     *fCFUKeys;
    283     uint16_t                    *fCFUValues;
    284     UChar                       *fCFUStrings;
    285 
    286     friend class ConfusabledataBuilder;
    287 };
    288 
    289 //---------------------------------------------------------------------------------------
    290 //
    291 //  Raw Binary Data Formats, as loaded from the ICU data file,
    292 //    or as built by the builder.
    293 //
    294 //---------------------------------------------------------------------------------------
    295 struct SpoofDataHeader {
    296     int32_t       fMagic;                // (0x3845fdef)
    297     uint8_t       fFormatVersion[4];     // Data Format. Same as the value in struct UDataInfo
    298                                          //   if there is one associated with this data.
    299     int32_t       fLength;               // Total lenght in bytes of this spoof data,
    300                                          //   including all sections, not just the header.
    301 
    302     // The following four sections refer to data representing the confusable data
    303     //   from the Unicode.org data from "confusables.txt"
    304 
    305     int32_t       fCFUKeys;               // byte offset to Keys table (from SpoofDataHeader *)
    306     int32_t       fCFUKeysSize;           // number of entries in keys table  (32 bits each)
    307 
    308     // TODO: change name to fCFUValues, for consistency.
    309     int32_t       fCFUStringIndex;        // byte offset to String Indexes table
    310     int32_t       fCFUStringIndexSize;    // number of entries in String Indexes table (16 bits each)
    311                                           //     (number of entries must be same as in Keys table
    312 
    313     int32_t       fCFUStringTable;        // byte offset of String table
    314     int32_t       fCFUStringTableLen;     // length of string table (in 16 bit UChars)
    315 
    316     // The following sections are for data from xidmodifications.txt
    317 
    318     int32_t       unused[15];              // Padding, Room for Expansion
    319 };
    320 
    321 
    322 
    323 U_NAMESPACE_END
    324 #endif /* __cplusplus */
    325 
    326 /**
    327   * Endianness swap function for binary spoof data.
    328   * @internal
    329   */
    330 U_CAPI int32_t U_EXPORT2
    331 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
    332             UErrorCode *status);
    333 
    334 
    335 #endif
    336 
    337 #endif  /* USPOOFIM_H */
    338 
    339