Home | History | Annotate | Download | only in i18n
      1 /*
      2 **********************************************************************
      3 *   Copyright (C) 2008-2010, International Business Machines
      4 *   Corporation and others.  All Rights Reserved.
      5 **********************************************************************
      6 */
      7 
      8 #include "unicode/utypes.h"
      9 #include "unicode/uspoof.h"
     10 #include "unicode/unorm.h"
     11 #include "unicode/uchar.h"
     12 #include "unicode/uniset.h"
     13 #include "utrie2.h"
     14 #include "cmemory.h"
     15 #include "cstring.h"
     16 #include "udatamem.h"
     17 #include "umutex.h"
     18 #include "udataswp.h"
     19 #include "uassert.h"
     20 #include "uspoof_impl.h"
     21 
     22 #if !UCONFIG_NO_NORMALIZATION
     23 
     24 
     25 U_NAMESPACE_BEGIN
     26 
     27 UOBJECT_DEFINE_RTTI_IMPLEMENTATION(SpoofImpl)
     28 
     29 SpoofImpl::SpoofImpl(SpoofData *data, UErrorCode &status) :
     30     fMagic(0), fSpoofData(NULL), fAllowedCharsSet(NULL) , fAllowedLocales(NULL) {
     31     if (U_FAILURE(status)) {
     32         return;
     33     }
     34     fMagic = USPOOF_MAGIC;
     35     fSpoofData = data;
     36     fChecks = USPOOF_ALL_CHECKS;
     37     UnicodeSet *allowedCharsSet = new UnicodeSet(0, 0x10ffff);
     38     if (allowedCharsSet == NULL) {
     39         status = U_MEMORY_ALLOCATION_ERROR;
     40     }
     41     allowedCharsSet->freeze();
     42     fAllowedCharsSet = allowedCharsSet;
     43     fAllowedLocales  = uprv_strdup("");
     44 }
     45 
     46 
     47 SpoofImpl::SpoofImpl() {
     48     fMagic = USPOOF_MAGIC;
     49     fSpoofData = NULL;
     50     fChecks = USPOOF_ALL_CHECKS;
     51     UnicodeSet *allowedCharsSet = new UnicodeSet(0, 0x10ffff);
     52     allowedCharsSet->freeze();
     53     fAllowedCharsSet = allowedCharsSet;
     54     fAllowedLocales  = uprv_strdup("");
     55 }
     56 
     57 
     58 // Copy Constructor, used by the user level clone() function.
     59 SpoofImpl::SpoofImpl(const SpoofImpl &src, UErrorCode &status)  :
     60     fMagic(0), fSpoofData(NULL), fAllowedCharsSet(NULL) {
     61     if (U_FAILURE(status)) {
     62         return;
     63     }
     64     fMagic = src.fMagic;
     65     fChecks = src.fChecks;
     66     if (src.fSpoofData != NULL) {
     67         fSpoofData = src.fSpoofData->addReference();
     68     }
     69     fCheckMask = src.fCheckMask;
     70     fAllowedCharsSet = static_cast<const UnicodeSet *>(src.fAllowedCharsSet->clone());
     71     if (fAllowedCharsSet == NULL) {
     72         status = U_MEMORY_ALLOCATION_ERROR;
     73     }
     74     fAllowedLocales = uprv_strdup(src.fAllowedLocales);
     75 }
     76 
     77 SpoofImpl::~SpoofImpl() {
     78     fMagic = 0;                // head off application errors by preventing use of
     79                                //    of deleted objects.
     80     if (fSpoofData != NULL) {
     81         fSpoofData->removeReference();   // Will delete if refCount goes to zero.
     82     }
     83     delete fAllowedCharsSet;
     84     uprv_free((void *)fAllowedLocales);
     85 }
     86 
     87 //
     88 //  Incoming parameter check on Status and the SpoofChecker object
     89 //    received from the C API.
     90 //
     91 const SpoofImpl *SpoofImpl::validateThis(const USpoofChecker *sc, UErrorCode &status) {
     92     if (U_FAILURE(status)) {
     93         return NULL;
     94     }
     95     if (sc == NULL) {
     96         status = U_ILLEGAL_ARGUMENT_ERROR;
     97         return NULL;
     98     };
     99     SpoofImpl *This = (SpoofImpl *)sc;
    100     if (This->fMagic != USPOOF_MAGIC ||
    101         This->fSpoofData == NULL) {
    102         status = U_INVALID_FORMAT_ERROR;
    103         return NULL;
    104     }
    105     if (!SpoofData::validateDataVersion(This->fSpoofData->fRawData, status)) {
    106         return NULL;
    107     }
    108     return This;
    109 }
    110 
    111 SpoofImpl *SpoofImpl::validateThis(USpoofChecker *sc, UErrorCode &status) {
    112     return const_cast<SpoofImpl *>
    113         (SpoofImpl::validateThis(const_cast<const USpoofChecker *>(sc), status));
    114 }
    115 
    116 
    117 
    118 //--------------------------------------------------------------------------------------
    119 //
    120 //  confusableLookup()    This is the heart of the confusable skeleton generation
    121 //                        implementation.
    122 //
    123 //                        Given a source character, produce the corresponding
    124 //                        replacement character(s)
    125 //
    126 //---------------------------------------------------------------------------------------
    127 int32_t SpoofImpl::confusableLookup(UChar32 inChar, int32_t tableMask, UChar *destBuf) const {
    128 
    129     // Binary search the spoof data key table for the inChar
    130     int32_t  *low   = fSpoofData->fCFUKeys;
    131     int32_t  *mid   = NULL;
    132     int32_t  *limit = low + fSpoofData->fRawData->fCFUKeysSize;
    133     UChar32   midc;
    134     do {
    135         int32_t delta = ((int32_t)(limit-low))/2;
    136         mid = low + delta;
    137         midc = *mid & 0x1fffff;
    138         if (inChar == midc) {
    139             goto foundChar;
    140         } else if (inChar < midc) {
    141             limit = mid;
    142         } else {
    143             low = mid;
    144         }
    145     } while (low < limit-1);
    146     mid = low;
    147     midc = *mid & 0x1fffff;
    148     if (inChar != midc) {
    149         // Char not found.  It maps to itself.
    150         int i = 0;
    151         U16_APPEND_UNSAFE(destBuf, i, inChar)
    152         return i;
    153     }
    154   foundChar:
    155     int32_t keyFlags = *mid & 0xff000000;
    156     if ((keyFlags & tableMask) == 0) {
    157         // We found the right key char, but the entry doesn't pertain to the
    158         //  table we need.  See if there is an adjacent key that does
    159         if (keyFlags & USPOOF_KEY_MULTIPLE_VALUES) {
    160             int32_t *altMid;
    161             for (altMid = mid-1; (*altMid&0x00ffffff) == inChar; altMid--) {
    162                 keyFlags = *altMid & 0xff000000;
    163                 if (keyFlags & tableMask) {
    164                     mid = altMid;
    165                     goto foundKey;
    166                 }
    167             }
    168             for (altMid = mid+1; (*altMid&0x00ffffff) == inChar; altMid++) {
    169                 keyFlags = *altMid & 0xff000000;
    170                 if (keyFlags & tableMask) {
    171                     mid = altMid;
    172                     goto foundKey;
    173                 }
    174             }
    175         }
    176         // No key entry for this char & table.
    177         // The input char maps to itself.
    178         int i = 0;
    179         U16_APPEND_UNSAFE(destBuf, i, inChar)
    180         return i;
    181     }
    182 
    183   foundKey:
    184     int32_t  stringLen = USPOOF_KEY_LENGTH_FIELD(keyFlags) + 1;
    185     int32_t keyTableIndex = (int32_t)(mid - fSpoofData->fCFUKeys);
    186 
    187     // Value is either a UChar  (for strings of length 1) or
    188     //                 an index into the string table (for longer strings)
    189     uint16_t value = fSpoofData->fCFUValues[keyTableIndex];
    190     if (stringLen == 1) {
    191         destBuf[0] = value;
    192         return 1;
    193     }
    194 
    195     // String length of 4 from the above lookup is used for all strings of length >= 4.
    196     // For these, get the real length from the string lengths table,
    197     //   which maps string table indexes to lengths.
    198     //   All strings of the same length are stored contiguously in the string table.
    199     //   'value' from the lookup above is the starting index for the desired string.
    200 
    201     int32_t ix;
    202     if (stringLen == 4) {
    203         int32_t stringLengthsLimit = fSpoofData->fRawData->fCFUStringLengthsSize;
    204         for (ix = 0; ix < stringLengthsLimit; ix++) {
    205             if (fSpoofData->fCFUStringLengths[ix].fLastString >= value) {
    206                 stringLen = fSpoofData->fCFUStringLengths[ix].fStrLength;
    207                 break;
    208             }
    209         }
    210         U_ASSERT(ix < stringLengthsLimit);
    211     }
    212 
    213     U_ASSERT(value + stringLen < fSpoofData->fRawData->fCFUStringTableLen);
    214     UChar *src = &fSpoofData->fCFUStrings[value];
    215     for (ix=0; ix<stringLen; ix++) {
    216         destBuf[ix] = src[ix];
    217     }
    218     return stringLen;
    219 }
    220 
    221 
    222 //---------------------------------------------------------------------------------------
    223 //
    224 //  wholeScriptCheck()
    225 //
    226 //      Input text is already normalized to NFKD
    227 //      Return the set of scripts, each of which can represent something that is
    228 //             confusable with the input text.  The script of the input text
    229 //             is included; input consisting of characters from a single script will
    230 //             always produce a result consisting of a set containing that script.
    231 //
    232 //---------------------------------------------------------------------------------------
    233 void SpoofImpl::wholeScriptCheck(
    234     const UChar *text, int32_t length, ScriptSet *result, UErrorCode &status) const {
    235 
    236     int32_t       inputIdx = 0;
    237     UChar32       c;
    238 
    239     UTrie2 *table =
    240         (fChecks & USPOOF_ANY_CASE) ? fSpoofData->fAnyCaseTrie : fSpoofData->fLowerCaseTrie;
    241     result->setAll();
    242     while (inputIdx < length) {
    243         U16_NEXT(text, inputIdx, length, c);
    244         uint32_t index = utrie2_get32(table, c);
    245         if (index == 0) {
    246             // No confusables in another script for this char.
    247             // TODO:  we should change the data to have sets with just the single script
    248             //        bit for the script of this char.  Gets rid of this special case.
    249             //        Until then, grab the script from the char and intersect it with the set.
    250             UScriptCode cpScript = uscript_getScript(c, &status);
    251             U_ASSERT(cpScript > USCRIPT_INHERITED);
    252             result->intersect(cpScript);
    253         } else if (index == 1) {
    254             // Script == Common or Inherited.  Nothing to do.
    255         } else {
    256             result->intersect(fSpoofData->fScriptSets[index]);
    257         }
    258     }
    259 }
    260 
    261 
    262 void SpoofImpl::setAllowedLocales(const char *localesList, UErrorCode &status) {
    263     UnicodeSet    allowedChars;
    264     UnicodeSet    *tmpSet = NULL;
    265     const char    *locStart = localesList;
    266     const char    *locEnd = NULL;
    267     const char    *localesListEnd = localesList + uprv_strlen(localesList);
    268     int32_t        localeListCount = 0;   // Number of locales provided by caller.
    269 
    270     // Loop runs once per locale from the localesList, a comma separated list of locales.
    271     do {
    272         locEnd = uprv_strchr(locStart, ',');
    273         if (locEnd == NULL) {
    274             locEnd = localesListEnd;
    275         }
    276         while (*locStart == ' ') {
    277             locStart++;
    278         }
    279         const char *trimmedEnd = locEnd-1;
    280         while (trimmedEnd > locStart && *trimmedEnd == ' ') {
    281             trimmedEnd--;
    282         }
    283         if (trimmedEnd <= locStart) {
    284             break;
    285         }
    286         const char *locale = uprv_strndup(locStart, (int32_t)(trimmedEnd + 1 - locStart));
    287         localeListCount++;
    288 
    289         // We have one locale from the locales list.
    290         // Add the script chars for this locale to the accumulating set of allowed chars.
    291         // If the locale is no good, we will be notified back via status.
    292         addScriptChars(locale, &allowedChars, status);
    293         uprv_free((void *)locale);
    294         if (U_FAILURE(status)) {
    295             break;
    296         }
    297         locStart = locEnd + 1;
    298     } while (locStart < localesListEnd);
    299 
    300     // If our caller provided an empty list of locales, we disable the allowed characters checking
    301     if (localeListCount == 0) {
    302         uprv_free((void *)fAllowedLocales);
    303         fAllowedLocales = uprv_strdup("");
    304         tmpSet = new UnicodeSet(0, 0x10ffff);
    305         if (fAllowedLocales == NULL || tmpSet == NULL) {
    306             status = U_MEMORY_ALLOCATION_ERROR;
    307             return;
    308         }
    309         tmpSet->freeze();
    310         delete fAllowedCharsSet;
    311         fAllowedCharsSet = tmpSet;
    312         fCheckMask &= ~USPOOF_CHAR_LIMIT;
    313         return;
    314     }
    315 
    316 
    317     // Add all common and inherited characters to the set of allowed chars.
    318     UnicodeSet tempSet;
    319     tempSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_COMMON, status);
    320     allowedChars.addAll(tempSet);
    321     tempSet.applyIntPropertyValue(UCHAR_SCRIPT, USCRIPT_INHERITED, status);
    322     allowedChars.addAll(tempSet);
    323 
    324     // If anything went wrong, we bail out without changing
    325     // the state of the spoof checker.
    326     if (U_FAILURE(status)) {
    327         return;
    328     }
    329 
    330     // Store the updated spoof checker state.
    331     tmpSet = static_cast<UnicodeSet *>(allowedChars.clone());
    332     const char *tmpLocalesList = uprv_strdup(localesList);
    333     if (tmpSet == NULL || tmpLocalesList == NULL) {
    334         status = U_MEMORY_ALLOCATION_ERROR;
    335         return;
    336     }
    337     uprv_free((void *)fAllowedLocales);
    338     fAllowedLocales = tmpLocalesList;
    339     tmpSet->freeze();
    340     delete fAllowedCharsSet;
    341     fAllowedCharsSet = tmpSet;
    342     fCheckMask |= USPOOF_CHAR_LIMIT;
    343 }
    344 
    345 
    346 const char * SpoofImpl::getAllowedLocales(UErrorCode &/*status*/) {
    347     return fAllowedLocales;
    348 }
    349 
    350 
    351 // Given a locale (a language), add all the characters from all of the scripts used with that language
    352 // to the allowedChars UnicodeSet
    353 
    354 void SpoofImpl::addScriptChars(const char *locale, UnicodeSet *allowedChars, UErrorCode &status) {
    355     UScriptCode scripts[30];
    356 
    357     int32_t numScripts = uscript_getCode(locale, scripts, sizeof(scripts)/sizeof(UScriptCode), &status);
    358     if (U_FAILURE(status)) {
    359         return;
    360     }
    361     if (status == U_USING_DEFAULT_WARNING) {
    362         status = U_ILLEGAL_ARGUMENT_ERROR;
    363         return;
    364     }
    365     UnicodeSet tmpSet;
    366     int32_t    i;
    367     for (i=0; i<numScripts; i++) {
    368         tmpSet.applyIntPropertyValue(UCHAR_SCRIPT, scripts[i], status);
    369         allowedChars->addAll(tmpSet);
    370     }
    371 }
    372 
    373 
    374 int32_t SpoofImpl::scriptScan
    375         (const UChar *text, int32_t length, int32_t &pos, UErrorCode &status) const {
    376     if (U_FAILURE(status)) {
    377         return 0;
    378     }
    379     int32_t       inputIdx = 0;
    380     UChar32       c;
    381     int32_t       scriptCount = 0;
    382     UScriptCode   lastScript = USCRIPT_INVALID_CODE;
    383     UScriptCode   sc = USCRIPT_INVALID_CODE;
    384     while ((inputIdx < length || length == -1) && scriptCount < 2) {
    385         U16_NEXT(text, inputIdx, length, c);
    386         if (c == 0 && length == -1) {
    387             break;
    388         }
    389         sc = uscript_getScript(c, &status);
    390         if (sc == USCRIPT_COMMON || sc == USCRIPT_INHERITED || sc == USCRIPT_UNKNOWN) {
    391             continue;
    392         }
    393         if (sc != lastScript) {
    394            scriptCount++;
    395            lastScript = sc;
    396         }
    397     }
    398     if (scriptCount == 2) {
    399         pos = inputIdx;
    400     }
    401     return scriptCount;
    402 }
    403 
    404 
    405 // Convert a text format hex number.  Utility function used by builder code.  Static.
    406 // Input: UChar *string text.  Output: a UChar32
    407 // Input has been pre-checked, and will have no non-hex chars.
    408 // The number must fall in the code point range of 0..0x10ffff
    409 // Static Function.
    410 UChar32 SpoofImpl::ScanHex(const UChar *s, int32_t start, int32_t limit, UErrorCode &status) {
    411     if (U_FAILURE(status)) {
    412         return 0;
    413     }
    414     U_ASSERT(limit-start > 0);
    415     uint32_t val = 0;
    416     int i;
    417     for (i=start; i<limit; i++) {
    418         int digitVal = s[i] - 0x30;
    419         if (digitVal>9) {
    420             digitVal = 0xa + (s[i] - 0x41);  // Upper Case 'A'
    421         }
    422         if (digitVal>15) {
    423             digitVal = 0xa + (s[i] - 0x61);  // Lower Case 'a'
    424         }
    425         U_ASSERT(digitVal <= 0xf);
    426         val <<= 4;
    427         val += digitVal;
    428     }
    429     if (val > 0x10ffff) {
    430         status = U_PARSE_ERROR;
    431         val = 0;
    432     }
    433     return (UChar32)val;
    434 }
    435 
    436 
    437 
    438 //----------------------------------------------------------------------------------------------
    439 //
    440 //   class SpoofData Implementation
    441 //
    442 //----------------------------------------------------------------------------------------------
    443 
    444 
    445 UBool SpoofData::validateDataVersion(const SpoofDataHeader *rawData, UErrorCode &status) {
    446     if (U_FAILURE(status) ||
    447         rawData == NULL ||
    448         rawData->fMagic != USPOOF_MAGIC ||
    449         rawData->fFormatVersion[0] > 1 ||
    450         rawData->fFormatVersion[1] > 0) {
    451             status = U_INVALID_FORMAT_ERROR;
    452             return FALSE;
    453     }
    454     return TRUE;
    455 }
    456 
    457 //
    458 //  SpoofData::getDefault() - return a wrapper around the spoof data that is
    459 //                           baked into the default ICU data.
    460 //
    461 SpoofData *SpoofData::getDefault(UErrorCode &status) {
    462     // TODO:  Cache it.  Lazy create, keep until cleanup.
    463 
    464     UDataMemory *udm = udata_open(NULL, "cfu", "confusables", &status);
    465     if (U_FAILURE(status)) {
    466         return NULL;
    467     }
    468     SpoofData *This = new SpoofData(udm, status);
    469     if (U_FAILURE(status)) {
    470         delete This;
    471         return NULL;
    472     }
    473     if (This == NULL) {
    474         status = U_MEMORY_ALLOCATION_ERROR;
    475     }
    476     return This;
    477 }
    478 
    479 
    480 SpoofData::SpoofData(UDataMemory *udm, UErrorCode &status)
    481 {
    482     reset();
    483     if (U_FAILURE(status)) {
    484         return;
    485     }
    486     fRawData = reinterpret_cast<SpoofDataHeader *>
    487                    ((char *)(udm->pHeader) + udm->pHeader->dataHeader.headerSize);
    488     fUDM = udm;
    489     validateDataVersion(fRawData, status);
    490     initPtrs(status);
    491 }
    492 
    493 
    494 SpoofData::SpoofData(const void *data, int32_t length, UErrorCode &status)
    495 {
    496     reset();
    497     if (U_FAILURE(status)) {
    498         return;
    499     }
    500     if ((size_t)length < sizeof(SpoofDataHeader)) {
    501         status = U_INVALID_FORMAT_ERROR;
    502         return;
    503     }
    504     void *ncData = const_cast<void *>(data);
    505     fRawData = static_cast<SpoofDataHeader *>(ncData);
    506     if (length < fRawData->fLength) {
    507         status = U_INVALID_FORMAT_ERROR;
    508         return;
    509     }
    510     validateDataVersion(fRawData, status);
    511     initPtrs(status);
    512 }
    513 
    514 
    515 // Spoof Data constructor for use from data builder.
    516 //   Initializes a new, empty data area that will be populated later.
    517 SpoofData::SpoofData(UErrorCode &status) {
    518     reset();
    519     if (U_FAILURE(status)) {
    520         return;
    521     }
    522     fDataOwned = true;
    523     fRefCount = 1;
    524 
    525     // The spoof header should already be sized to be a multiple of 16 bytes.
    526     // Just in case it's not, round it up.
    527     uint32_t initialSize = (sizeof(SpoofDataHeader) + 15) & ~15;
    528     U_ASSERT(initialSize == sizeof(SpoofDataHeader));
    529 
    530     fRawData = static_cast<SpoofDataHeader *>(uprv_malloc(initialSize));
    531     fMemLimit = initialSize;
    532     if (fRawData == NULL) {
    533         status = U_MEMORY_ALLOCATION_ERROR;
    534         return;
    535     }
    536     uprv_memset(fRawData, 0, initialSize);
    537 
    538     fRawData->fMagic = USPOOF_MAGIC;
    539     fRawData->fFormatVersion[0] = 1;
    540     fRawData->fFormatVersion[1] = 0;
    541     fRawData->fFormatVersion[2] = 0;
    542     fRawData->fFormatVersion[3] = 0;
    543     initPtrs(status);
    544 }
    545 
    546 // reset() - initialize all fields.
    547 //           Should be updated if any new fields are added.
    548 //           Called by constructors to put things in a known initial state.
    549 void SpoofData::reset() {
    550    fRawData = NULL;
    551    fDataOwned = FALSE;
    552    fUDM      = NULL;
    553    fMemLimit = 0;
    554    fRefCount = 1;
    555    fCFUKeys = NULL;
    556    fCFUValues = NULL;
    557    fCFUStringLengths = NULL;
    558    fCFUStrings = NULL;
    559    fAnyCaseTrie = NULL;
    560    fLowerCaseTrie = NULL;
    561    fScriptSets = NULL;
    562 }
    563 
    564 
    565 //  SpoofData::initPtrs()
    566 //            Initialize the pointers to the various sections of the raw data.
    567 //
    568 //            This function is used both during the Trie building process (multiple
    569 //            times, as the individual data sections are added), and
    570 //            during the opening of a Spoof Checker from prebuilt data.
    571 //
    572 //            The pointers for non-existent data sections (identified by an offset of 0)
    573 //            are set to NULL.
    574 //
    575 //            Note:  During building the data, adding each new data section
    576 //            reallocs the raw data area, which likely relocates it, which
    577 //            in turn requires reinitializing all of the pointers into it, hence
    578 //            multiple calls to this function during building.
    579 //
    580 void SpoofData::initPtrs(UErrorCode &status) {
    581     fCFUKeys = NULL;
    582     fCFUValues = NULL;
    583     fCFUStringLengths = NULL;
    584     fCFUStrings = NULL;
    585     if (U_FAILURE(status)) {
    586         return;
    587     }
    588     if (fRawData->fCFUKeys != 0) {
    589         fCFUKeys = (int32_t *)((char *)fRawData + fRawData->fCFUKeys);
    590     }
    591     if (fRawData->fCFUStringIndex != 0) {
    592         fCFUValues = (uint16_t *)((char *)fRawData + fRawData->fCFUStringIndex);
    593     }
    594     if (fRawData->fCFUStringLengths != 0) {
    595         fCFUStringLengths = (SpoofStringLengthsElement *)((char *)fRawData + fRawData->fCFUStringLengths);
    596     }
    597     if (fRawData->fCFUStringTable != 0) {
    598         fCFUStrings = (UChar *)((char *)fRawData + fRawData->fCFUStringTable);
    599     }
    600 
    601     if (fAnyCaseTrie ==  NULL && fRawData->fAnyCaseTrie != 0) {
    602         fAnyCaseTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
    603             (char *)fRawData + fRawData->fAnyCaseTrie, fRawData->fAnyCaseTrieLength, NULL, &status);
    604     }
    605     if (fLowerCaseTrie ==  NULL && fRawData->fLowerCaseTrie != 0) {
    606         fLowerCaseTrie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS,
    607             (char *)fRawData + fRawData->fLowerCaseTrie, fRawData->fLowerCaseTrieLength, NULL, &status);
    608     }
    609 
    610     if (fRawData->fScriptSets != 0) {
    611         fScriptSets = (ScriptSet *)((char *)fRawData + fRawData->fScriptSets);
    612     }
    613 }
    614 
    615 
    616 SpoofData::~SpoofData() {
    617     utrie2_close(fAnyCaseTrie);
    618     fAnyCaseTrie = NULL;
    619     utrie2_close(fLowerCaseTrie);
    620     fLowerCaseTrie = NULL;
    621     if (fDataOwned) {
    622         uprv_free(fRawData);
    623     }
    624     fRawData = NULL;
    625     if (fUDM != NULL) {
    626         udata_close(fUDM);
    627     }
    628     fUDM = NULL;
    629 }
    630 
    631 
    632 void SpoofData::removeReference() {
    633     if (umtx_atomic_dec(&fRefCount) == 0) {
    634         delete this;
    635     }
    636 }
    637 
    638 
    639 SpoofData *SpoofData::addReference() {
    640     umtx_atomic_inc(&fRefCount);
    641     return this;
    642 }
    643 
    644 
    645 void *SpoofData::reserveSpace(int32_t numBytes,  UErrorCode &status) {
    646     if (U_FAILURE(status)) {
    647         return NULL;
    648     }
    649     if (!fDataOwned) {
    650         U_ASSERT(FALSE);
    651         status = U_INTERNAL_PROGRAM_ERROR;
    652         return NULL;
    653     }
    654 
    655     numBytes = (numBytes + 15) & ~15;   // Round up to a multiple of 16
    656     uint32_t returnOffset = fMemLimit;
    657     fMemLimit += numBytes;
    658     fRawData = static_cast<SpoofDataHeader *>(uprv_realloc(fRawData, fMemLimit));
    659     fRawData->fLength = fMemLimit;
    660     uprv_memset((char *)fRawData + returnOffset, 0, numBytes);
    661     initPtrs(status);
    662     return (char *)fRawData + returnOffset;
    663 }
    664 
    665 
    666 //----------------------------------------------------------------------------
    667 //
    668 //  ScriptSet implementation
    669 //
    670 //----------------------------------------------------------------------------
    671 ScriptSet::ScriptSet() {
    672     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    673         bits[i] = 0;
    674     }
    675 }
    676 
    677 ScriptSet::~ScriptSet() {
    678 }
    679 
    680 UBool ScriptSet::operator == (const ScriptSet &other) {
    681     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    682         if (bits[i] != other.bits[i]) {
    683             return FALSE;
    684         }
    685     }
    686     return TRUE;
    687 }
    688 
    689 void ScriptSet::Union(UScriptCode script) {
    690     uint32_t index = script / 32;
    691     uint32_t bit   = 1 << (script & 31);
    692     U_ASSERT(index < sizeof(bits)*4);
    693     bits[index] |= bit;
    694 }
    695 
    696 
    697 void ScriptSet::Union(const ScriptSet &other) {
    698     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    699         bits[i] |= other.bits[i];
    700     }
    701 }
    702 
    703 void ScriptSet::intersect(const ScriptSet &other) {
    704     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    705         bits[i] &= other.bits[i];
    706     }
    707 }
    708 
    709 void ScriptSet::intersect(UScriptCode script) {
    710     uint32_t index = script / 32;
    711     uint32_t bit   = 1 << (script & 31);
    712     U_ASSERT(index < sizeof(bits)*4);
    713     uint32_t i;
    714     for (i=0; i<index; i++) {
    715         bits[i] = 0;
    716     }
    717     bits[index] &= bit;
    718     for (i=index+1; i<sizeof(bits)/sizeof(uint32_t); i++) {
    719         bits[i] = 0;
    720     }
    721 }
    722 
    723 
    724 ScriptSet & ScriptSet::operator =(const ScriptSet &other) {
    725     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    726         bits[i] = other.bits[i];
    727     }
    728     return *this;
    729 }
    730 
    731 
    732 void ScriptSet::setAll() {
    733     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    734         bits[i] = 0xffffffffu;
    735     }
    736 }
    737 
    738 
    739 void ScriptSet::resetAll() {
    740     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    741         bits[i] = 0;
    742     }
    743 }
    744 
    745 int32_t ScriptSet::countMembers() {
    746     // This bit counter is good for sparse numbers of '1's, which is
    747     //  very much the case that we will usually have.
    748     int32_t count = 0;
    749     for (uint32_t i=0; i<sizeof(bits)/sizeof(uint32_t); i++) {
    750         uint32_t x = bits[i];
    751         while (x > 0) {
    752             count++;
    753             x &= (x - 1);    // and off the least significant one bit.
    754         }
    755     }
    756     return count;
    757 }
    758 
    759 
    760 
    761 //-----------------------------------------------------------------------------
    762 //
    763 //  NFKDBuffer Implementation.
    764 //
    765 //-----------------------------------------------------------------------------
    766 
    767 NFKDBuffer::NFKDBuffer(const UChar *text, int32_t length, UErrorCode &status) {
    768     fNormalizedText = NULL;
    769     fNormalizedTextLength = 0;
    770     fOriginalText = text;
    771     if (U_FAILURE(status)) {
    772         return;
    773     }
    774     fNormalizedText = fSmallBuf;
    775     fNormalizedTextLength = unorm_normalize(
    776         text, length, UNORM_NFKD, 0, fNormalizedText, USPOOF_STACK_BUFFER_SIZE, &status);
    777     if (status == U_BUFFER_OVERFLOW_ERROR) {
    778         status = U_ZERO_ERROR;
    779         fNormalizedText = (UChar *)uprv_malloc((fNormalizedTextLength+1)*sizeof(UChar));
    780         if (fNormalizedText == NULL) {
    781             status = U_MEMORY_ALLOCATION_ERROR;
    782         } else {
    783             fNormalizedTextLength = unorm_normalize(text, length, UNORM_NFKD, 0,
    784                                         fNormalizedText, fNormalizedTextLength+1, &status);
    785         }
    786     }
    787 }
    788 
    789 
    790 NFKDBuffer::~NFKDBuffer() {
    791     if (fNormalizedText != fSmallBuf) {
    792         uprv_free(fNormalizedText);
    793     }
    794     fNormalizedText = 0;
    795 }
    796 
    797 const UChar *NFKDBuffer::getBuffer() {
    798     return fNormalizedText;
    799 }
    800 
    801 int32_t NFKDBuffer::getLength() {
    802     return fNormalizedTextLength;
    803 }
    804 
    805 
    806 
    807 
    808 
    809 U_NAMESPACE_END
    810 
    811 U_NAMESPACE_USE
    812 
    813 //-----------------------------------------------------------------------------
    814 //
    815 //  uspoof_swap   -  byte swap and char encoding swap of spoof data
    816 //
    817 //-----------------------------------------------------------------------------
    818 U_CAPI int32_t U_EXPORT2
    819 uspoof_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
    820            UErrorCode *status) {
    821 
    822     if (status == NULL || U_FAILURE(*status)) {
    823         return 0;
    824     }
    825     if(ds==NULL || inData==NULL || length<-1 || (length>0 && outData==NULL)) {
    826         *status=U_ILLEGAL_ARGUMENT_ERROR;
    827         return 0;
    828     }
    829 
    830     //
    831     //  Check that the data header is for spoof data.
    832     //    (Header contents are defined in gencfu.cpp)
    833     //
    834     const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData+4);
    835     if(!(  pInfo->dataFormat[0]==0x43 &&   /* dataFormat="Cfu " */
    836            pInfo->dataFormat[1]==0x66 &&
    837            pInfo->dataFormat[2]==0x75 &&
    838            pInfo->dataFormat[3]==0x20 &&
    839            pInfo->formatVersion[0]==1  )) {
    840         udata_printError(ds, "uspoof_swap(): data format %02x.%02x.%02x.%02x "
    841                              "(format version %02x %02x %02x %02x) is not recognized\n",
    842                          pInfo->dataFormat[0], pInfo->dataFormat[1],
    843                          pInfo->dataFormat[2], pInfo->dataFormat[3],
    844                          pInfo->formatVersion[0], pInfo->formatVersion[1],
    845                          pInfo->formatVersion[2], pInfo->formatVersion[3]);
    846         *status=U_UNSUPPORTED_ERROR;
    847         return 0;
    848     }
    849 
    850     //
    851     // Swap the data header.  (This is the generic ICU Data Header, not the uspoof Specific
    852     //                         header).  This swap also conveniently gets us
    853     //                         the size of the ICU d.h., which lets us locate the start
    854     //                         of the uspoof specific data.
    855     //
    856     int32_t headerSize=udata_swapDataHeader(ds, inData, length, outData, status);
    857 
    858 
    859     //
    860     // Get the Spoof Data Header, and check that it appears to be OK.
    861     //
    862     //
    863     const uint8_t   *inBytes =(const uint8_t *)inData+headerSize;
    864     SpoofDataHeader *spoofDH = (SpoofDataHeader *)inBytes;
    865     if (ds->readUInt32(spoofDH->fMagic)   != USPOOF_MAGIC ||
    866         ds->readUInt32(spoofDH->fLength)  <  sizeof(SpoofDataHeader))
    867     {
    868         udata_printError(ds, "uspoof_swap(): Spoof Data header is invalid.\n");
    869         *status=U_UNSUPPORTED_ERROR;
    870         return 0;
    871     }
    872 
    873     //
    874     // Prefight operation?  Just return the size
    875     //
    876     int32_t spoofDataLength = ds->readUInt32(spoofDH->fLength);
    877     int32_t totalSize = headerSize + spoofDataLength;
    878     if (length < 0) {
    879         return totalSize;
    880     }
    881 
    882     //
    883     // Check that length passed in is consistent with length from Spoof data header.
    884     //
    885     if (length < totalSize) {
    886         udata_printError(ds, "uspoof_swap(): too few bytes (%d after ICU Data header) for spoof data.\n",
    887                             spoofDataLength);
    888         *status=U_INDEX_OUTOFBOUNDS_ERROR;
    889         return 0;
    890         }
    891 
    892 
    893     //
    894     // Swap the Data.  Do the data itself first, then the Spoof Data Header, because
    895     //                 we need to reference the header to locate the data, and an
    896     //                 inplace swap of the header leaves it unusable.
    897     //
    898     uint8_t          *outBytes = (uint8_t *)outData + headerSize;
    899     SpoofDataHeader  *outputDH = (SpoofDataHeader *)outBytes;
    900 
    901     int32_t   sectionStart;
    902     int32_t   sectionLength;
    903 
    904     //
    905     // If not swapping in place, zero out the output buffer before starting.
    906     //    Gaps may exist between the individual sections, and these must be zeroed in
    907     //    the output buffer.  The simplest way to do that is to just zero the whole thing.
    908     //
    909     if (inBytes != outBytes) {
    910         uprv_memset(outBytes, 0, spoofDataLength);
    911     }
    912 
    913     // Confusables Keys Section   (fCFUKeys)
    914     sectionStart  = ds->readUInt32(spoofDH->fCFUKeys);
    915     sectionLength = ds->readUInt32(spoofDH->fCFUKeysSize) * 4;
    916     ds->swapArray32(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    917 
    918     // String Index Section
    919     sectionStart  = ds->readUInt32(spoofDH->fCFUStringIndex);
    920     sectionLength = ds->readUInt32(spoofDH->fCFUStringIndexSize) * 2;
    921     ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    922 
    923     // String Table Section
    924     sectionStart  = ds->readUInt32(spoofDH->fCFUStringTable);
    925     sectionLength = ds->readUInt32(spoofDH->fCFUStringTableLen) * 2;
    926     ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    927 
    928     // String Lengths Section
    929     sectionStart  = ds->readUInt32(spoofDH->fCFUStringLengths);
    930     sectionLength = ds->readUInt32(spoofDH->fCFUStringLengthsSize) * 4;
    931     ds->swapArray16(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    932 
    933     // Any Case Trie
    934     sectionStart  = ds->readUInt32(spoofDH->fAnyCaseTrie);
    935     sectionLength = ds->readUInt32(spoofDH->fAnyCaseTrieLength);
    936     utrie2_swap(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    937 
    938     // Lower Case Trie
    939     sectionStart  = ds->readUInt32(spoofDH->fLowerCaseTrie);
    940     sectionLength = ds->readUInt32(spoofDH->fLowerCaseTrieLength);
    941     utrie2_swap(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    942 
    943     // Script Sets.  The data is an array of int32_t
    944     sectionStart  = ds->readUInt32(spoofDH->fScriptSets);
    945     sectionLength = ds->readUInt32(spoofDH->fScriptSetsLength) * sizeof(ScriptSet);
    946     ds->swapArray32(ds, inBytes+sectionStart, sectionLength, outBytes+sectionStart, status);
    947 
    948     // And, last, swap the header itself.
    949     //   int32_t   fMagic             // swap this
    950     //   uint8_t   fFormatVersion[4]  // Do not swap this, just copy
    951     //   int32_t   fLength and all the rest       // Swap the rest, all is 32 bit stuff.
    952     //
    953     uint32_t magic = ds->readUInt32(spoofDH->fMagic);
    954     ds->writeUInt32((uint32_t *)&outputDH->fMagic, magic);
    955     uprv_memcpy(outputDH->fFormatVersion, spoofDH->fFormatVersion, sizeof(spoofDH->fFormatVersion));
    956     // swap starting at fLength
    957     ds->swapArray32(ds, &spoofDH->fLength, sizeof(SpoofDataHeader)-8 /* minus magic and fFormatVersion[4] */, &outputDH->fLength, status);
    958 
    959     return totalSize;
    960 }
    961 
    962 #endif
    963 
    964 
    965