Home | History | Annotate | Download | only in i18n 
      1 /*
      2  **********************************************************************
      3  *   Copyright (C) 2005-2013, International Business Machines
      4  *   Corporation and others.  All Rights Reserved.
      5  **********************************************************************
      6  */
      7 
      8 #include "unicode/utypes.h"
      9 
     10 #if !UCONFIG_NO_CONVERSION
     11 
     12 #include "csrucode.h"
     13 #include "csmatch.h"
     14 
     15 U_NAMESPACE_BEGIN
     16 
     17 CharsetRecog_Unicode::~CharsetRecog_Unicode()
     18 {
     19     // nothing to do
     20 }
     21 
     22 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
     23 {
     24     // nothing to do
     25 }
     26 
     27 const char *CharsetRecog_UTF_16_BE::getName() const
     28 {
     29     return "UTF-16BE";
     30 }
     31 
     32 // UTF-16 confidence calculation. Very simple minded, but better than nothing.
     33 //   Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
     34 //     and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
     35 //   NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
     36 //   NULs should be rare in actual text.
     37 
     38 static int32_t adjustConfidence(UChar codeUnit, int32_t confidence) {
     39     if (codeUnit == 0) {
     40         confidence -= 10;
     41     } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
     42         confidence += 10;
     43     }
     44     if (confidence < 0) {
     45         confidence = 0;
     46     } else if (confidence > 100) {
     47         confidence = 100;
     48     }
     49     return confidence;
     50 }
     51 
     52 
     53 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
     54 {
     55     const uint8_t *input = textIn->fRawInput;
     56     int32_t confidence = 10;
     57     int32_t length = textIn->fRawLength;
     58 
     59     int32_t bytesToCheck = (length > 30) ? 30 : length;
     60     for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
     61         UChar codeUnit = (input[charIndex] << 8) | input[charIndex + 1];
     62         if (charIndex == 0 && codeUnit == 0xFEFF) {
     63             confidence = 100;
     64             break;
     65         }
     66         confidence = adjustConfidence(codeUnit, confidence);
     67         if (confidence == 0 || confidence == 100) {
     68             break;
     69         }
     70     }
     71     if (bytesToCheck < 4 && confidence < 100) {
     72         confidence = 0;
     73     }
     74     results->set(textIn, this, confidence);
     75     return (confidence > 0);
     76 }
     77 
     78 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
     79 {
     80     // nothing to do
     81 }
     82 
     83 const char *CharsetRecog_UTF_16_LE::getName() const
     84 {
     85     return "UTF-16LE";
     86 }
     87 
     88 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
     89 {
     90     const uint8_t *input = textIn->fRawInput;
     91     int32_t confidence = 10;
     92     int32_t length = textIn->fRawLength;
     93 
     94     int32_t bytesToCheck = (length > 30) ? 30 : length;
     95     for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
     96         UChar codeUnit = input[charIndex] | (input[charIndex + 1] << 8);
     97         if (charIndex == 0 && codeUnit == 0xFEFF) {
     98             confidence = 100;     // UTF-16 BOM
     99             if (length >= 4 && input[2] == 0 && input[3] == 0) {
    100                 confidence = 0;   // UTF-32 BOM
    101             }
    102             break;
    103         }
    104         confidence = adjustConfidence(codeUnit, confidence);
    105         if (confidence == 0 || confidence == 100) {
    106             break;
    107         }
    108     }
    109     if (bytesToCheck < 4 && confidence < 100) {
    110         confidence = 0;
    111     }
    112     results->set(textIn, this, confidence);
    113     return (confidence > 0);
    114 }
    115 
    116 CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
    117 {
    118     // nothing to do
    119 }
    120 
    121 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
    122 {
    123     const uint8_t *input = textIn->fRawInput;
    124     int32_t limit = (textIn->fRawLength / 4) * 4;
    125     int32_t numValid = 0;
    126     int32_t numInvalid = 0;
    127     bool hasBOM = FALSE;
    128     int32_t confidence = 0;
    129 
    130     if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {
    131         hasBOM = TRUE;
    132     }
    133 
    134     for(int32_t i = 0; i < limit; i += 4) {
    135         int32_t ch = getChar(input, i);
    136 
    137         if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
    138             numInvalid += 1;
    139         } else {
    140             numValid += 1;
    141         }
    142     }
    143 
    144 
    145     // Cook up some sort of confidence score, based on presense of a BOM
    146     //    and the existence of valid and/or invalid multi-byte sequences.
    147     if (hasBOM && numInvalid==0) {
    148         confidence = 100;
    149     } else if (hasBOM && numValid > numInvalid*10) {
    150         confidence = 80;
    151     } else if (numValid > 3 && numInvalid == 0) {
    152         confidence = 100;
    153     } else if (numValid > 0 && numInvalid == 0) {
    154         confidence = 80;
    155     } else if (numValid > numInvalid*10) {
    156         // Probably corruput UTF-32BE data.  Valid sequences aren't likely by chance.
    157         confidence = 25;
    158     }
    159 
    160     results->set(textIn, this, confidence);
    161     return (confidence > 0);
    162 }
    163 
    164 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
    165 {
    166     // nothing to do
    167 }
    168 
    169 const char *CharsetRecog_UTF_32_BE::getName() const
    170 {
    171     return "UTF-32BE";
    172 }
    173 
    174 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
    175 {
    176     return input[index + 0] << 24 | input[index + 1] << 16 |
    177            input[index + 2] <<  8 | input[index + 3];
    178 }
    179 
    180 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
    181 {
    182     // nothing to do
    183 }
    184 
    185 const char *CharsetRecog_UTF_32_LE::getName() const
    186 {
    187     return "UTF-32LE";
    188 }
    189 
    190 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
    191 {
    192     return input[index + 3] << 24 | input[index + 2] << 16 |
    193            input[index + 1] <<  8 | input[index + 0];
    194 }
    195 
    196 U_NAMESPACE_END
    197 #endif
    198 
    199