1 /* 2 ********************************************************************** 3 * Copyright (C) 2005-2013, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 */ 7 8 #include "unicode/utypes.h" 9 10 #if !UCONFIG_NO_CONVERSION 11 12 #include "csrucode.h" 13 #include "csmatch.h" 14 15 U_NAMESPACE_BEGIN 16 17 CharsetRecog_Unicode::~CharsetRecog_Unicode() 18 { 19 // nothing to do 20 } 21 22 CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE() 23 { 24 // nothing to do 25 } 26 27 const char *CharsetRecog_UTF_16_BE::getName() const 28 { 29 return "UTF-16BE"; 30 } 31 32 UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const 33 { 34 const uint8_t *input = textIn->fRawInput; 35 int32_t confidence = 0; 36 int32_t length = textIn->fRawLength; 37 38 if (length >=2 && input[0] == 0xFE && input[1] == 0xFF) { 39 confidence = 100; 40 } 41 42 // TODO: Do some statastics to check for unsigned UTF-16BE 43 results->set(textIn, this, confidence); 44 return (confidence > 0); 45 } 46 47 CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE() 48 { 49 // nothing to do 50 } 51 52 const char *CharsetRecog_UTF_16_LE::getName() const 53 { 54 return "UTF-16LE"; 55 } 56 57 UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const 58 { 59 const uint8_t *input = textIn->fRawInput; 60 int32_t confidence = 0; 61 int32_t length = textIn->fRawLength; 62 63 if (length >= 4 && input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) { 64 confidence = 100; 65 } 66 67 // TODO: Do some statastics to check for unsigned UTF-16LE 68 results->set(textIn, this, confidence); 69 return (confidence > 0); 70 } 71 72 CharsetRecog_UTF_32::~CharsetRecog_UTF_32() 73 { 74 // nothing to do 75 } 76 77 UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const 78 { 79 const uint8_t *input = textIn->fRawInput; 80 int32_t limit = (textIn->fRawLength / 4) * 4; 81 int32_t numValid = 0; 82 int32_t numInvalid = 0; 83 bool hasBOM = FALSE; 84 int32_t confidence = 0; 85 86 if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) { 87 hasBOM = TRUE; 88 } 89 90 for(int32_t i = 0; i < limit; i += 4) { 91 int32_t ch = getChar(input, i); 92 93 if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) { 94 numInvalid += 1; 95 } else { 96 numValid += 1; 97 } 98 } 99 100 101 // Cook up some sort of confidence score, based on presense of a BOM 102 // and the existence of valid and/or invalid multi-byte sequences. 103 if (hasBOM && numInvalid==0) { 104 confidence = 100; 105 } else if (hasBOM && numValid > numInvalid*10) { 106 confidence = 80; 107 } else if (numValid > 3 && numInvalid == 0) { 108 confidence = 100; 109 } else if (numValid > 0 && numInvalid == 0) { 110 confidence = 80; 111 } else if (numValid > numInvalid*10) { 112 // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance. 113 confidence = 25; 114 } 115 116 results->set(textIn, this, confidence); 117 return (confidence > 0); 118 } 119 120 CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE() 121 { 122 // nothing to do 123 } 124 125 const char *CharsetRecog_UTF_32_BE::getName() const 126 { 127 return "UTF-32BE"; 128 } 129 130 int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const 131 { 132 return input[index + 0] << 24 | input[index + 1] << 16 | 133 input[index + 2] << 8 | input[index + 3]; 134 } 135 136 CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE() 137 { 138 // nothing to do 139 } 140 141 const char *CharsetRecog_UTF_32_LE::getName() const 142 { 143 return "UTF-32LE"; 144 } 145 146 int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const 147 { 148 return input[index + 3] << 24 | input[index + 2] << 16 | 149 input[index + 1] << 8 | input[index + 0]; 150 } 151 152 U_NAMESPACE_END 153 #endif 154 155
