1 // Copyright 2007-2010 the V8 project authors. All rights reserved. 2 // Redistribution and use in source and binary forms, with or without 3 // modification, are permitted provided that the following conditions are 4 // met: 5 // 6 // * Redistributions of source code must retain the above copyright 7 // notice, this list of conditions and the following disclaimer. 8 // * Redistributions in binary form must reproduce the above 9 // copyright notice, this list of conditions and the following 10 // disclaimer in the documentation and/or other materials provided 11 // with the distribution. 12 // * Neither the name of Google Inc. nor the names of its 13 // contributors may be used to endorse or promote products derived 14 // from this software without specific prior written permission. 15 // 16 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 #ifndef V8_UNICODE_INL_H_ 29 #define V8_UNICODE_INL_H_ 30 31 #include "unicode.h" 32 33 namespace unibrow { 34 35 template <class T, int s> bool Predicate<T, s>::get(uchar code_point) { 36 CacheEntry entry = entries_[code_point & kMask]; 37 if (entry.code_point_ == code_point) return entry.value_; 38 return CalculateValue(code_point); 39 } 40 41 template <class T, int s> bool Predicate<T, s>::CalculateValue( 42 uchar code_point) { 43 bool result = T::Is(code_point); 44 entries_[code_point & kMask] = CacheEntry(code_point, result); 45 return result; 46 } 47 48 template <class T, int s> int Mapping<T, s>::get(uchar c, uchar n, 49 uchar* result) { 50 CacheEntry entry = entries_[c & kMask]; 51 if (entry.code_point_ == c) { 52 if (entry.offset_ == 0) { 53 return 0; 54 } else { 55 result[0] = c + entry.offset_; 56 return 1; 57 } 58 } else { 59 return CalculateValue(c, n, result); 60 } 61 } 62 63 template <class T, int s> int Mapping<T, s>::CalculateValue(uchar c, uchar n, 64 uchar* result) { 65 bool allow_caching = true; 66 int length = T::Convert(c, n, result, &allow_caching); 67 if (allow_caching) { 68 if (length == 1) { 69 entries_[c & kMask] = CacheEntry(c, result[0] - c); 70 return 1; 71 } else { 72 entries_[c & kMask] = CacheEntry(c, 0); 73 return 0; 74 } 75 } else { 76 return length; 77 } 78 } 79 80 81 unsigned Utf8::Encode(char* str, uchar c, int previous) { 82 static const int kMask = ~(1 << 6); 83 if (c <= kMaxOneByteChar) { 84 str[0] = c; 85 return 1; 86 } else if (c <= kMaxTwoByteChar) { 87 str[0] = 0xC0 | (c >> 6); 88 str[1] = 0x80 | (c & kMask); 89 return 2; 90 } else if (c <= kMaxThreeByteChar) { 91 if (Utf16::IsTrailSurrogate(c) && 92 Utf16::IsLeadSurrogate(previous)) { 93 const int kUnmatchedSize = kSizeOfUnmatchedSurrogate; 94 return Encode(str - kUnmatchedSize, 95 Utf16::CombineSurrogatePair(previous, c), 96 Utf16::kNoPreviousCharacter) - kUnmatchedSize; 97 } 98 str[0] = 0xE0 | (c >> 12); 99 str[1] = 0x80 | ((c >> 6) & kMask); 100 str[2] = 0x80 | (c & kMask); 101 return 3; 102 } else { 103 str[0] = 0xF0 | (c >> 18); 104 str[1] = 0x80 | ((c >> 12) & kMask); 105 str[2] = 0x80 | ((c >> 6) & kMask); 106 str[3] = 0x80 | (c & kMask); 107 return 4; 108 } 109 } 110 111 112 uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) { 113 if (length <= 0) return kBadChar; 114 byte first = bytes[0]; 115 // Characters between 0000 and 0007F are encoded as a single character 116 if (first <= kMaxOneByteChar) { 117 *cursor += 1; 118 return first; 119 } 120 return CalculateValue(bytes, length, cursor); 121 } 122 123 unsigned Utf8::Length(uchar c, int previous) { 124 if (c <= kMaxOneByteChar) { 125 return 1; 126 } else if (c <= kMaxTwoByteChar) { 127 return 2; 128 } else if (c <= kMaxThreeByteChar) { 129 if (Utf16::IsTrailSurrogate(c) && 130 Utf16::IsLeadSurrogate(previous)) { 131 return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates; 132 } 133 return 3; 134 } else { 135 return 4; 136 } 137 } 138 139 uchar CharacterStream::GetNext() { 140 uchar result = DecodeCharacter(buffer_, &cursor_); 141 if (remaining_ == 1) { 142 cursor_ = 0; 143 FillBuffer(); 144 } else { 145 remaining_--; 146 } 147 return result; 148 } 149 150 #if __BYTE_ORDER == __LITTLE_ENDIAN 151 #define IF_LITTLE(expr) expr 152 #define IF_BIG(expr) ((void) 0) 153 #elif __BYTE_ORDER == __BIG_ENDIAN 154 #define IF_LITTLE(expr) ((void) 0) 155 #define IF_BIG(expr) expr 156 #else 157 #warning Unknown byte ordering 158 #endif 159 160 bool CharacterStream::EncodeAsciiCharacter(uchar c, byte* buffer, 161 unsigned capacity, unsigned& offset) { 162 if (offset >= capacity) return false; 163 buffer[offset] = c; 164 offset += 1; 165 return true; 166 } 167 168 bool CharacterStream::EncodeNonAsciiCharacter(uchar c, byte* buffer, 169 unsigned capacity, unsigned& offset) { 170 unsigned aligned = (offset + 0x3) & ~0x3; 171 if ((aligned + sizeof(uchar)) > capacity) 172 return false; 173 if (offset == aligned) { 174 IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = (c << 8) | 0x80); 175 IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c | (1 << 31)); 176 } else { 177 buffer[offset] = 0x80; 178 IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = c << 8); 179 IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c); 180 } 181 offset = aligned + sizeof(uchar); 182 return true; 183 } 184 185 bool CharacterStream::EncodeCharacter(uchar c, byte* buffer, unsigned capacity, 186 unsigned& offset) { 187 if (c <= Utf8::kMaxOneByteChar) { 188 return EncodeAsciiCharacter(c, buffer, capacity, offset); 189 } else { 190 return EncodeNonAsciiCharacter(c, buffer, capacity, offset); 191 } 192 } 193 194 uchar CharacterStream::DecodeCharacter(const byte* buffer, unsigned* offset) { 195 byte b = buffer[*offset]; 196 if (b <= Utf8::kMaxOneByteChar) { 197 (*offset)++; 198 return b; 199 } else { 200 unsigned aligned = (*offset + 0x3) & ~0x3; 201 *offset = aligned + sizeof(uchar); 202 IF_LITTLE(return *reinterpret_cast<const uchar*>(buffer + aligned) >> 8); 203 IF_BIG(return *reinterpret_cast<const uchar*>(buffer + aligned) & 204 ~(1 << 31)); 205 } 206 } 207 208 #undef IF_LITTLE 209 #undef IF_BIG 210 211 template <class R, class I, unsigned s> 212 void InputBuffer<R, I, s>::FillBuffer() { 213 buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_); 214 } 215 216 template <class R, class I, unsigned s> 217 void InputBuffer<R, I, s>::Rewind() { 218 Reset(input_); 219 } 220 221 template <class R, class I, unsigned s> 222 void InputBuffer<R, I, s>::Reset(unsigned position, I input) { 223 input_ = input; 224 remaining_ = 0; 225 cursor_ = 0; 226 offset_ = position; 227 buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_); 228 } 229 230 template <class R, class I, unsigned s> 231 void InputBuffer<R, I, s>::Reset(I input) { 232 Reset(0, input); 233 } 234 235 template <class R, class I, unsigned s> 236 void InputBuffer<R, I, s>::Seek(unsigned position) { 237 offset_ = position; 238 buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_); 239 } 240 241 template <unsigned s> 242 Utf8InputBuffer<s>::Utf8InputBuffer(const char* data, unsigned length) 243 : InputBuffer<Utf8, Buffer<const char*>, s>(Buffer<const char*>(data, 244 length)) { 245 } 246 247 } // namespace unibrow 248 249 #endif // V8_UNICODE_INL_H_ 250
