1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef URL_URL_CANON_INTERNAL_H_ 6 #define URL_URL_CANON_INTERNAL_H_ 7 8 // This file is intended to be included in another C++ file where the character 9 // types are defined. This allows us to write mostly generic code, but not have 10 // templace bloat because everything is inlined when anybody calls any of our 11 // functions. 12 13 #include <stdlib.h> 14 15 #include "base/logging.h" 16 #include "url/url_canon.h" 17 18 namespace url_canon { 19 20 // Character type handling ----------------------------------------------------- 21 22 // Bits that identify different character types. These types identify different 23 // bits that are set for each 8-bit character in the kSharedCharTypeTable. 24 enum SharedCharTypes { 25 // Characters that do not require escaping in queries. Characters that do 26 // not have this flag will be escaped; see url_canon_query.cc 27 CHAR_QUERY = 1, 28 29 // Valid in the username/password field. 30 CHAR_USERINFO = 2, 31 32 // Valid in a IPv4 address (digits plus dot and 'x' for hex). 33 CHAR_IPV4 = 4, 34 35 // Valid in an ASCII-representation of a hex digit (as in %-escaped). 36 CHAR_HEX = 8, 37 38 // Valid in an ASCII-representation of a decimal digit. 39 CHAR_DEC = 16, 40 41 // Valid in an ASCII-representation of an octal digit. 42 CHAR_OCT = 32, 43 44 // Characters that do not require escaping in encodeURIComponent. Characters 45 // that do not have this flag will be escaped; see url_util.cc. 46 CHAR_COMPONENT = 64, 47 }; 48 49 // This table contains the flags in SharedCharTypes for each 8-bit character. 50 // Some canonicalization functions have their own specialized lookup table. 51 // For those with simple requirements, we have collected the flags in one 52 // place so there are fewer lookup tables to load into the CPU cache. 53 // 54 // Using an unsigned char type has a small but measurable performance benefit 55 // over using a 32-bit number. 56 extern const unsigned char kSharedCharTypeTable[0x100]; 57 58 // More readable wrappers around the character type lookup table. 59 inline bool IsCharOfType(unsigned char c, SharedCharTypes type) { 60 return !!(kSharedCharTypeTable[c] & type); 61 } 62 inline bool IsQueryChar(unsigned char c) { 63 return IsCharOfType(c, CHAR_QUERY); 64 } 65 inline bool IsIPv4Char(unsigned char c) { 66 return IsCharOfType(c, CHAR_IPV4); 67 } 68 inline bool IsHexChar(unsigned char c) { 69 return IsCharOfType(c, CHAR_HEX); 70 } 71 inline bool IsComponentChar(unsigned char c) { 72 return IsCharOfType(c, CHAR_COMPONENT); 73 } 74 75 // Appends the given string to the output, escaping characters that do not 76 // match the given |type| in SharedCharTypes. 77 void AppendStringOfType(const char* source, int length, 78 SharedCharTypes type, 79 CanonOutput* output); 80 void AppendStringOfType(const base::char16* source, int length, 81 SharedCharTypes type, 82 CanonOutput* output); 83 84 // Maps the hex numerical values 0x0 to 0xf to the corresponding ASCII digit 85 // that will be used to represent it. 86 URL_EXPORT extern const char kHexCharLookup[0x10]; 87 88 // This lookup table allows fast conversion between ASCII hex letters and their 89 // corresponding numerical value. The 8-bit range is divided up into 8 90 // regions of 0x20 characters each. Each of the three character types (numbers, 91 // uppercase, lowercase) falls into different regions of this range. The table 92 // contains the amount to subtract from characters in that range to get at 93 // the corresponding numerical value. 94 // 95 // See HexDigitToValue for the lookup. 96 extern const char kCharToHexLookup[8]; 97 98 // Assumes the input is a valid hex digit! Call IsHexChar before using this. 99 inline unsigned char HexCharToValue(unsigned char c) { 100 return c - kCharToHexLookup[c / 0x20]; 101 } 102 103 // Indicates if the given character is a dot or dot equivalent, returning the 104 // number of characters taken by it. This will be one for a literal dot, 3 for 105 // an escaped dot. If the character is not a dot, this will return 0. 106 template<typename CHAR> 107 inline int IsDot(const CHAR* spec, int offset, int end) { 108 if (spec[offset] == '.') { 109 return 1; 110 } else if (spec[offset] == '%' && offset + 3 <= end && 111 spec[offset + 1] == '2' && 112 (spec[offset + 2] == 'e' || spec[offset + 2] == 'E')) { 113 // Found "%2e" 114 return 3; 115 } 116 return 0; 117 } 118 119 // Returns the canonicalized version of the input character according to scheme 120 // rules. This is implemented alongside the scheme canonicalizer, and is 121 // required for relative URL resolving to test for scheme equality. 122 // 123 // Returns 0 if the input character is not a valid scheme character. 124 char CanonicalSchemeChar(base::char16 ch); 125 126 // Write a single character, escaped, to the output. This always escapes: it 127 // does no checking that thee character requires escaping. 128 // Escaping makes sense only 8 bit chars, so code works in all cases of 129 // input parameters (8/16bit). 130 template<typename UINCHAR, typename OUTCHAR> 131 inline void AppendEscapedChar(UINCHAR ch, 132 CanonOutputT<OUTCHAR>* output) { 133 output->push_back('%'); 134 output->push_back(kHexCharLookup[(ch >> 4) & 0xf]); 135 output->push_back(kHexCharLookup[ch & 0xf]); 136 } 137 138 // The character we'll substitute for undecodable or invalid characters. 139 extern const base::char16 kUnicodeReplacementCharacter; 140 141 // UTF-8 functions ------------------------------------------------------------ 142 143 // Reads one character in UTF-8 starting at |*begin| in |str| and places 144 // the decoded value into |*code_point|. If the character is valid, we will 145 // return true. If invalid, we'll return false and put the 146 // kUnicodeReplacementCharacter into |*code_point|. 147 // 148 // |*begin| will be updated to point to the last character consumed so it 149 // can be incremented in a loop and will be ready for the next character. 150 // (for a single-byte ASCII character, it will not be changed). 151 // 152 // Implementation is in url_canon_icu.cc. 153 URL_EXPORT bool ReadUTFChar(const char* str, int* begin, int length, 154 unsigned* code_point_out); 155 156 // Generic To-UTF-8 converter. This will call the given append method for each 157 // character that should be appended, with the given output method. Wrappers 158 // are provided below for escaped and non-escaped versions of this. 159 // 160 // The char_value must have already been checked that it's a valid Unicode 161 // character. 162 template<class Output, void Appender(unsigned char, Output*)> 163 inline void DoAppendUTF8(unsigned char_value, Output* output) { 164 if (char_value <= 0x7f) { 165 Appender(static_cast<unsigned char>(char_value), output); 166 } else if (char_value <= 0x7ff) { 167 // 110xxxxx 10xxxxxx 168 Appender(static_cast<unsigned char>(0xC0 | (char_value >> 6)), 169 output); 170 Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)), 171 output); 172 } else if (char_value <= 0xffff) { 173 // 1110xxxx 10xxxxxx 10xxxxxx 174 Appender(static_cast<unsigned char>(0xe0 | (char_value >> 12)), 175 output); 176 Appender(static_cast<unsigned char>(0x80 | ((char_value >> 6) & 0x3f)), 177 output); 178 Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)), 179 output); 180 } else if (char_value <= 0x10FFFF) { // Max unicode code point. 181 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 182 Appender(static_cast<unsigned char>(0xf0 | (char_value >> 18)), 183 output); 184 Appender(static_cast<unsigned char>(0x80 | ((char_value >> 12) & 0x3f)), 185 output); 186 Appender(static_cast<unsigned char>(0x80 | ((char_value >> 6) & 0x3f)), 187 output); 188 Appender(static_cast<unsigned char>(0x80 | (char_value & 0x3f)), 189 output); 190 } else { 191 // Invalid UTF-8 character (>20 bits). 192 NOTREACHED(); 193 } 194 } 195 196 // Helper used by AppendUTF8Value below. We use an unsigned parameter so there 197 // are no funny sign problems with the input, but then have to convert it to 198 // a regular char for appending. 199 inline void AppendCharToOutput(unsigned char ch, CanonOutput* output) { 200 output->push_back(static_cast<char>(ch)); 201 } 202 203 // Writes the given character to the output as UTF-8. This does NO checking 204 // of the validity of the unicode characters; the caller should ensure that 205 // the value it is appending is valid to append. 206 inline void AppendUTF8Value(unsigned char_value, CanonOutput* output) { 207 DoAppendUTF8<CanonOutput, AppendCharToOutput>(char_value, output); 208 } 209 210 // Writes the given character to the output as UTF-8, escaping ALL 211 // characters (even when they are ASCII). This does NO checking of the 212 // validity of the unicode characters; the caller should ensure that the value 213 // it is appending is valid to append. 214 inline void AppendUTF8EscapedValue(unsigned char_value, CanonOutput* output) { 215 DoAppendUTF8<CanonOutput, AppendEscapedChar>(char_value, output); 216 } 217 218 // UTF-16 functions ----------------------------------------------------------- 219 220 // Reads one character in UTF-16 starting at |*begin| in |str| and places 221 // the decoded value into |*code_point|. If the character is valid, we will 222 // return true. If invalid, we'll return false and put the 223 // kUnicodeReplacementCharacter into |*code_point|. 224 // 225 // |*begin| will be updated to point to the last character consumed so it 226 // can be incremented in a loop and will be ready for the next character. 227 // (for a single-16-bit-word character, it will not be changed). 228 // 229 // Implementation is in url_canon_icu.cc. 230 URL_EXPORT bool ReadUTFChar(const base::char16* str, int* begin, int length, 231 unsigned* code_point); 232 233 // Equivalent to U16_APPEND_UNSAFE in ICU but uses our output method. 234 inline void AppendUTF16Value(unsigned code_point, 235 CanonOutputT<base::char16>* output) { 236 if (code_point > 0xffff) { 237 output->push_back(static_cast<base::char16>((code_point >> 10) + 0xd7c0)); 238 output->push_back(static_cast<base::char16>((code_point & 0x3ff) | 0xdc00)); 239 } else { 240 output->push_back(static_cast<base::char16>(code_point)); 241 } 242 } 243 244 // Escaping functions --------------------------------------------------------- 245 246 // Writes the given character to the output as UTF-8, escaped. Call this 247 // function only when the input is wide. Returns true on success. Failure 248 // means there was some problem with the encoding, we'll still try to 249 // update the |*begin| pointer and add a placeholder character to the 250 // output so processing can continue. 251 // 252 // We will append the character starting at ch[begin] with the buffer ch 253 // being |length|. |*begin| will be updated to point to the last character 254 // consumed (we may consume more than one for UTF-16) so that if called in 255 // a loop, incrementing the pointer will move to the next character. 256 // 257 // Every single output character will be escaped. This means that if you 258 // give it an ASCII character as input, it will be escaped. Some code uses 259 // this when it knows that a character is invalid according to its rules 260 // for validity. If you don't want escaping for ASCII characters, you will 261 // have to filter them out prior to calling this function. 262 // 263 // Assumes that ch[begin] is within range in the array, but does not assume 264 // that any following characters are. 265 inline bool AppendUTF8EscapedChar(const base::char16* str, int* begin, 266 int length, CanonOutput* output) { 267 // UTF-16 input. Readchar16 will handle invalid characters for us and give 268 // us the kUnicodeReplacementCharacter, so we don't have to do special 269 // checking after failure, just pass through the failure to the caller. 270 unsigned char_value; 271 bool success = ReadUTFChar(str, begin, length, &char_value); 272 AppendUTF8EscapedValue(char_value, output); 273 return success; 274 } 275 276 // Handles UTF-8 input. See the wide version above for usage. 277 inline bool AppendUTF8EscapedChar(const char* str, int* begin, int length, 278 CanonOutput* output) { 279 // ReadUTF8Char will handle invalid characters for us and give us the 280 // kUnicodeReplacementCharacter, so we don't have to do special checking 281 // after failure, just pass through the failure to the caller. 282 unsigned ch; 283 bool success = ReadUTFChar(str, begin, length, &ch); 284 AppendUTF8EscapedValue(ch, output); 285 return success; 286 } 287 288 // Given a '%' character at |*begin| in the string |spec|, this will decode 289 // the escaped value and put it into |*unescaped_value| on success (returns 290 // true). On failure, this will return false, and will not write into 291 // |*unescaped_value|. 292 // 293 // |*begin| will be updated to point to the last character of the escape 294 // sequence so that when called with the index of a for loop, the next time 295 // through it will point to the next character to be considered. On failure, 296 // |*begin| will be unchanged. 297 inline bool Is8BitChar(char c) { 298 return true; // this case is specialized to avoid a warning 299 } 300 inline bool Is8BitChar(base::char16 c) { 301 return c <= 255; 302 } 303 304 template<typename CHAR> 305 inline bool DecodeEscaped(const CHAR* spec, int* begin, int end, 306 unsigned char* unescaped_value) { 307 if (*begin + 3 > end || 308 !Is8BitChar(spec[*begin + 1]) || !Is8BitChar(spec[*begin + 2])) { 309 // Invalid escape sequence because there's not enough room, or the 310 // digits are not ASCII. 311 return false; 312 } 313 314 unsigned char first = static_cast<unsigned char>(spec[*begin + 1]); 315 unsigned char second = static_cast<unsigned char>(spec[*begin + 2]); 316 if (!IsHexChar(first) || !IsHexChar(second)) { 317 // Invalid hex digits, fail. 318 return false; 319 } 320 321 // Valid escape sequence. 322 *unescaped_value = (HexCharToValue(first) << 4) + HexCharToValue(second); 323 *begin += 2; 324 return true; 325 } 326 327 // Appends the given substring to the output, escaping "some" characters that 328 // it feels may not be safe. It assumes the input values are all contained in 329 // 8-bit although it allows any type. 330 // 331 // This is used in error cases to append invalid output so that it looks 332 // approximately correct. Non-error cases should not call this function since 333 // the escaping rules are not guaranteed! 334 void AppendInvalidNarrowString(const char* spec, int begin, int end, 335 CanonOutput* output); 336 void AppendInvalidNarrowString(const base::char16* spec, int begin, int end, 337 CanonOutput* output); 338 339 // Misc canonicalization helpers ---------------------------------------------- 340 341 // Converts between UTF-8 and UTF-16, returning true on successful conversion. 342 // The output will be appended to the given canonicalizer output (so make sure 343 // it's empty if you want to replace). 344 // 345 // On invalid input, this will still write as much output as possible, 346 // replacing the invalid characters with the "invalid character". It will 347 // return false in the failure case, and the caller should not continue as 348 // normal. 349 URL_EXPORT bool ConvertUTF16ToUTF8(const base::char16* input, int input_len, 350 CanonOutput* output); 351 URL_EXPORT bool ConvertUTF8ToUTF16(const char* input, int input_len, 352 CanonOutputT<base::char16>* output); 353 354 // Converts from UTF-16 to 8-bit using the character set converter. If the 355 // converter is NULL, this will use UTF-8. 356 void ConvertUTF16ToQueryEncoding(const base::char16* input, 357 const url_parse::Component& query, 358 CharsetConverter* converter, 359 CanonOutput* output); 360 361 // Applies the replacements to the given component source. The component source 362 // should be pre-initialized to the "old" base. That is, all pointers will 363 // point to the spec of the old URL, and all of the Parsed components will 364 // be indices into that string. 365 // 366 // The pointers and components in the |source| for all non-NULL strings in the 367 // |repl| (replacements) will be updated to reference those strings. 368 // Canonicalizing with the new |source| and |parsed| can then combine URL 369 // components from many different strings. 370 void SetupOverrideComponents(const char* base, 371 const Replacements<char>& repl, 372 URLComponentSource<char>* source, 373 url_parse::Parsed* parsed); 374 375 // Like the above 8-bit version, except that it additionally converts the 376 // UTF-16 input to UTF-8 before doing the overrides. 377 // 378 // The given utf8_buffer is used to store the converted components. They will 379 // be appended one after another, with the parsed structure identifying the 380 // appropriate substrings. This buffer is a parameter because the source has 381 // no storage, so the buffer must have the same lifetime as the source 382 // parameter owned by the caller. 383 // 384 // THE CALLER MUST NOT ADD TO THE |utf8_buffer| AFTER THIS CALL. Members of 385 // |source| will point into this buffer, which could be invalidated if 386 // additional data is added and the CanonOutput resizes its buffer. 387 // 388 // Returns true on success. Fales means that the input was not valid UTF-16, 389 // although we will have still done the override with "invalid characters" in 390 // place of errors. 391 bool SetupUTF16OverrideComponents(const char* base, 392 const Replacements<base::char16>& repl, 393 CanonOutput* utf8_buffer, 394 URLComponentSource<char>* source, 395 url_parse::Parsed* parsed); 396 397 // Implemented in url_canon_path.cc, these are required by the relative URL 398 // resolver as well, so we declare them here. 399 bool CanonicalizePartialPath(const char* spec, 400 const url_parse::Component& path, 401 int path_begin_in_output, 402 CanonOutput* output); 403 bool CanonicalizePartialPath(const base::char16* spec, 404 const url_parse::Component& path, 405 int path_begin_in_output, 406 CanonOutput* output); 407 408 #ifndef WIN32 409 410 // Implementations of Windows' int-to-string conversions 411 URL_EXPORT int _itoa_s(int value, char* buffer, size_t size_in_chars, 412 int radix); 413 URL_EXPORT int _itow_s(int value, base::char16* buffer, size_t size_in_chars, 414 int radix); 415 416 // Secure template overloads for these functions 417 template<size_t N> 418 inline int _itoa_s(int value, char (&buffer)[N], int radix) { 419 return _itoa_s(value, buffer, N, radix); 420 } 421 422 template<size_t N> 423 inline int _itow_s(int value, base::char16 (&buffer)[N], int radix) { 424 return _itow_s(value, buffer, N, radix); 425 } 426 427 // _strtoui64 and strtoull behave the same 428 inline unsigned long long _strtoui64(const char* nptr, 429 char** endptr, int base) { 430 return strtoull(nptr, endptr, base); 431 } 432 433 #endif // WIN32 434 435 } // namespace url_canon 436 437 #endif // URL_URL_CANON_INTERNAL_H_ 438