1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "net/base/escape.h" 6 7 #include <algorithm> 8 9 #include "base/logging.h" 10 #include "base/memory/scoped_ptr.h" 11 #include "base/strings/string_piece.h" 12 #include "base/strings/string_util.h" 13 #include "base/strings/utf_offset_string_conversions.h" 14 #include "base/strings/utf_string_conversions.h" 15 16 namespace net { 17 18 namespace { 19 20 const char kHexString[] = "0123456789ABCDEF"; 21 inline char IntToHex(int i) { 22 DCHECK_GE(i, 0) << i << " not a hex value"; 23 DCHECK_LE(i, 15) << i << " not a hex value"; 24 return kHexString[i]; 25 } 26 27 // A fast bit-vector map for ascii characters. 28 // 29 // Internally stores 256 bits in an array of 8 ints. 30 // Does quick bit-flicking to lookup needed characters. 31 struct Charmap { 32 bool Contains(unsigned char c) const { 33 return ((map[c >> 5] & (1 << (c & 31))) != 0); 34 } 35 36 uint32 map[8]; 37 }; 38 39 // Given text to escape and a Charmap defining which values to escape, 40 // return an escaped string. If use_plus is true, spaces are converted 41 // to +, otherwise, if spaces are in the charmap, they are converted to 42 // %20. 43 std::string Escape(const std::string& text, const Charmap& charmap, 44 bool use_plus) { 45 std::string escaped; 46 escaped.reserve(text.length() * 3); 47 for (unsigned int i = 0; i < text.length(); ++i) { 48 unsigned char c = static_cast<unsigned char>(text[i]); 49 if (use_plus && ' ' == c) { 50 escaped.push_back('+'); 51 } else if (charmap.Contains(c)) { 52 escaped.push_back('%'); 53 escaped.push_back(IntToHex(c >> 4)); 54 escaped.push_back(IntToHex(c & 0xf)); 55 } else { 56 escaped.push_back(c); 57 } 58 } 59 return escaped; 60 } 61 62 // Contains nonzero when the corresponding character is unescapable for normal 63 // URLs. These characters are the ones that may change the parsing of a URL, so 64 // we don't want to unescape them sometimes. In many case we won't want to 65 // unescape spaces, but that is controlled by parameters to Unescape*. 66 // 67 // The basic rule is that we can't unescape anything that would changing parsing 68 // like # or ?. We also can't unescape &, =, or + since that could be part of a 69 // query and that could change the server's parsing of the query. Nor can we 70 // unescape \ since src/url/ will convert it to a /. 71 // 72 // Lastly, we can't unescape anything that doesn't have a canonical 73 // representation in a URL. This means that unescaping will change the URL, and 74 // you could get different behavior if you copy and paste the URL, or press 75 // enter in the URL bar. The list of characters that fall into this category 76 // are the ones labeled PASS (allow either escaped or unescaped) in the big 77 // lookup table at the top of url/url_canon_path.cc. Also, characters 78 // that have CHAR_QUERY set in url/url_canon_internal.cc but are not 79 // allowed in query strings according to http://www.ietf.org/rfc/rfc3261.txt are 80 // not unescaped, to avoid turning a valid url according to spec into an 81 // invalid one. 82 const char kUrlUnescape[128] = { 83 // NULL, control chars... 84 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86 // ' ' ! " # $ % & ' ( ) * + , - . / 87 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 88 // 0 1 2 3 4 5 6 7 8 9 : ; < = > ? 89 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 90 // @ A B C D E F G H I J K L M N O 91 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 92 // P Q R S T U V W X Y Z [ \ ] ^ _ 93 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 94 // ` a b c d e f g h i j k l m n o 95 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 96 // p q r s t u v w x y z { | } ~ <NBSP> 97 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0 98 }; 99 100 // Attempts to unescape the sequence at |index| within |escaped_text|. If 101 // successful, sets |value| to the unescaped value. Returns whether 102 // unescaping succeeded. 103 template<typename STR> 104 bool UnescapeUnsignedCharAtIndex(const STR& escaped_text, 105 size_t index, 106 unsigned char* value) { 107 if ((index + 2) >= escaped_text.size()) 108 return false; 109 if (escaped_text[index] != '%') 110 return false; 111 const typename STR::value_type most_sig_digit( 112 static_cast<typename STR::value_type>(escaped_text[index + 1])); 113 const typename STR::value_type least_sig_digit( 114 static_cast<typename STR::value_type>(escaped_text[index + 2])); 115 if (IsHexDigit(most_sig_digit) && IsHexDigit(least_sig_digit)) { 116 *value = HexDigitToInt(most_sig_digit) * 16 + 117 HexDigitToInt(least_sig_digit); 118 return true; 119 } 120 return false; 121 } 122 123 // Unescapes |escaped_text| according to |rules|, returning the resulting 124 // string. Fills in an |adjustments| parameter, if non-NULL, so it reflects 125 // the alterations done to the string that are not one-character-to-one- 126 // character. The resulting |adjustments| will always be sorted by increasing 127 // offset. 128 template<typename STR> 129 STR UnescapeURLWithAdjustmentsImpl( 130 const STR& escaped_text, 131 UnescapeRule::Type rules, 132 base::OffsetAdjuster::Adjustments* adjustments) { 133 if (adjustments) 134 adjustments->clear(); 135 // Do not unescape anything, return the |escaped_text| text. 136 if (rules == UnescapeRule::NONE) 137 return escaped_text; 138 139 // The output of the unescaping is always smaller than the input, so we can 140 // reserve the input size to make sure we have enough buffer and don't have 141 // to allocate in the loop below. 142 STR result; 143 result.reserve(escaped_text.length()); 144 145 // Locations of adjusted text. 146 for (size_t i = 0, max = escaped_text.size(); i < max; ++i) { 147 if (static_cast<unsigned char>(escaped_text[i]) >= 128) { 148 // Non ASCII character, append as is. 149 result.push_back(escaped_text[i]); 150 continue; 151 } 152 153 unsigned char first_byte; 154 if (UnescapeUnsignedCharAtIndex(escaped_text, i, &first_byte)) { 155 // Per http://tools.ietf.org/html/rfc3987#section-4.1, the following BiDi 156 // control characters are not allowed to appear unescaped in URLs: 157 // 158 // U+200E LEFT-TO-RIGHT MARK (%E2%80%8E) 159 // U+200F RIGHT-TO-LEFT MARK (%E2%80%8F) 160 // U+202A LEFT-TO-RIGHT EMBEDDING (%E2%80%AA) 161 // U+202B RIGHT-TO-LEFT EMBEDDING (%E2%80%AB) 162 // U+202C POP DIRECTIONAL FORMATTING (%E2%80%AC) 163 // U+202D LEFT-TO-RIGHT OVERRIDE (%E2%80%AD) 164 // U+202E RIGHT-TO-LEFT OVERRIDE (%E2%80%AE) 165 // 166 // Additionally, the Unicode Technical Report (TR9) as referenced by RFC 167 // 3987 above has since added some new BiDi control characters. 168 // http://www.unicode.org/reports/tr9 169 // 170 // U+061C ARABIC LETTER MARK (%D8%9C) 171 // U+2066 LEFT-TO-RIGHT ISOLATE (%E2%81%A6) 172 // U+2067 RIGHT-TO-LEFT ISOLATE (%E2%81%A7) 173 // U+2068 FIRST STRONG ISOLATE (%E2%81%A8) 174 // U+2069 POP DIRECTIONAL ISOLATE (%E2%81%A9) 175 176 unsigned char second_byte; 177 // Check for ALM. 178 if ((first_byte == 0xD8) && 179 UnescapeUnsignedCharAtIndex(escaped_text, i + 3, &second_byte) && 180 (second_byte == 0x9c)) { 181 result.append(escaped_text, i, 6); 182 i += 5; 183 continue; 184 } 185 186 // Check for other BiDi control characters. 187 if ((first_byte == 0xE2) && 188 UnescapeUnsignedCharAtIndex(escaped_text, i + 3, &second_byte) && 189 ((second_byte == 0x80) || (second_byte == 0x81))) { 190 unsigned char third_byte; 191 if (UnescapeUnsignedCharAtIndex(escaped_text, i + 6, &third_byte) && 192 ((second_byte == 0x80) ? 193 ((third_byte == 0x8E) || (third_byte == 0x8F) || 194 ((third_byte >= 0xAA) && (third_byte <= 0xAE))) : 195 ((third_byte >= 0xA6) && (third_byte <= 0xA9)))) { 196 result.append(escaped_text, i, 9); 197 i += 8; 198 continue; 199 } 200 } 201 202 if (first_byte >= 0x80 || // Unescape all high-bit characters. 203 // For 7-bit characters, the lookup table tells us all valid chars. 204 (kUrlUnescape[first_byte] || 205 // ...and we allow some additional unescaping when flags are set. 206 (first_byte == ' ' && (rules & UnescapeRule::SPACES)) || 207 // Allow any of the prohibited but non-control characters when 208 // we're doing "special" chars. 209 (first_byte > ' ' && (rules & UnescapeRule::URL_SPECIAL_CHARS)) || 210 // Additionally allow control characters if requested. 211 (first_byte < ' ' && (rules & UnescapeRule::CONTROL_CHARS)))) { 212 // Use the unescaped version of the character. 213 if (adjustments) 214 adjustments->push_back(base::OffsetAdjuster::Adjustment(i, 3, 1)); 215 result.push_back(first_byte); 216 i += 2; 217 } else { 218 // Keep escaped. Append a percent and we'll get the following two 219 // digits on the next loops through. 220 result.push_back('%'); 221 } 222 } else if ((rules & UnescapeRule::REPLACE_PLUS_WITH_SPACE) && 223 escaped_text[i] == '+') { 224 result.push_back(' '); 225 } else { 226 // Normal case for unescaped characters. 227 result.push_back(escaped_text[i]); 228 } 229 } 230 231 return result; 232 } 233 234 template <class str> 235 void AppendEscapedCharForHTMLImpl(typename str::value_type c, str* output) { 236 static const struct { 237 char key; 238 const char* replacement; 239 } kCharsToEscape[] = { 240 { '<', "<" }, 241 { '>', ">" }, 242 { '&', "&" }, 243 { '"', """ }, 244 { '\'', "'" }, 245 }; 246 size_t k; 247 for (k = 0; k < ARRAYSIZE_UNSAFE(kCharsToEscape); ++k) { 248 if (c == kCharsToEscape[k].key) { 249 const char* p = kCharsToEscape[k].replacement; 250 while (*p) 251 output->push_back(*p++); 252 break; 253 } 254 } 255 if (k == ARRAYSIZE_UNSAFE(kCharsToEscape)) 256 output->push_back(c); 257 } 258 259 template <class str> 260 str EscapeForHTMLImpl(const str& input) { 261 str result; 262 result.reserve(input.size()); // Optimize for no escaping. 263 264 for (typename str::const_iterator i = input.begin(); i != input.end(); ++i) 265 AppendEscapedCharForHTMLImpl(*i, &result); 266 267 return result; 268 } 269 270 // Everything except alphanumerics and !'()*-._~ 271 // See RFC 2396 for the list of reserved characters. 272 static const Charmap kQueryCharmap = {{ 273 0xffffffffL, 0xfc00987dL, 0x78000001L, 0xb8000001L, 274 0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL 275 }}; 276 277 // non-printable, non-7bit, and (including space) "#%:<>?[\]^`{|} 278 static const Charmap kPathCharmap = {{ 279 0xffffffffL, 0xd400002dL, 0x78000000L, 0xb8000001L, 280 0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL 281 }}; 282 283 // non-printable, non-7bit, and (including space) ?>=<;+'&%$#"![\]^`{|} 284 static const Charmap kUrlEscape = {{ 285 0xffffffffL, 0xf80008fdL, 0x78000001L, 0xb8000001L, 286 0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL 287 }}; 288 289 // non-7bit 290 static const Charmap kNonASCIICharmap = {{ 291 0x00000000L, 0x00000000L, 0x00000000L, 0x00000000L, 292 0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL 293 }}; 294 295 // Everything except alphanumerics, the reserved characters(;/?:@&=+$,) and 296 // !'()*-._~% 297 static const Charmap kExternalHandlerCharmap = {{ 298 0xffffffffL, 0x5000080dL, 0x68000000L, 0xb8000001L, 299 0xffffffffL, 0xffffffffL, 0xffffffffL, 0xffffffffL 300 }}; 301 302 } // namespace 303 304 std::string EscapeQueryParamValue(const std::string& text, bool use_plus) { 305 return Escape(text, kQueryCharmap, use_plus); 306 } 307 308 std::string EscapePath(const std::string& path) { 309 return Escape(path, kPathCharmap, false); 310 } 311 312 std::string EscapeUrlEncodedData(const std::string& path, bool use_plus) { 313 return Escape(path, kUrlEscape, use_plus); 314 } 315 316 std::string EscapeNonASCII(const std::string& input) { 317 return Escape(input, kNonASCIICharmap, false); 318 } 319 320 std::string EscapeExternalHandlerValue(const std::string& text) { 321 return Escape(text, kExternalHandlerCharmap, false); 322 } 323 324 void AppendEscapedCharForHTML(char c, std::string* output) { 325 AppendEscapedCharForHTMLImpl(c, output); 326 } 327 328 std::string EscapeForHTML(const std::string& input) { 329 return EscapeForHTMLImpl(input); 330 } 331 332 base::string16 EscapeForHTML(const base::string16& input) { 333 return EscapeForHTMLImpl(input); 334 } 335 336 std::string UnescapeURLComponent(const std::string& escaped_text, 337 UnescapeRule::Type rules) { 338 return UnescapeURLWithAdjustmentsImpl(escaped_text, rules, NULL); 339 } 340 341 base::string16 UnescapeURLComponent(const base::string16& escaped_text, 342 UnescapeRule::Type rules) { 343 return UnescapeURLWithAdjustmentsImpl(escaped_text, rules, NULL); 344 } 345 346 base::string16 UnescapeAndDecodeUTF8URLComponent(const std::string& text, 347 UnescapeRule::Type rules) { 348 return UnescapeAndDecodeUTF8URLComponentWithAdjustments(text, rules, NULL); 349 } 350 351 base::string16 UnescapeAndDecodeUTF8URLComponentWithAdjustments( 352 const std::string& text, 353 UnescapeRule::Type rules, 354 base::OffsetAdjuster::Adjustments* adjustments) { 355 base::string16 result; 356 base::OffsetAdjuster::Adjustments unescape_adjustments; 357 std::string unescaped_url(UnescapeURLWithAdjustmentsImpl( 358 text, rules, &unescape_adjustments)); 359 if (base::UTF8ToUTF16WithAdjustments(unescaped_url.data(), 360 unescaped_url.length(), 361 &result, adjustments)) { 362 // Character set looks like it's valid. 363 if (adjustments) { 364 base::OffsetAdjuster::MergeSequentialAdjustments(unescape_adjustments, 365 adjustments); 366 } 367 return result; 368 } 369 // Character set is not valid. Return the escaped version. 370 return base::UTF8ToUTF16WithAdjustments(text, adjustments); 371 } 372 373 base::string16 UnescapeForHTML(const base::string16& input) { 374 static const struct { 375 const char* ampersand_code; 376 const char replacement; 377 } kEscapeToChars[] = { 378 { "<", '<' }, 379 { ">", '>' }, 380 { "&", '&' }, 381 { """, '"' }, 382 { "'", '\''}, 383 }; 384 385 if (input.find(base::ASCIIToUTF16("&")) == std::string::npos) 386 return input; 387 388 base::string16 ampersand_chars[ARRAYSIZE_UNSAFE(kEscapeToChars)]; 389 base::string16 text(input); 390 for (base::string16::iterator iter = text.begin(); 391 iter != text.end(); ++iter) { 392 if (*iter == '&') { 393 // Potential ampersand encode char. 394 size_t index = iter - text.begin(); 395 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kEscapeToChars); i++) { 396 if (ampersand_chars[i].empty()) { 397 ampersand_chars[i] = 398 base::ASCIIToUTF16(kEscapeToChars[i].ampersand_code); 399 } 400 if (text.find(ampersand_chars[i], index) == index) { 401 text.replace(iter, iter + ampersand_chars[i].length(), 402 1, kEscapeToChars[i].replacement); 403 break; 404 } 405 } 406 } 407 } 408 return text; 409 } 410 411 } // namespace net 412
