1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // The rules for parsing content-types were borrowed from Firefox: 6 // http://lxr.mozilla.org/mozilla/source/netwerk/base/src/nsURLHelper.cpp#834 7 8 #include "net/http/http_util.h" 9 10 #include <algorithm> 11 12 #include "base/logging.h" 13 #include "base/string_piece.h" 14 #include "base/string_util.h" 15 #include "net/base/net_util.h" 16 17 using std::string; 18 19 namespace net { 20 21 //----------------------------------------------------------------------------- 22 23 // Return the index of the closing quote of the string, if any. 24 static size_t FindStringEnd(const string& line, size_t start, char delim) { 25 DCHECK(start < line.length() && line[start] == delim && 26 (delim == '"' || delim == '\'')); 27 28 const char set[] = { delim, '\\', '\0' }; 29 for (;;) { 30 // start points to either the start quote or the last 31 // escaped char (the char following a '\\') 32 33 size_t end = line.find_first_of(set, start + 1); 34 if (end == string::npos) 35 return line.length(); 36 37 if (line[end] == '\\') { 38 // Hit a backslash-escaped char. Need to skip over it. 39 start = end + 1; 40 if (start == line.length()) 41 return start; 42 43 // Go back to looking for the next escape or the string end 44 continue; 45 } 46 47 return end; 48 } 49 50 NOTREACHED(); 51 return line.length(); 52 } 53 54 //----------------------------------------------------------------------------- 55 56 // static 57 size_t HttpUtil::FindDelimiter(const string& line, size_t search_start, 58 char delimiter) { 59 do { 60 // search_start points to the spot from which we should start looking 61 // for the delimiter. 62 const char delim_str[] = { delimiter, '"', '\'', '\0' }; 63 size_t cur_delim_pos = line.find_first_of(delim_str, search_start); 64 if (cur_delim_pos == string::npos) 65 return line.length(); 66 67 char ch = line[cur_delim_pos]; 68 if (ch == delimiter) { 69 // Found delimiter 70 return cur_delim_pos; 71 } 72 73 // We hit the start of a quoted string. Look for its end. 74 search_start = FindStringEnd(line, cur_delim_pos, ch); 75 if (search_start == line.length()) 76 return search_start; 77 78 ++search_start; 79 80 // search_start now points to the first char after the end of the 81 // string, so just go back to the top of the loop and look for 82 // |delimiter| again. 83 } while (true); 84 85 NOTREACHED(); 86 return line.length(); 87 } 88 89 // static 90 void HttpUtil::ParseContentType(const string& content_type_str, 91 string* mime_type, string* charset, 92 bool *had_charset) { 93 // Trim leading and trailing whitespace from type. We include '(' in 94 // the trailing trim set to catch media-type comments, which are not at all 95 // standard, but may occur in rare cases. 96 size_t type_val = content_type_str.find_first_not_of(HTTP_LWS); 97 type_val = std::min(type_val, content_type_str.length()); 98 size_t type_end = content_type_str.find_first_of(HTTP_LWS ";(", type_val); 99 if (string::npos == type_end) 100 type_end = content_type_str.length(); 101 102 size_t charset_val = 0; 103 size_t charset_end = 0; 104 105 // Iterate over parameters 106 bool type_has_charset = false; 107 size_t param_start = content_type_str.find_first_of(';', type_end); 108 if (param_start != string::npos) { 109 // We have parameters. Iterate over them. 110 size_t cur_param_start = param_start + 1; 111 do { 112 size_t cur_param_end = 113 FindDelimiter(content_type_str, cur_param_start, ';'); 114 115 size_t param_name_start = content_type_str.find_first_not_of(HTTP_LWS, 116 cur_param_start); 117 param_name_start = std::min(param_name_start, cur_param_end); 118 119 static const char charset_str[] = "charset="; 120 size_t charset_end_offset = std::min(param_name_start + 121 sizeof(charset_str) - 1, cur_param_end); 122 if (LowerCaseEqualsASCII(content_type_str.begin() + param_name_start, 123 content_type_str.begin() + charset_end_offset, charset_str)) { 124 charset_val = param_name_start + sizeof(charset_str) - 1; 125 charset_end = cur_param_end; 126 type_has_charset = true; 127 } 128 129 cur_param_start = cur_param_end + 1; 130 } while (cur_param_start < content_type_str.length()); 131 } 132 133 if (type_has_charset) { 134 // Trim leading and trailing whitespace from charset_val. We include 135 // '(' in the trailing trim set to catch media-type comments, which are 136 // not at all standard, but may occur in rare cases. 137 charset_val = content_type_str.find_first_not_of(HTTP_LWS, charset_val); 138 charset_val = std::min(charset_val, charset_end); 139 char first_char = content_type_str[charset_val]; 140 if (first_char == '"' || first_char == '\'') { 141 charset_end = FindStringEnd(content_type_str, charset_val, first_char); 142 ++charset_val; 143 DCHECK(charset_end >= charset_val); 144 } else { 145 charset_end = std::min(content_type_str.find_first_of(HTTP_LWS ";(", 146 charset_val), 147 charset_end); 148 } 149 } 150 151 // if the server sent "*/*", it is meaningless, so do not store it. 152 // also, if type_val is the same as mime_type, then just update the 153 // charset. however, if charset is empty and mime_type hasn't 154 // changed, then don't wipe-out an existing charset. We 155 // also want to reject a mime-type if it does not include a slash. 156 // some servers give junk after the charset parameter, which may 157 // include a comma, so this check makes us a bit more tolerant. 158 if (content_type_str.length() != 0 && 159 content_type_str != "*/*" && 160 content_type_str.find_first_of('/') != string::npos) { 161 // Common case here is that mime_type is empty 162 bool eq = !mime_type->empty() && 163 LowerCaseEqualsASCII(content_type_str.begin() + type_val, 164 content_type_str.begin() + type_end, 165 mime_type->data()); 166 if (!eq) { 167 mime_type->assign(content_type_str.begin() + type_val, 168 content_type_str.begin() + type_end); 169 StringToLowerASCII(mime_type); 170 } 171 if ((!eq && *had_charset) || type_has_charset) { 172 *had_charset = true; 173 charset->assign(content_type_str.begin() + charset_val, 174 content_type_str.begin() + charset_end); 175 StringToLowerASCII(charset); 176 } 177 } 178 } 179 180 // static 181 // Parse the Range header according to RFC 2616 14.35.1 182 // ranges-specifier = byte-ranges-specifier 183 // byte-ranges-specifier = bytes-unit "=" byte-range-set 184 // byte-range-set = 1#( byte-range-spec | suffix-byte-range-spec ) 185 // byte-range-spec = first-byte-pos "-" [last-byte-pos] 186 // first-byte-pos = 1*DIGIT 187 // last-byte-pos = 1*DIGIT 188 bool HttpUtil::ParseRanges(const std::string& headers, 189 std::vector<HttpByteRange>* ranges) { 190 std::string ranges_specifier; 191 HttpUtil::HeadersIterator it(headers.begin(), headers.end(), "\r\n"); 192 193 while (it.GetNext()) { 194 // Look for "Range" header. 195 if (!LowerCaseEqualsASCII(it.name(), "range")) 196 continue; 197 ranges_specifier = it.values(); 198 // We just care about the first "Range" header, so break here. 199 break; 200 } 201 202 if (ranges_specifier.empty()) 203 return false; 204 205 size_t equal_char_offset = ranges_specifier.find('='); 206 if (equal_char_offset == std::string::npos) 207 return false; 208 209 // Try to extract bytes-unit part. 210 std::string::const_iterator bytes_unit_begin = ranges_specifier.begin(); 211 std::string::const_iterator bytes_unit_end = bytes_unit_begin + 212 equal_char_offset; 213 std::string::const_iterator byte_range_set_begin = bytes_unit_end + 1; 214 std::string::const_iterator byte_range_set_end = ranges_specifier.end(); 215 216 TrimLWS(&bytes_unit_begin, &bytes_unit_end); 217 // "bytes" unit identifier is not found. 218 if (!LowerCaseEqualsASCII(bytes_unit_begin, bytes_unit_end, "bytes")) 219 return false; 220 221 ValuesIterator byte_range_set_iterator(byte_range_set_begin, 222 byte_range_set_end, ','); 223 while (byte_range_set_iterator.GetNext()) { 224 size_t minus_char_offset = byte_range_set_iterator.value().find('-'); 225 // If '-' character is not found, reports failure. 226 if (minus_char_offset == std::string::npos) 227 return false; 228 229 std::string::const_iterator first_byte_pos_begin = 230 byte_range_set_iterator.value_begin(); 231 std::string::const_iterator first_byte_pos_end = 232 first_byte_pos_begin + minus_char_offset; 233 TrimLWS(&first_byte_pos_begin, &first_byte_pos_end); 234 std::string first_byte_pos(first_byte_pos_begin, first_byte_pos_end); 235 236 HttpByteRange range; 237 // Try to obtain first-byte-pos. 238 if (!first_byte_pos.empty()) { 239 int64 first_byte_position = -1; 240 if (!StringToInt64(first_byte_pos, &first_byte_position)) 241 return false; 242 range.set_first_byte_position(first_byte_position); 243 } 244 245 std::string::const_iterator last_byte_pos_begin = 246 byte_range_set_iterator.value_begin() + minus_char_offset + 1; 247 std::string::const_iterator last_byte_pos_end = 248 byte_range_set_iterator.value_end(); 249 TrimLWS(&last_byte_pos_begin, &last_byte_pos_end); 250 std::string last_byte_pos(last_byte_pos_begin, last_byte_pos_end); 251 252 // We have last-byte-pos or suffix-byte-range-spec in this case. 253 if (!last_byte_pos.empty()) { 254 int64 last_byte_position; 255 if (!StringToInt64(last_byte_pos, &last_byte_position)) 256 return false; 257 if (range.HasFirstBytePosition()) 258 range.set_last_byte_position(last_byte_position); 259 else 260 range.set_suffix_length(last_byte_position); 261 } else if (!range.HasFirstBytePosition()) { 262 return false; 263 } 264 265 // Do a final check on the HttpByteRange object. 266 if (!range.IsValid()) 267 return false; 268 ranges->push_back(range); 269 } 270 return ranges->size() > 0; 271 } 272 273 // static 274 bool HttpUtil::HasHeader(const std::string& headers, const char* name) { 275 size_t name_len = strlen(name); 276 string::const_iterator it = 277 std::search(headers.begin(), 278 headers.end(), 279 name, 280 name + name_len, 281 CaseInsensitiveCompareASCII<char>()); 282 if (it == headers.end()) 283 return false; 284 285 // ensure match is prefixed by newline 286 if (it != headers.begin() && it[-1] != '\n') 287 return false; 288 289 // ensure match is suffixed by colon 290 if (it + name_len >= headers.end() || it[name_len] != ':') 291 return false; 292 293 return true; 294 } 295 296 // static 297 std::string HttpUtil::StripHeaders(const std::string& headers, 298 const char* const headers_to_remove[], 299 size_t headers_to_remove_len) { 300 std::string stripped_headers; 301 net::HttpUtil::HeadersIterator it(headers.begin(), headers.end(), "\r\n"); 302 303 while (it.GetNext()) { 304 bool should_remove = false; 305 for (size_t i = 0; i < headers_to_remove_len; ++i) { 306 if (LowerCaseEqualsASCII(it.name_begin(), it.name_end(), 307 headers_to_remove[i])) { 308 should_remove = true; 309 break; 310 } 311 } 312 if (!should_remove) { 313 // Assume that name and values are on the same line. 314 stripped_headers.append(it.name_begin(), it.values_end()); 315 stripped_headers.append("\r\n"); 316 } 317 } 318 return stripped_headers; 319 } 320 321 // static 322 bool HttpUtil::IsNonCoalescingHeader(string::const_iterator name_begin, 323 string::const_iterator name_end) { 324 // NOTE: "set-cookie2" headers do not support expires attributes, so we don't 325 // have to list them here. 326 const char* kNonCoalescingHeaders[] = { 327 "date", 328 "expires", 329 "last-modified", 330 "location", // See bug 1050541 for details 331 "retry-after", 332 "set-cookie", 333 // The format of auth-challenges mixes both space separated tokens and 334 // comma separated properties, so coalescing on comma won't work. 335 "www-authenticate", 336 "proxy-authenticate" 337 }; 338 for (size_t i = 0; i < arraysize(kNonCoalescingHeaders); ++i) { 339 if (LowerCaseEqualsASCII(name_begin, name_end, kNonCoalescingHeaders[i])) 340 return true; 341 } 342 return false; 343 } 344 345 bool HttpUtil::IsLWS(char c) { 346 return strchr(HTTP_LWS, c) != NULL; 347 } 348 349 void HttpUtil::TrimLWS(string::const_iterator* begin, 350 string::const_iterator* end) { 351 // leading whitespace 352 while (*begin < *end && IsLWS((*begin)[0])) 353 ++(*begin); 354 355 // trailing whitespace 356 while (*begin < *end && IsLWS((*end)[-1])) 357 --(*end); 358 } 359 360 // static 361 bool HttpUtil::IsQuote(char c) { 362 // Single quote mark isn't actually part of quoted-text production, 363 // but apparently some servers rely on this. 364 return c == '"' || c == '\''; 365 } 366 367 // static 368 std::string HttpUtil::Unquote(std::string::const_iterator begin, 369 std::string::const_iterator end) { 370 // Empty string 371 if (begin == end) 372 return std::string(); 373 374 // Nothing to unquote. 375 if (!IsQuote(*begin)) 376 return std::string(begin, end); 377 378 // No terminal quote mark. 379 if (end - begin < 2 || *begin != *(end - 1)) 380 return std::string(begin, end); 381 382 // Strip quotemarks 383 ++begin; 384 --end; 385 386 // Unescape quoted-pair (defined in RFC 2616 section 2.2) 387 std::string unescaped; 388 bool prev_escape = false; 389 for (; begin != end; ++begin) { 390 char c = *begin; 391 if (c == '\\' && !prev_escape) { 392 prev_escape = true; 393 continue; 394 } 395 prev_escape = false; 396 unescaped.push_back(c); 397 } 398 return unescaped; 399 } 400 401 // static 402 std::string HttpUtil::Unquote(const std::string& str) { 403 return Unquote(str.begin(), str.end()); 404 } 405 406 // static 407 std::string HttpUtil::Quote(const std::string& str) { 408 std::string escaped; 409 escaped.reserve(2 + str.size()); 410 411 std::string::const_iterator begin = str.begin(); 412 std::string::const_iterator end = str.end(); 413 414 // Esape any backslashes or quotemarks within the string, and 415 // then surround with quotes. 416 escaped.push_back('"'); 417 for (; begin != end; ++begin) { 418 char c = *begin; 419 if (c == '"' || c == '\\') 420 escaped.push_back('\\'); 421 escaped.push_back(c); 422 } 423 escaped.push_back('"'); 424 return escaped; 425 } 426 427 // Find the "http" substring in a status line. This allows for 428 // some slop at the start. If the "http" string could not be found 429 // then returns -1. 430 // static 431 int HttpUtil::LocateStartOfStatusLine(const char* buf, int buf_len) { 432 const int slop = 4; 433 const int http_len = 4; 434 435 if (buf_len >= http_len) { 436 int i_max = std::min(buf_len - http_len, slop); 437 for (int i = 0; i <= i_max; ++i) { 438 if (LowerCaseEqualsASCII(buf + i, buf + i + http_len, "http")) 439 return i; 440 } 441 } 442 return -1; // Not found 443 } 444 445 int HttpUtil::LocateEndOfHeaders(const char* buf, int buf_len, int i) { 446 bool was_lf = false; 447 char last_c = '\0'; 448 for (; i < buf_len; ++i) { 449 char c = buf[i]; 450 if (c == '\n') { 451 if (was_lf) 452 return i + 1; 453 was_lf = true; 454 } else if (c != '\r' || last_c != '\n') { 455 was_lf = false; 456 } 457 last_c = c; 458 } 459 return -1; 460 } 461 462 // In order for a line to be continuable, it must specify a 463 // non-blank header-name. Line continuations are specifically for 464 // header values -- do not allow headers names to span lines. 465 static bool IsLineSegmentContinuable(const char* begin, const char* end) { 466 if (begin == end) 467 return false; 468 469 const char* colon = std::find(begin, end, ':'); 470 if (colon == end) 471 return false; 472 473 const char* name_begin = begin; 474 const char* name_end = colon; 475 476 // Name can't be empty. 477 if (name_begin == name_end) 478 return false; 479 480 // Can't start with LWS (this would imply the segment is a continuation) 481 if (HttpUtil::IsLWS(*name_begin)) 482 return false; 483 484 return true; 485 } 486 487 // Helper used by AssembleRawHeaders, to find the end of the status line. 488 static const char* FindStatusLineEnd(const char* begin, const char* end) { 489 size_t i = base::StringPiece(begin, end - begin).find_first_of("\r\n"); 490 if (i == base::StringPiece::npos) 491 return end; 492 return begin + i; 493 } 494 495 // Helper used by AssembleRawHeaders, to skip past leading LWS. 496 static const char* FindFirstNonLWS(const char* begin, const char* end) { 497 for (const char* cur = begin; cur != end; ++cur) { 498 if (!HttpUtil::IsLWS(*cur)) 499 return cur; 500 } 501 return end; // Not found. 502 } 503 504 std::string HttpUtil::AssembleRawHeaders(const char* input_begin, 505 int input_len) { 506 std::string raw_headers; 507 raw_headers.reserve(input_len); 508 509 const char* input_end = input_begin + input_len; 510 511 // Skip any leading slop, since the consumers of this output 512 // (HttpResponseHeaders) don't deal with it. 513 int status_begin_offset = LocateStartOfStatusLine(input_begin, input_len); 514 if (status_begin_offset != -1) 515 input_begin += status_begin_offset; 516 517 // Copy the status line. 518 const char* status_line_end = FindStatusLineEnd(input_begin, input_end); 519 raw_headers.append(input_begin, status_line_end); 520 521 // After the status line, every subsequent line is a header line segment. 522 // Should a segment start with LWS, it is a continuation of the previous 523 // line's field-value. 524 525 // TODO(ericroman): is this too permissive? (delimits on [\r\n]+) 526 CStringTokenizer lines(status_line_end, input_end, "\r\n"); 527 528 // This variable is true when the previous line was continuable. 529 bool prev_line_continuable = false; 530 531 while (lines.GetNext()) { 532 const char* line_begin = lines.token_begin(); 533 const char* line_end = lines.token_end(); 534 535 if (prev_line_continuable && IsLWS(*line_begin)) { 536 // Join continuation; reduce the leading LWS to a single SP. 537 raw_headers.push_back(' '); 538 raw_headers.append(FindFirstNonLWS(line_begin, line_end), line_end); 539 } else { 540 // Terminate the previous line. 541 raw_headers.push_back('\0'); 542 543 // Copy the raw data to output. 544 raw_headers.append(line_begin, line_end); 545 546 // Check if the current line can be continued. 547 prev_line_continuable = IsLineSegmentContinuable(line_begin, line_end); 548 } 549 } 550 551 raw_headers.append("\0\0", 2); 552 return raw_headers; 553 } 554 555 // TODO(jungshik): 1. If the list is 'fr-CA,fr-FR,en,de', we have to add 556 // 'fr' after 'fr-CA' with the same q-value as 'fr-CA' because 557 // web servers, in general, do not fall back to 'fr' and may end up picking 558 // 'en' which has a lower preference than 'fr-CA' and 'fr-FR'. 559 // 2. This function assumes that the input is a comma separated list 560 // without any whitespace. As long as it comes from the preference and 561 // a user does not manually edit the preference file, it's the case. Still, 562 // we may have to make it more robust. 563 std::string HttpUtil::GenerateAcceptLanguageHeader( 564 const std::string& raw_language_list) { 565 // We use integers for qvalue and qvalue decrement that are 10 times 566 // larger than actual values to avoid a problem with comparing 567 // two floating point numbers. 568 const unsigned int kQvalueDecrement10 = 2; 569 unsigned int qvalue10 = 10; 570 StringTokenizer t(raw_language_list, ","); 571 std::string lang_list_with_q; 572 while (t.GetNext()) { 573 std::string language = t.token(); 574 if (qvalue10 == 10) { 575 // q=1.0 is implicit. 576 lang_list_with_q = language; 577 } else { 578 DCHECK_LT(qvalue10, 10U); 579 StringAppendF(&lang_list_with_q, ",%s;q=0.%d", language.c_str(), 580 qvalue10); 581 } 582 // It does not make sense to have 'q=0'. 583 if (qvalue10 > kQvalueDecrement10) 584 qvalue10 -= kQvalueDecrement10; 585 } 586 return lang_list_with_q; 587 } 588 589 std::string HttpUtil::GenerateAcceptCharsetHeader(const std::string& charset) { 590 std::string charset_with_q = charset; 591 if (LowerCaseEqualsASCII(charset, "utf-8")) { 592 charset_with_q += ",*;q=0.5"; 593 } else { 594 charset_with_q += ",utf-8;q=0.7,*;q=0.3"; 595 } 596 return charset_with_q; 597 } 598 599 void HttpUtil::AppendHeaderIfMissing(const char* header_name, 600 const std::string& header_value, 601 std::string* headers) { 602 if (header_value.empty()) 603 return; 604 if (net::HttpUtil::HasHeader(*headers, header_name)) 605 return; 606 *headers += std::string(header_name) + ": " + header_value + "\r\n"; 607 } 608 609 // BNF from section 4.2 of RFC 2616: 610 // 611 // message-header = field-name ":" [ field-value ] 612 // field-name = token 613 // field-value = *( field-content | LWS ) 614 // field-content = <the OCTETs making up the field-value 615 // and consisting of either *TEXT or combinations 616 // of token, separators, and quoted-string> 617 // 618 619 HttpUtil::HeadersIterator::HeadersIterator(string::const_iterator headers_begin, 620 string::const_iterator headers_end, 621 const std::string& line_delimiter) 622 : lines_(headers_begin, headers_end, line_delimiter) { 623 } 624 625 bool HttpUtil::HeadersIterator::GetNext() { 626 while (lines_.GetNext()) { 627 name_begin_ = lines_.token_begin(); 628 values_end_ = lines_.token_end(); 629 630 string::const_iterator colon = find(name_begin_, values_end_, ':'); 631 if (colon == values_end_) 632 continue; // skip malformed header 633 634 name_end_ = colon; 635 636 // If the name starts with LWS, it is an invalid line. 637 // Leading LWS implies a line continuation, and these should have 638 // already been joined by AssembleRawHeaders(). 639 if (name_begin_ == name_end_ || IsLWS(*name_begin_)) 640 continue; 641 642 TrimLWS(&name_begin_, &name_end_); 643 if (name_begin_ == name_end_) 644 continue; // skip malformed header 645 646 values_begin_ = colon + 1; 647 TrimLWS(&values_begin_, &values_end_); 648 649 // if we got a header name, then we are done. 650 return true; 651 } 652 return false; 653 } 654 655 bool HttpUtil::HeadersIterator::AdvanceTo(const char* name) { 656 DCHECK(name != NULL); 657 DCHECK_EQ(0, StringToLowerASCII<std::string>(name).compare(name)) 658 << "the header name must be in all lower case"; 659 660 while (GetNext()) { 661 if (LowerCaseEqualsASCII(name_begin_, name_end_, name)) { 662 return true; 663 } 664 } 665 666 return false; 667 } 668 669 HttpUtil::ValuesIterator::ValuesIterator( 670 string::const_iterator values_begin, 671 string::const_iterator values_end, 672 char delimiter) 673 : values_(values_begin, values_end, string(1, delimiter)) { 674 values_.set_quote_chars("\'\""); 675 } 676 677 bool HttpUtil::ValuesIterator::GetNext() { 678 while (values_.GetNext()) { 679 value_begin_ = values_.token_begin(); 680 value_end_ = values_.token_end(); 681 TrimLWS(&value_begin_, &value_end_); 682 683 // bypass empty values. 684 if (value_begin_ != value_end_) 685 return true; 686 } 687 return false; 688 } 689 690 } // namespace net 691