1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/autocomplete/autocomplete_input.h" 6 7 #include "base/strings/string_util.h" 8 #include "base/strings/utf_string_conversions.h" 9 #include "chrome/browser/external_protocol/external_protocol_handler.h" 10 #include "chrome/browser/profiles/profile_io_data.h" 11 #include "components/metrics/proto/omnibox_event.pb.h" 12 #include "components/url_fixer/url_fixer.h" 13 #include "content/public/common/url_constants.h" 14 #include "net/base/net_util.h" 15 #include "net/base/registry_controlled_domains/registry_controlled_domain.h" 16 #include "url/url_canon_ip.h" 17 #include "url/url_util.h" 18 19 namespace { 20 21 void AdjustCursorPositionIfNecessary(size_t num_leading_chars_removed, 22 size_t* cursor_position) { 23 if (*cursor_position == base::string16::npos) 24 return; 25 if (num_leading_chars_removed < *cursor_position) 26 *cursor_position -= num_leading_chars_removed; 27 else 28 *cursor_position = 0; 29 } 30 31 } // namespace 32 33 AutocompleteInput::AutocompleteInput() 34 : cursor_position_(base::string16::npos), 35 current_page_classification_(metrics::OmniboxEventProto::INVALID_SPEC), 36 type_(metrics::OmniboxInputType::INVALID), 37 prevent_inline_autocomplete_(false), 38 prefer_keyword_(false), 39 allow_exact_keyword_match_(true), 40 want_asynchronous_matches_(true) { 41 } 42 43 AutocompleteInput::AutocompleteInput( 44 const base::string16& text, 45 size_t cursor_position, 46 const base::string16& desired_tld, 47 const GURL& current_url, 48 metrics::OmniboxEventProto::PageClassification current_page_classification, 49 bool prevent_inline_autocomplete, 50 bool prefer_keyword, 51 bool allow_exact_keyword_match, 52 bool want_asynchronous_matches) 53 : cursor_position_(cursor_position), 54 current_url_(current_url), 55 current_page_classification_(current_page_classification), 56 prevent_inline_autocomplete_(prevent_inline_autocomplete), 57 prefer_keyword_(prefer_keyword), 58 allow_exact_keyword_match_(allow_exact_keyword_match), 59 want_asynchronous_matches_(want_asynchronous_matches) { 60 DCHECK(cursor_position <= text.length() || 61 cursor_position == base::string16::npos) 62 << "Text: '" << text << "', cp: " << cursor_position; 63 // None of the providers care about leading white space so we always trim it. 64 // Providers that care about trailing white space handle trimming themselves. 65 if ((base::TrimWhitespace(text, base::TRIM_LEADING, &text_) & 66 base::TRIM_LEADING) != 0) 67 AdjustCursorPositionIfNecessary(text.length() - text_.length(), 68 &cursor_position_); 69 70 GURL canonicalized_url; 71 type_ = Parse(text_, desired_tld, &parts_, &scheme_, &canonicalized_url); 72 73 if (type_ == metrics::OmniboxInputType::INVALID) 74 return; 75 76 if (((type_ == metrics::OmniboxInputType::UNKNOWN) || 77 (type_ == metrics::OmniboxInputType::URL)) && 78 canonicalized_url.is_valid() && 79 (!canonicalized_url.IsStandard() || canonicalized_url.SchemeIsFile() || 80 canonicalized_url.SchemeIsFileSystem() || 81 !canonicalized_url.host().empty())) 82 canonicalized_url_ = canonicalized_url; 83 84 size_t chars_removed = RemoveForcedQueryStringIfNecessary(type_, &text_); 85 AdjustCursorPositionIfNecessary(chars_removed, &cursor_position_); 86 if (chars_removed) { 87 // Remove spaces between opening question mark and first actual character. 88 base::string16 trimmed_text; 89 if ((base::TrimWhitespace(text_, base::TRIM_LEADING, &trimmed_text) & 90 base::TRIM_LEADING) != 0) { 91 AdjustCursorPositionIfNecessary(text_.length() - trimmed_text.length(), 92 &cursor_position_); 93 text_ = trimmed_text; 94 } 95 } 96 } 97 98 AutocompleteInput::~AutocompleteInput() { 99 } 100 101 // static 102 size_t AutocompleteInput::RemoveForcedQueryStringIfNecessary( 103 metrics::OmniboxInputType::Type type, 104 base::string16* text) { 105 if ((type != metrics::OmniboxInputType::FORCED_QUERY) || text->empty() || 106 (*text)[0] != L'?') 107 return 0; 108 // Drop the leading '?'. 109 text->erase(0, 1); 110 return 1; 111 } 112 113 // static 114 std::string AutocompleteInput::TypeToString( 115 metrics::OmniboxInputType::Type type) { 116 switch (type) { 117 case metrics::OmniboxInputType::INVALID: return "invalid"; 118 case metrics::OmniboxInputType::UNKNOWN: return "unknown"; 119 case metrics::OmniboxInputType::DEPRECATED_REQUESTED_URL: 120 return "deprecated-requested-url"; 121 case metrics::OmniboxInputType::URL: return "url"; 122 case metrics::OmniboxInputType::QUERY: return "query"; 123 case metrics::OmniboxInputType::FORCED_QUERY: return "forced-query"; 124 } 125 return std::string(); 126 } 127 128 // static 129 metrics::OmniboxInputType::Type AutocompleteInput::Parse( 130 const base::string16& text, 131 const base::string16& desired_tld, 132 url::Parsed* parts, 133 base::string16* scheme, 134 GURL* canonicalized_url) { 135 size_t first_non_white = text.find_first_not_of(base::kWhitespaceUTF16, 0); 136 if (first_non_white == base::string16::npos) 137 return metrics::OmniboxInputType::INVALID; // All whitespace. 138 139 if (text[first_non_white] == L'?') { 140 // If the first non-whitespace character is a '?', we magically treat this 141 // as a query. 142 return metrics::OmniboxInputType::FORCED_QUERY; 143 } 144 145 // Ask our parsing back-end to help us understand what the user typed. We 146 // use the URLFixerUpper here because we want to be smart about what we 147 // consider a scheme. For example, we shouldn't consider www.google.com:80 148 // to have a scheme. 149 url::Parsed local_parts; 150 if (!parts) 151 parts = &local_parts; 152 const base::string16 parsed_scheme(url_fixer::SegmentURL(text, parts)); 153 if (scheme) 154 *scheme = parsed_scheme; 155 156 // If we can't canonicalize the user's input, the rest of the autocomplete 157 // system isn't going to be able to produce a navigable URL match for it. 158 // So we just return QUERY immediately in these cases. 159 GURL placeholder_canonicalized_url; 160 if (!canonicalized_url) 161 canonicalized_url = &placeholder_canonicalized_url; 162 *canonicalized_url = url_fixer::FixupURL(base::UTF16ToUTF8(text), 163 base::UTF16ToUTF8(desired_tld)); 164 if (!canonicalized_url->is_valid()) 165 return metrics::OmniboxInputType::QUERY; 166 167 if (LowerCaseEqualsASCII(parsed_scheme, url::kFileScheme)) { 168 // A user might or might not type a scheme when entering a file URL. In 169 // either case, |parsed_scheme| will tell us that this is a file URL, but 170 // |parts->scheme| might be empty, e.g. if the user typed "C:\foo". 171 return metrics::OmniboxInputType::URL; 172 } 173 174 // If the user typed a scheme, and it's HTTP or HTTPS, we know how to parse it 175 // well enough that we can fall through to the heuristics below. If it's 176 // something else, we can just determine our action based on what we do with 177 // any input of this scheme. In theory we could do better with some schemes 178 // (e.g. "ftp" or "view-source") but I'll wait to spend the effort on that 179 // until I run into some cases that really need it. 180 if (parts->scheme.is_nonempty() && 181 !LowerCaseEqualsASCII(parsed_scheme, url::kHttpScheme) && 182 !LowerCaseEqualsASCII(parsed_scheme, url::kHttpsScheme)) { 183 // See if we know how to handle the URL internally. There are some schemes 184 // that we convert to other things before they reach the renderer or else 185 // the renderer handles internally without reaching the net::URLRequest 186 // logic. They thus won't be listed as "handled protocols", but we should 187 // still claim to handle them. 188 if (ProfileIOData::IsHandledProtocol(base::UTF16ToASCII(parsed_scheme)) || 189 LowerCaseEqualsASCII(parsed_scheme, content::kViewSourceScheme) || 190 LowerCaseEqualsASCII(parsed_scheme, url::kJavaScriptScheme) || 191 LowerCaseEqualsASCII(parsed_scheme, url::kDataScheme)) 192 return metrics::OmniboxInputType::URL; 193 194 // Not an internal protocol. Check and see if the user has explicitly 195 // opened this scheme as a URL before, or if the "scheme" is actually a 196 // username. We need to do this after the check above because some 197 // handlable schemes (e.g. "javascript") may be treated as "blocked" by the 198 // external protocol handler because we don't want pages to open them, but 199 // users still can. 200 ExternalProtocolHandler::BlockState block_state = 201 ExternalProtocolHandler::GetBlockState( 202 base::UTF16ToUTF8(parsed_scheme)); 203 switch (block_state) { 204 case ExternalProtocolHandler::DONT_BLOCK: 205 return metrics::OmniboxInputType::URL; 206 207 case ExternalProtocolHandler::BLOCK: 208 // If we don't want the user to open the URL, don't let it be navigated 209 // to at all. 210 return metrics::OmniboxInputType::QUERY; 211 212 default: { 213 // We don't know about this scheme. It might be that the user typed a 214 // URL of the form "username:password (at) foo.com". 215 const base::string16 http_scheme_prefix = 216 base::ASCIIToUTF16(std::string(url::kHttpScheme) + 217 url::kStandardSchemeSeparator); 218 url::Parsed http_parts; 219 base::string16 http_scheme; 220 GURL http_canonicalized_url; 221 metrics::OmniboxInputType::Type http_type = 222 Parse(http_scheme_prefix + text, desired_tld, &http_parts, 223 &http_scheme, &http_canonicalized_url); 224 DCHECK_EQ(std::string(url::kHttpScheme), 225 base::UTF16ToUTF8(http_scheme)); 226 227 if ((http_type == metrics::OmniboxInputType::URL) && 228 http_parts.username.is_nonempty() && 229 http_parts.password.is_nonempty()) { 230 // Manually re-jigger the parsed parts to match |text| (without the 231 // http scheme added). 232 http_parts.scheme.reset(); 233 url::Component* components[] = { 234 &http_parts.username, 235 &http_parts.password, 236 &http_parts.host, 237 &http_parts.port, 238 &http_parts.path, 239 &http_parts.query, 240 &http_parts.ref, 241 }; 242 for (size_t i = 0; i < arraysize(components); ++i) { 243 url_fixer::OffsetComponent( 244 -static_cast<int>(http_scheme_prefix.length()), components[i]); 245 } 246 247 *parts = http_parts; 248 if (scheme) 249 scheme->clear(); 250 *canonicalized_url = http_canonicalized_url; 251 252 return metrics::OmniboxInputType::URL; 253 } 254 255 // We don't know about this scheme and it doesn't look like the user 256 // typed a username and password. It's likely to be a search operator 257 // like "site:" or "link:". We classify it as UNKNOWN so the user has 258 // the option of treating it as a URL if we're wrong. 259 // Note that SegmentURL() is smart so we aren't tricked by "c:\foo" or 260 // "www.example.com:81" in this case. 261 return metrics::OmniboxInputType::UNKNOWN; 262 } 263 } 264 } 265 266 // Either the user didn't type a scheme, in which case we need to distinguish 267 // between an HTTP URL and a query, or the scheme is HTTP or HTTPS, in which 268 // case we should reject invalid formulations. 269 270 // If we have an empty host it can't be a valid HTTP[S] URL. (This should 271 // only trigger for input that begins with a colon, which GURL will parse as a 272 // valid, non-standard URL; for standard URLs, an empty host would have 273 // resulted in an invalid |canonicalized_url| above.) 274 if (!parts->host.is_nonempty()) 275 return metrics::OmniboxInputType::QUERY; 276 277 // Sanity-check: GURL should have failed to canonicalize this URL if it had an 278 // invalid port. 279 DCHECK_NE(url::PORT_INVALID, url::ParsePort(text.c_str(), parts->port)); 280 281 // Likewise, the RCDS can reject certain obviously-invalid hosts. (We also 282 // use the registry length later below.) 283 const base::string16 host(text.substr(parts->host.begin, parts->host.len)); 284 const size_t registry_length = 285 net::registry_controlled_domains::GetRegistryLength( 286 base::UTF16ToUTF8(host), 287 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES, 288 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES); 289 if (registry_length == std::string::npos) { 290 // Try to append the desired_tld. 291 if (!desired_tld.empty()) { 292 base::string16 host_with_tld(host); 293 if (host[host.length() - 1] != '.') 294 host_with_tld += '.'; 295 host_with_tld += desired_tld; 296 const size_t tld_length = 297 net::registry_controlled_domains::GetRegistryLength( 298 base::UTF16ToUTF8(host_with_tld), 299 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES, 300 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES); 301 if (tld_length != std::string::npos) { 302 // Something like "99999999999" that looks like a bad IP 303 // address, but becomes valid on attaching a TLD. 304 return metrics::OmniboxInputType::URL; 305 } 306 } 307 // Could be a broken IP address, etc. 308 return metrics::OmniboxInputType::QUERY; 309 } 310 311 312 // See if the hostname is valid. While IE and GURL allow hostnames to contain 313 // many other characters (perhaps for weird intranet machines), it's extremely 314 // unlikely that a user would be trying to type those in for anything other 315 // than a search query. 316 url::CanonHostInfo host_info; 317 const std::string canonicalized_host(net::CanonicalizeHost( 318 base::UTF16ToUTF8(host), &host_info)); 319 if ((host_info.family == url::CanonHostInfo::NEUTRAL) && 320 !net::IsCanonicalizedHostCompliant(canonicalized_host, 321 base::UTF16ToUTF8(desired_tld))) { 322 // Invalid hostname. There are several possible cases: 323 // * Our checker is too strict and the user pasted in a real-world URL 324 // that's "invalid" but resolves. To catch these, we return UNKNOWN when 325 // the user explicitly typed a scheme, so we'll still search by default 326 // but we'll show the accidental search infobar if necessary. 327 // * The user is typing a multi-word query. If we see a space anywhere in 328 // the hostname we assume this is a search and return QUERY. 329 // * Our checker is too strict and the user is typing a real-world hostname 330 // that's "invalid" but resolves. We return UNKNOWN if the TLD is known. 331 // Note that we explicitly excluded hosts with spaces above so that 332 // "toys at amazon.com" will be treated as a search. 333 // * The user is typing some garbage string. Return QUERY. 334 // 335 // Thus we fall down in the following cases: 336 // * Trying to navigate to a hostname with spaces 337 // * Trying to navigate to a hostname with invalid characters and an unknown 338 // TLD 339 // These are rare, though probably possible in intranets. 340 return (parts->scheme.is_nonempty() || 341 ((registry_length != 0) && 342 (host.find(' ') == base::string16::npos))) ? 343 metrics::OmniboxInputType::UNKNOWN : metrics::OmniboxInputType::QUERY; 344 } 345 346 // Now that we've ruled out all schemes other than http or https and done a 347 // little more sanity checking, the presence of a scheme means this is likely 348 // a URL. 349 if (parts->scheme.is_nonempty()) 350 return metrics::OmniboxInputType::URL; 351 352 // See if the host is an IP address. 353 if (host_info.family == url::CanonHostInfo::IPV6) 354 return metrics::OmniboxInputType::URL; 355 // If the user originally typed a host that looks like an IP address (a 356 // dotted quad), they probably want to open it. If the original input was 357 // something else (like a single number), they probably wanted to search for 358 // it, unless they explicitly typed a scheme. This is true even if the URL 359 // appears to have a path: "1.2/45" is more likely a search (for the answer 360 // to a math problem) than a URL. However, if there are more non-host 361 // components, then maybe this really was intended to be a navigation. For 362 // this reason we only check the dotted-quad case here, and save the "other 363 // IP addresses" case for after we check the number of non-host components 364 // below. 365 if ((host_info.family == url::CanonHostInfo::IPV4) && 366 (host_info.num_ipv4_components == 4)) 367 return metrics::OmniboxInputType::URL; 368 369 // Presence of a password means this is likely a URL. Note that unless the 370 // user has typed an explicit "http://" or similar, we'll probably think that 371 // the username is some unknown scheme, and bail out in the scheme-handling 372 // code above. 373 if (parts->password.is_nonempty()) 374 return metrics::OmniboxInputType::URL; 375 376 // Trailing slashes force the input to be treated as a URL. 377 if (parts->path.is_nonempty()) { 378 char c = text[parts->path.end() - 1]; 379 if ((c == '\\') || (c == '/')) 380 return metrics::OmniboxInputType::URL; 381 } 382 383 // If there is more than one recognized non-host component, this is likely to 384 // be a URL, even if the TLD is unknown (in which case this is likely an 385 // intranet URL). 386 if (NumNonHostComponents(*parts) > 1) 387 return metrics::OmniboxInputType::URL; 388 389 // If the host has a known TLD or a port, it's probably a URL, with the 390 // following exceptions: 391 // * Any "IP addresses" that make it here are more likely searches 392 // (see above). 393 // * If we reach here with a username, our input looks like "user@host[.tld]". 394 // Because there is no scheme explicitly specified, we think this is more 395 // likely an email address than an HTTP auth attempt. Hence, we search by 396 // default and let users correct us on a case-by-case basis. 397 // Note that we special-case "localhost" as a known hostname. 398 if ((host_info.family != url::CanonHostInfo::IPV4) && 399 ((registry_length != 0) || (host == base::ASCIIToUTF16("localhost") || 400 parts->port.is_nonempty()))) { 401 return parts->username.is_nonempty() ? metrics::OmniboxInputType::UNKNOWN : 402 metrics::OmniboxInputType::URL; 403 } 404 405 // If we reach this point, we know there's no known TLD on the input, so if 406 // the user wishes to add a desired_tld, the fixup code will oblige; thus this 407 // is a URL. 408 if (!desired_tld.empty()) 409 return metrics::OmniboxInputType::URL; 410 411 // No scheme, password, port, path, and no known TLD on the host. 412 // This could be: 413 // * An "incomplete IP address"; likely a search (see above). 414 // * An email-like input like "user@host", where "host" has no known TLD. 415 // It's not clear what the user means here and searching seems reasonable. 416 // * A single word "foo"; possibly an intranet site, but more likely a search. 417 // This is ideally an UNKNOWN, and we can let the Alternate Nav URL code 418 // catch our mistakes. 419 // * A URL with a valid TLD we don't know about yet. If e.g. a registrar adds 420 // "xxx" as a TLD, then until we add it to our data file, Chrome won't know 421 // "foo.xxx" is a real URL. So ideally this is a URL, but we can't really 422 // distinguish this case from: 423 // * A "URL-like" string that's not really a URL (like 424 // "browser.tabs.closeButtons" or "java.awt.event.*"). This is ideally a 425 // QUERY. Since this is indistinguishable from the case above, and this 426 // case is much more likely, claim these are UNKNOWN, which should default 427 // to the right thing and let users correct us on a case-by-case basis. 428 return metrics::OmniboxInputType::UNKNOWN; 429 } 430 431 // static 432 void AutocompleteInput::ParseForEmphasizeComponents(const base::string16& text, 433 url::Component* scheme, 434 url::Component* host) { 435 url::Parsed parts; 436 base::string16 scheme_str; 437 Parse(text, base::string16(), &parts, &scheme_str, NULL); 438 439 *scheme = parts.scheme; 440 *host = parts.host; 441 442 int after_scheme_and_colon = parts.scheme.end() + 1; 443 // For the view-source scheme, we should emphasize the scheme and host of the 444 // URL qualified by the view-source prefix. 445 if (LowerCaseEqualsASCII(scheme_str, content::kViewSourceScheme) && 446 (static_cast<int>(text.length()) > after_scheme_and_colon)) { 447 // Obtain the URL prefixed by view-source and parse it. 448 base::string16 real_url(text.substr(after_scheme_and_colon)); 449 url::Parsed real_parts; 450 AutocompleteInput::Parse(real_url, base::string16(), &real_parts, NULL, NULL); 451 if (real_parts.scheme.is_nonempty() || real_parts.host.is_nonempty()) { 452 if (real_parts.scheme.is_nonempty()) { 453 *scheme = url::Component( 454 after_scheme_and_colon + real_parts.scheme.begin, 455 real_parts.scheme.len); 456 } else { 457 scheme->reset(); 458 } 459 if (real_parts.host.is_nonempty()) { 460 *host = url::Component(after_scheme_and_colon + real_parts.host.begin, 461 real_parts.host.len); 462 } else { 463 host->reset(); 464 } 465 } 466 } else if (LowerCaseEqualsASCII(scheme_str, url::kFileSystemScheme) && 467 parts.inner_parsed() && parts.inner_parsed()->scheme.is_valid()) { 468 *host = parts.inner_parsed()->host; 469 } 470 } 471 472 // static 473 base::string16 AutocompleteInput::FormattedStringWithEquivalentMeaning( 474 const GURL& url, 475 const base::string16& formatted_url) { 476 if (!net::CanStripTrailingSlash(url)) 477 return formatted_url; 478 const base::string16 url_with_path(formatted_url + base::char16('/')); 479 return (AutocompleteInput::Parse(formatted_url, base::string16(), NULL, NULL, 480 NULL) == 481 AutocompleteInput::Parse(url_with_path, base::string16(), NULL, NULL, 482 NULL)) ? 483 formatted_url : url_with_path; 484 } 485 486 // static 487 int AutocompleteInput::NumNonHostComponents(const url::Parsed& parts) { 488 int num_nonhost_components = 0; 489 if (parts.scheme.is_nonempty()) 490 ++num_nonhost_components; 491 if (parts.username.is_nonempty()) 492 ++num_nonhost_components; 493 if (parts.password.is_nonempty()) 494 ++num_nonhost_components; 495 if (parts.port.is_nonempty()) 496 ++num_nonhost_components; 497 if (parts.path.is_nonempty()) 498 ++num_nonhost_components; 499 if (parts.query.is_nonempty()) 500 ++num_nonhost_components; 501 if (parts.ref.is_nonempty()) 502 ++num_nonhost_components; 503 return num_nonhost_components; 504 } 505 506 // static 507 bool AutocompleteInput::HasHTTPScheme(const base::string16& input) { 508 std::string utf8_input(base::UTF16ToUTF8(input)); 509 url::Component scheme; 510 if (url::FindAndCompareScheme(utf8_input, content::kViewSourceScheme, 511 &scheme)) { 512 utf8_input.erase(0, scheme.end() + 1); 513 } 514 return url::FindAndCompareScheme(utf8_input, url::kHttpScheme, NULL); 515 } 516 517 void AutocompleteInput::UpdateText(const base::string16& text, 518 size_t cursor_position, 519 const url::Parsed& parts) { 520 DCHECK(cursor_position <= text.length() || 521 cursor_position == base::string16::npos) 522 << "Text: '" << text << "', cp: " << cursor_position; 523 text_ = text; 524 cursor_position_ = cursor_position; 525 parts_ = parts; 526 } 527 528 void AutocompleteInput::Clear() { 529 text_.clear(); 530 cursor_position_ = base::string16::npos; 531 current_url_ = GURL(); 532 current_page_classification_ = metrics::OmniboxEventProto::INVALID_SPEC; 533 type_ = metrics::OmniboxInputType::INVALID; 534 parts_ = url::Parsed(); 535 scheme_.clear(); 536 canonicalized_url_ = GURL(); 537 prevent_inline_autocomplete_ = false; 538 prefer_keyword_ = false; 539 allow_exact_keyword_match_ = false; 540 want_asynchronous_matches_ = true; 541 } 542
