1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "components/omnibox/autocomplete_input.h" 6 7 #include "base/strings/string_util.h" 8 #include "base/strings/utf_string_conversions.h" 9 #include "components/metrics/proto/omnibox_event.pb.h" 10 #include "components/omnibox/autocomplete_scheme_classifier.h" 11 #include "components/url_fixer/url_fixer.h" 12 #include "net/base/net_util.h" 13 #include "net/base/registry_controlled_domains/registry_controlled_domain.h" 14 #include "url/url_canon_ip.h" 15 #include "url/url_util.h" 16 17 namespace { 18 19 // Hardcode constant to avoid any dependencies on content/. 20 const char kViewSourceScheme[] = "view-source"; 21 22 void AdjustCursorPositionIfNecessary(size_t num_leading_chars_removed, 23 size_t* cursor_position) { 24 if (*cursor_position == base::string16::npos) 25 return; 26 if (num_leading_chars_removed < *cursor_position) 27 *cursor_position -= num_leading_chars_removed; 28 else 29 *cursor_position = 0; 30 } 31 32 } // namespace 33 34 AutocompleteInput::AutocompleteInput() 35 : cursor_position_(base::string16::npos), 36 current_page_classification_(metrics::OmniboxEventProto::INVALID_SPEC), 37 type_(metrics::OmniboxInputType::INVALID), 38 prevent_inline_autocomplete_(false), 39 prefer_keyword_(false), 40 allow_exact_keyword_match_(true), 41 want_asynchronous_matches_(true) { 42 } 43 44 AutocompleteInput::AutocompleteInput( 45 const base::string16& text, 46 size_t cursor_position, 47 const base::string16& desired_tld, 48 const GURL& current_url, 49 metrics::OmniboxEventProto::PageClassification current_page_classification, 50 bool prevent_inline_autocomplete, 51 bool prefer_keyword, 52 bool allow_exact_keyword_match, 53 bool want_asynchronous_matches, 54 const AutocompleteSchemeClassifier& scheme_classifier) 55 : cursor_position_(cursor_position), 56 current_url_(current_url), 57 current_page_classification_(current_page_classification), 58 prevent_inline_autocomplete_(prevent_inline_autocomplete), 59 prefer_keyword_(prefer_keyword), 60 allow_exact_keyword_match_(allow_exact_keyword_match), 61 want_asynchronous_matches_(want_asynchronous_matches) { 62 DCHECK(cursor_position <= text.length() || 63 cursor_position == base::string16::npos) 64 << "Text: '" << text << "', cp: " << cursor_position; 65 // None of the providers care about leading white space so we always trim it. 66 // Providers that care about trailing white space handle trimming themselves. 67 if ((base::TrimWhitespace(text, base::TRIM_LEADING, &text_) & 68 base::TRIM_LEADING) != 0) 69 AdjustCursorPositionIfNecessary(text.length() - text_.length(), 70 &cursor_position_); 71 72 GURL canonicalized_url; 73 type_ = Parse(text_, desired_tld, scheme_classifier, &parts_, &scheme_, 74 &canonicalized_url); 75 76 if (type_ == metrics::OmniboxInputType::INVALID) 77 return; 78 79 if (((type_ == metrics::OmniboxInputType::UNKNOWN) || 80 (type_ == metrics::OmniboxInputType::URL)) && 81 canonicalized_url.is_valid() && 82 (!canonicalized_url.IsStandard() || canonicalized_url.SchemeIsFile() || 83 canonicalized_url.SchemeIsFileSystem() || 84 !canonicalized_url.host().empty())) 85 canonicalized_url_ = canonicalized_url; 86 87 size_t chars_removed = RemoveForcedQueryStringIfNecessary(type_, &text_); 88 AdjustCursorPositionIfNecessary(chars_removed, &cursor_position_); 89 if (chars_removed) { 90 // Remove spaces between opening question mark and first actual character. 91 base::string16 trimmed_text; 92 if ((base::TrimWhitespace(text_, base::TRIM_LEADING, &trimmed_text) & 93 base::TRIM_LEADING) != 0) { 94 AdjustCursorPositionIfNecessary(text_.length() - trimmed_text.length(), 95 &cursor_position_); 96 text_ = trimmed_text; 97 } 98 } 99 } 100 101 AutocompleteInput::~AutocompleteInput() { 102 } 103 104 // static 105 size_t AutocompleteInput::RemoveForcedQueryStringIfNecessary( 106 metrics::OmniboxInputType::Type type, 107 base::string16* text) { 108 if ((type != metrics::OmniboxInputType::FORCED_QUERY) || text->empty() || 109 (*text)[0] != L'?') 110 return 0; 111 // Drop the leading '?'. 112 text->erase(0, 1); 113 return 1; 114 } 115 116 // static 117 std::string AutocompleteInput::TypeToString( 118 metrics::OmniboxInputType::Type type) { 119 switch (type) { 120 case metrics::OmniboxInputType::INVALID: return "invalid"; 121 case metrics::OmniboxInputType::UNKNOWN: return "unknown"; 122 case metrics::OmniboxInputType::DEPRECATED_REQUESTED_URL: 123 return "deprecated-requested-url"; 124 case metrics::OmniboxInputType::URL: return "url"; 125 case metrics::OmniboxInputType::QUERY: return "query"; 126 case metrics::OmniboxInputType::FORCED_QUERY: return "forced-query"; 127 } 128 return std::string(); 129 } 130 131 // static 132 metrics::OmniboxInputType::Type AutocompleteInput::Parse( 133 const base::string16& text, 134 const base::string16& desired_tld, 135 const AutocompleteSchemeClassifier& scheme_classifier, 136 url::Parsed* parts, 137 base::string16* scheme, 138 GURL* canonicalized_url) { 139 size_t first_non_white = text.find_first_not_of(base::kWhitespaceUTF16, 0); 140 if (first_non_white == base::string16::npos) 141 return metrics::OmniboxInputType::INVALID; // All whitespace. 142 143 if (text[first_non_white] == L'?') { 144 // If the first non-whitespace character is a '?', we magically treat this 145 // as a query. 146 return metrics::OmniboxInputType::FORCED_QUERY; 147 } 148 149 // Ask our parsing back-end to help us understand what the user typed. We 150 // use the URLFixerUpper here because we want to be smart about what we 151 // consider a scheme. For example, we shouldn't consider www.google.com:80 152 // to have a scheme. 153 url::Parsed local_parts; 154 if (!parts) 155 parts = &local_parts; 156 const base::string16 parsed_scheme(url_fixer::SegmentURL(text, parts)); 157 if (scheme) 158 *scheme = parsed_scheme; 159 const std::string parsed_scheme_utf8(base::UTF16ToUTF8(parsed_scheme)); 160 161 // If we can't canonicalize the user's input, the rest of the autocomplete 162 // system isn't going to be able to produce a navigable URL match for it. 163 // So we just return QUERY immediately in these cases. 164 GURL placeholder_canonicalized_url; 165 if (!canonicalized_url) 166 canonicalized_url = &placeholder_canonicalized_url; 167 *canonicalized_url = url_fixer::FixupURL(base::UTF16ToUTF8(text), 168 base::UTF16ToUTF8(desired_tld)); 169 if (!canonicalized_url->is_valid()) 170 return metrics::OmniboxInputType::QUERY; 171 172 if (LowerCaseEqualsASCII(parsed_scheme_utf8, url::kFileScheme)) { 173 // A user might or might not type a scheme when entering a file URL. In 174 // either case, |parsed_scheme_utf8| will tell us that this is a file URL, 175 // but |parts->scheme| might be empty, e.g. if the user typed "C:\foo". 176 return metrics::OmniboxInputType::URL; 177 } 178 179 // If the user typed a scheme, and it's HTTP or HTTPS, we know how to parse it 180 // well enough that we can fall through to the heuristics below. If it's 181 // something else, we can just determine our action based on what we do with 182 // any input of this scheme. In theory we could do better with some schemes 183 // (e.g. "ftp" or "view-source") but I'll wait to spend the effort on that 184 // until I run into some cases that really need it. 185 if (parts->scheme.is_nonempty() && 186 !LowerCaseEqualsASCII(parsed_scheme_utf8, url::kHttpScheme) && 187 !LowerCaseEqualsASCII(parsed_scheme_utf8, url::kHttpsScheme)) { 188 metrics::OmniboxInputType::Type type = 189 scheme_classifier.GetInputTypeForScheme(parsed_scheme_utf8); 190 if (type != metrics::OmniboxInputType::INVALID) 191 return type; 192 193 // We don't know about this scheme. It might be that the user typed a 194 // URL of the form "username:password (at) foo.com". 195 const base::string16 http_scheme_prefix = 196 base::ASCIIToUTF16(std::string(url::kHttpScheme) + 197 url::kStandardSchemeSeparator); 198 url::Parsed http_parts; 199 base::string16 http_scheme; 200 GURL http_canonicalized_url; 201 metrics::OmniboxInputType::Type http_type = 202 Parse(http_scheme_prefix + text, desired_tld, scheme_classifier, 203 &http_parts, &http_scheme, &http_canonicalized_url); 204 DCHECK_EQ(std::string(url::kHttpScheme), 205 base::UTF16ToUTF8(http_scheme)); 206 207 if ((http_type == metrics::OmniboxInputType::URL) && 208 http_parts.username.is_nonempty() && 209 http_parts.password.is_nonempty()) { 210 // Manually re-jigger the parsed parts to match |text| (without the 211 // http scheme added). 212 http_parts.scheme.reset(); 213 url::Component* components[] = { 214 &http_parts.username, 215 &http_parts.password, 216 &http_parts.host, 217 &http_parts.port, 218 &http_parts.path, 219 &http_parts.query, 220 &http_parts.ref, 221 }; 222 for (size_t i = 0; i < arraysize(components); ++i) { 223 url_fixer::OffsetComponent( 224 -static_cast<int>(http_scheme_prefix.length()), components[i]); 225 } 226 227 *parts = http_parts; 228 if (scheme) 229 scheme->clear(); 230 *canonicalized_url = http_canonicalized_url; 231 232 return metrics::OmniboxInputType::URL; 233 } 234 235 // We don't know about this scheme and it doesn't look like the user 236 // typed a username and password. It's likely to be a search operator 237 // like "site:" or "link:". We classify it as UNKNOWN so the user has 238 // the option of treating it as a URL if we're wrong. 239 // Note that SegmentURL() is smart so we aren't tricked by "c:\foo" or 240 // "www.example.com:81" in this case. 241 return metrics::OmniboxInputType::UNKNOWN; 242 } 243 244 // Either the user didn't type a scheme, in which case we need to distinguish 245 // between an HTTP URL and a query, or the scheme is HTTP or HTTPS, in which 246 // case we should reject invalid formulations. 247 248 // If we have an empty host it can't be a valid HTTP[S] URL. (This should 249 // only trigger for input that begins with a colon, which GURL will parse as a 250 // valid, non-standard URL; for standard URLs, an empty host would have 251 // resulted in an invalid |canonicalized_url| above.) 252 if (!parts->host.is_nonempty()) 253 return metrics::OmniboxInputType::QUERY; 254 255 // Sanity-check: GURL should have failed to canonicalize this URL if it had an 256 // invalid port. 257 DCHECK_NE(url::PORT_INVALID, url::ParsePort(text.c_str(), parts->port)); 258 259 // Likewise, the RCDS can reject certain obviously-invalid hosts. (We also 260 // use the registry length later below.) 261 const base::string16 host(text.substr(parts->host.begin, parts->host.len)); 262 const size_t registry_length = 263 net::registry_controlled_domains::GetRegistryLength( 264 base::UTF16ToUTF8(host), 265 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES, 266 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES); 267 if (registry_length == std::string::npos) { 268 // Try to append the desired_tld. 269 if (!desired_tld.empty()) { 270 base::string16 host_with_tld(host); 271 if (host[host.length() - 1] != '.') 272 host_with_tld += '.'; 273 host_with_tld += desired_tld; 274 const size_t tld_length = 275 net::registry_controlled_domains::GetRegistryLength( 276 base::UTF16ToUTF8(host_with_tld), 277 net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES, 278 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES); 279 if (tld_length != std::string::npos) { 280 // Something like "99999999999" that looks like a bad IP 281 // address, but becomes valid on attaching a TLD. 282 return metrics::OmniboxInputType::URL; 283 } 284 } 285 // Could be a broken IP address, etc. 286 return metrics::OmniboxInputType::QUERY; 287 } 288 289 290 // See if the hostname is valid. While IE and GURL allow hostnames to contain 291 // many other characters (perhaps for weird intranet machines), it's extremely 292 // unlikely that a user would be trying to type those in for anything other 293 // than a search query. 294 url::CanonHostInfo host_info; 295 const std::string canonicalized_host(net::CanonicalizeHost( 296 base::UTF16ToUTF8(host), &host_info)); 297 if ((host_info.family == url::CanonHostInfo::NEUTRAL) && 298 !net::IsCanonicalizedHostCompliant(canonicalized_host, 299 base::UTF16ToUTF8(desired_tld))) { 300 // Invalid hostname. There are several possible cases: 301 // * Our checker is too strict and the user pasted in a real-world URL 302 // that's "invalid" but resolves. To catch these, we return UNKNOWN when 303 // the user explicitly typed a scheme, so we'll still search by default 304 // but we'll show the accidental search infobar if necessary. 305 // * The user is typing a multi-word query. If we see a space anywhere in 306 // the hostname we assume this is a search and return QUERY. 307 // * Our checker is too strict and the user is typing a real-world hostname 308 // that's "invalid" but resolves. We return UNKNOWN if the TLD is known. 309 // Note that we explicitly excluded hosts with spaces above so that 310 // "toys at amazon.com" will be treated as a search. 311 // * The user is typing some garbage string. Return QUERY. 312 // 313 // Thus we fall down in the following cases: 314 // * Trying to navigate to a hostname with spaces 315 // * Trying to navigate to a hostname with invalid characters and an unknown 316 // TLD 317 // These are rare, though probably possible in intranets. 318 return (parts->scheme.is_nonempty() || 319 ((registry_length != 0) && 320 (host.find(' ') == base::string16::npos))) ? 321 metrics::OmniboxInputType::UNKNOWN : metrics::OmniboxInputType::QUERY; 322 } 323 324 // Now that we've ruled out all schemes other than http or https and done a 325 // little more sanity checking, the presence of a scheme means this is likely 326 // a URL. 327 if (parts->scheme.is_nonempty()) 328 return metrics::OmniboxInputType::URL; 329 330 // See if the host is an IP address. 331 if (host_info.family == url::CanonHostInfo::IPV6) 332 return metrics::OmniboxInputType::URL; 333 // If the user originally typed a host that looks like an IP address (a 334 // dotted quad), they probably want to open it. If the original input was 335 // something else (like a single number), they probably wanted to search for 336 // it, unless they explicitly typed a scheme. This is true even if the URL 337 // appears to have a path: "1.2/45" is more likely a search (for the answer 338 // to a math problem) than a URL. However, if there are more non-host 339 // components, then maybe this really was intended to be a navigation. For 340 // this reason we only check the dotted-quad case here, and save the "other 341 // IP addresses" case for after we check the number of non-host components 342 // below. 343 if ((host_info.family == url::CanonHostInfo::IPV4) && 344 (host_info.num_ipv4_components == 4)) 345 return metrics::OmniboxInputType::URL; 346 347 // Presence of a password means this is likely a URL. Note that unless the 348 // user has typed an explicit "http://" or similar, we'll probably think that 349 // the username is some unknown scheme, and bail out in the scheme-handling 350 // code above. 351 if (parts->password.is_nonempty()) 352 return metrics::OmniboxInputType::URL; 353 354 // Trailing slashes force the input to be treated as a URL. 355 if (parts->path.is_nonempty()) { 356 char c = text[parts->path.end() - 1]; 357 if ((c == '\\') || (c == '/')) 358 return metrics::OmniboxInputType::URL; 359 } 360 361 // If there is more than one recognized non-host component, this is likely to 362 // be a URL, even if the TLD is unknown (in which case this is likely an 363 // intranet URL). 364 if (NumNonHostComponents(*parts) > 1) 365 return metrics::OmniboxInputType::URL; 366 367 // If the host has a known TLD or a port, it's probably a URL, with the 368 // following exceptions: 369 // * Any "IP addresses" that make it here are more likely searches 370 // (see above). 371 // * If we reach here with a username, our input looks like "user@host[.tld]". 372 // Because there is no scheme explicitly specified, we think this is more 373 // likely an email address than an HTTP auth attempt. Hence, we search by 374 // default and let users correct us on a case-by-case basis. 375 // Note that we special-case "localhost" as a known hostname. 376 if ((host_info.family != url::CanonHostInfo::IPV4) && 377 ((registry_length != 0) || (host == base::ASCIIToUTF16("localhost") || 378 parts->port.is_nonempty()))) { 379 return parts->username.is_nonempty() ? metrics::OmniboxInputType::UNKNOWN : 380 metrics::OmniboxInputType::URL; 381 } 382 383 // If we reach this point, we know there's no known TLD on the input, so if 384 // the user wishes to add a desired_tld, the fixup code will oblige; thus this 385 // is a URL. 386 if (!desired_tld.empty()) 387 return metrics::OmniboxInputType::URL; 388 389 // No scheme, password, port, path, and no known TLD on the host. 390 // This could be: 391 // * An "incomplete IP address"; likely a search (see above). 392 // * An email-like input like "user@host", where "host" has no known TLD. 393 // It's not clear what the user means here and searching seems reasonable. 394 // * A single word "foo"; possibly an intranet site, but more likely a search. 395 // This is ideally an UNKNOWN, and we can let the Alternate Nav URL code 396 // catch our mistakes. 397 // * A URL with a valid TLD we don't know about yet. If e.g. a registrar adds 398 // "xxx" as a TLD, then until we add it to our data file, Chrome won't know 399 // "foo.xxx" is a real URL. So ideally this is a URL, but we can't really 400 // distinguish this case from: 401 // * A "URL-like" string that's not really a URL (like 402 // "browser.tabs.closeButtons" or "java.awt.event.*"). This is ideally a 403 // QUERY. Since this is indistinguishable from the case above, and this 404 // case is much more likely, claim these are UNKNOWN, which should default 405 // to the right thing and let users correct us on a case-by-case basis. 406 return metrics::OmniboxInputType::UNKNOWN; 407 } 408 409 // static 410 void AutocompleteInput::ParseForEmphasizeComponents( 411 const base::string16& text, 412 const AutocompleteSchemeClassifier& scheme_classifier, 413 url::Component* scheme, 414 url::Component* host) { 415 url::Parsed parts; 416 base::string16 scheme_str; 417 Parse(text, base::string16(), scheme_classifier, &parts, &scheme_str, NULL); 418 419 *scheme = parts.scheme; 420 *host = parts.host; 421 422 int after_scheme_and_colon = parts.scheme.end() + 1; 423 // For the view-source scheme, we should emphasize the scheme and host of the 424 // URL qualified by the view-source prefix. 425 if (LowerCaseEqualsASCII(scheme_str, kViewSourceScheme) && 426 (static_cast<int>(text.length()) > after_scheme_and_colon)) { 427 // Obtain the URL prefixed by view-source and parse it. 428 base::string16 real_url(text.substr(after_scheme_and_colon)); 429 url::Parsed real_parts; 430 AutocompleteInput::Parse(real_url, base::string16(), scheme_classifier, 431 &real_parts, NULL, NULL); 432 if (real_parts.scheme.is_nonempty() || real_parts.host.is_nonempty()) { 433 if (real_parts.scheme.is_nonempty()) { 434 *scheme = url::Component( 435 after_scheme_and_colon + real_parts.scheme.begin, 436 real_parts.scheme.len); 437 } else { 438 scheme->reset(); 439 } 440 if (real_parts.host.is_nonempty()) { 441 *host = url::Component(after_scheme_and_colon + real_parts.host.begin, 442 real_parts.host.len); 443 } else { 444 host->reset(); 445 } 446 } 447 } else if (LowerCaseEqualsASCII(scheme_str, url::kFileSystemScheme) && 448 parts.inner_parsed() && parts.inner_parsed()->scheme.is_valid()) { 449 *host = parts.inner_parsed()->host; 450 } 451 } 452 453 // static 454 base::string16 AutocompleteInput::FormattedStringWithEquivalentMeaning( 455 const GURL& url, 456 const base::string16& formatted_url, 457 const AutocompleteSchemeClassifier& scheme_classifier) { 458 if (!net::CanStripTrailingSlash(url)) 459 return formatted_url; 460 const base::string16 url_with_path(formatted_url + base::char16('/')); 461 return (AutocompleteInput::Parse(formatted_url, base::string16(), 462 scheme_classifier, NULL, NULL, NULL) == 463 AutocompleteInput::Parse(url_with_path, base::string16(), 464 scheme_classifier, NULL, NULL, NULL)) ? 465 formatted_url : url_with_path; 466 } 467 468 // static 469 int AutocompleteInput::NumNonHostComponents(const url::Parsed& parts) { 470 int num_nonhost_components = 0; 471 if (parts.scheme.is_nonempty()) 472 ++num_nonhost_components; 473 if (parts.username.is_nonempty()) 474 ++num_nonhost_components; 475 if (parts.password.is_nonempty()) 476 ++num_nonhost_components; 477 if (parts.port.is_nonempty()) 478 ++num_nonhost_components; 479 if (parts.path.is_nonempty()) 480 ++num_nonhost_components; 481 if (parts.query.is_nonempty()) 482 ++num_nonhost_components; 483 if (parts.ref.is_nonempty()) 484 ++num_nonhost_components; 485 return num_nonhost_components; 486 } 487 488 // static 489 bool AutocompleteInput::HasHTTPScheme(const base::string16& input) { 490 std::string utf8_input(base::UTF16ToUTF8(input)); 491 url::Component scheme; 492 if (url::FindAndCompareScheme(utf8_input, kViewSourceScheme, &scheme)) { 493 utf8_input.erase(0, scheme.end() + 1); 494 } 495 return url::FindAndCompareScheme(utf8_input, url::kHttpScheme, NULL); 496 } 497 498 void AutocompleteInput::UpdateText(const base::string16& text, 499 size_t cursor_position, 500 const url::Parsed& parts) { 501 DCHECK(cursor_position <= text.length() || 502 cursor_position == base::string16::npos) 503 << "Text: '" << text << "', cp: " << cursor_position; 504 text_ = text; 505 cursor_position_ = cursor_position; 506 parts_ = parts; 507 } 508 509 void AutocompleteInput::Clear() { 510 text_.clear(); 511 cursor_position_ = base::string16::npos; 512 current_url_ = GURL(); 513 current_page_classification_ = metrics::OmniboxEventProto::INVALID_SPEC; 514 type_ = metrics::OmniboxInputType::INVALID; 515 parts_ = url::Parsed(); 516 scheme_.clear(); 517 canonicalized_url_ = GURL(); 518 prevent_inline_autocomplete_ = false; 519 prefer_keyword_ = false; 520 allow_exact_keyword_match_ = false; 521 want_asynchronous_matches_ = true; 522 } 523
