1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/safe_browsing/browser_feature_extractor.h" 6 7 #include <map> 8 #include <string> 9 #include <vector> 10 11 #include "base/memory/scoped_ptr.h" 12 #include "base/message_loop/message_loop.h" 13 #include "base/strings/stringprintf.h" 14 #include "base/time/time.h" 15 #include "chrome/browser/history/history_backend.h" 16 #include "chrome/browser/history/history_service.h" 17 #include "chrome/browser/history/history_service_factory.h" 18 #include "chrome/browser/profiles/profile.h" 19 #include "chrome/browser/safe_browsing/browser_features.h" 20 #include "chrome/browser/safe_browsing/client_side_detection_host.h" 21 #include "chrome/browser/safe_browsing/database_manager.h" 22 #include "chrome/browser/safe_browsing/safe_browsing_service.h" 23 #include "chrome/browser/safe_browsing/ui_manager.h" 24 #include "chrome/common/safe_browsing/csd.pb.h" 25 #include "chrome/test/base/chrome_render_view_host_test_harness.h" 26 #include "chrome/test/base/testing_profile.h" 27 #include "content/public/browser/navigation_controller.h" 28 #include "content/public/browser/web_contents.h" 29 #include "content/public/common/page_transition_types.h" 30 #include "content/public/common/referrer.h" 31 #include "content/public/test/test_browser_thread.h" 32 #include "content/public/test/web_contents_tester.h" 33 #include "testing/gmock/include/gmock/gmock.h" 34 #include "testing/gtest/include/gtest/gtest.h" 35 #include "url/gurl.h" 36 37 using content::BrowserThread; 38 using content::WebContentsTester; 39 40 using testing::DoAll; 41 using testing::Return; 42 using testing::StrictMock; 43 44 namespace safe_browsing { 45 46 namespace { 47 48 class MockSafeBrowsingDatabaseManager : public SafeBrowsingDatabaseManager { 49 public: 50 explicit MockSafeBrowsingDatabaseManager( 51 const scoped_refptr<SafeBrowsingService>& service) 52 : SafeBrowsingDatabaseManager(service) { } 53 54 MOCK_METHOD1(MatchMalwareIP, bool(const std::string& ip_address)); 55 56 protected: 57 virtual ~MockSafeBrowsingDatabaseManager() {} 58 59 private: 60 DISALLOW_COPY_AND_ASSIGN(MockSafeBrowsingDatabaseManager); 61 }; 62 63 class MockClientSideDetectionHost : public ClientSideDetectionHost { 64 public: 65 MockClientSideDetectionHost( 66 content::WebContents* tab, 67 SafeBrowsingDatabaseManager* database_manager) 68 : ClientSideDetectionHost(tab) { 69 set_safe_browsing_managers(NULL, database_manager); 70 } 71 72 virtual ~MockClientSideDetectionHost() {}; 73 74 MOCK_METHOD1(IsBadIpAddress, bool(const std::string&)); 75 }; 76 } // namespace 77 78 class BrowserFeatureExtractorTest : public ChromeRenderViewHostTestHarness { 79 protected: 80 virtual void SetUp() { 81 ChromeRenderViewHostTestHarness::SetUp(); 82 ASSERT_TRUE(profile()->CreateHistoryService( 83 true /* delete_file */, false /* no_db */)); 84 85 db_manager_ = new StrictMock<MockSafeBrowsingDatabaseManager>( 86 SafeBrowsingService::CreateSafeBrowsingService()); 87 host_.reset(new StrictMock<MockClientSideDetectionHost>( 88 web_contents(), db_manager_.get())); 89 extractor_.reset( 90 new BrowserFeatureExtractor(web_contents(), host_.get())); 91 num_pending_ = 0; 92 browse_info_.reset(new BrowseInfo); 93 } 94 95 virtual void TearDown() { 96 extractor_.reset(); 97 host_.reset(); 98 db_manager_ = NULL; 99 profile()->DestroyHistoryService(); 100 ChromeRenderViewHostTestHarness::TearDown(); 101 ASSERT_EQ(0, num_pending_); 102 } 103 104 HistoryService* history_service() { 105 return HistoryServiceFactory::GetForProfile(profile(), 106 Profile::EXPLICIT_ACCESS); 107 } 108 109 void SetRedirectChain(const std::vector<GURL>& redirect_chain, 110 bool new_host) { 111 browse_info_->url_redirects = redirect_chain; 112 if (new_host) { 113 browse_info_->host_redirects = redirect_chain; 114 } 115 } 116 117 // Wrapper around NavigateAndCommit that also sets the redirect chain to 118 // a sane value. 119 void SimpleNavigateAndCommit(const GURL& url) { 120 std::vector<GURL> redirect_chain; 121 redirect_chain.push_back(url); 122 SetRedirectChain(redirect_chain, true); 123 NavigateAndCommit(url, GURL(), content::PAGE_TRANSITION_LINK); 124 } 125 126 // This is similar to NavigateAndCommit that is in WebContentsTester, but 127 // allows us to specify the referrer and page_transition_type. 128 void NavigateAndCommit(const GURL& url, 129 const GURL& referrer, 130 content::PageTransition type) { 131 web_contents()->GetController().LoadURL( 132 url, content::Referrer(referrer, blink::WebReferrerPolicyDefault), 133 type, std::string()); 134 135 static int page_id = 0; 136 content::RenderViewHost* rvh = 137 WebContentsTester::For(web_contents())->GetPendingRenderViewHost(); 138 if (!rvh) { 139 rvh = web_contents()->GetRenderViewHost(); 140 } 141 WebContentsTester::For(web_contents())->ProceedWithCrossSiteNavigation(); 142 WebContentsTester::For(web_contents())->TestDidNavigateWithReferrer( 143 rvh, ++page_id, url, 144 content::Referrer(referrer, blink::WebReferrerPolicyDefault), type); 145 } 146 147 bool ExtractFeatures(ClientPhishingRequest* request) { 148 StartExtractFeatures(request); 149 base::MessageLoop::current()->Run(); 150 EXPECT_EQ(1U, success_.count(request)); 151 return success_.count(request) ? success_[request] : false; 152 } 153 154 void StartExtractFeatures(ClientPhishingRequest* request) { 155 success_.erase(request); 156 ++num_pending_; 157 extractor_->ExtractFeatures( 158 browse_info_.get(), 159 request, 160 base::Bind(&BrowserFeatureExtractorTest::ExtractFeaturesDone, 161 base::Unretained(this))); 162 } 163 164 void GetFeatureMap(const ClientPhishingRequest& request, 165 std::map<std::string, double>* features) { 166 for (int i = 0; i < request.non_model_feature_map_size(); ++i) { 167 const ClientPhishingRequest::Feature& feature = 168 request.non_model_feature_map(i); 169 EXPECT_EQ(0U, features->count(feature.name())); 170 (*features)[feature.name()] = feature.value(); 171 } 172 } 173 174 void ExtractMalwareFeatures(ClientMalwareRequest* request) { 175 // Feature extraction takes ownership of the request object 176 // and passes it along to the done callback in the end. 177 StartExtractMalwareFeatures(request); 178 base::MessageLoopForUI::current()->Run(); 179 EXPECT_EQ(1U, success_.count(request)); 180 EXPECT_TRUE(success_[request]); 181 } 182 183 void StartExtractMalwareFeatures(ClientMalwareRequest* request) { 184 success_.erase(request); 185 ++num_pending_; 186 // We temporarily give up ownership of request to ExtractMalwareFeatures 187 // but we'll regain ownership of it in ExtractMalwareFeaturesDone. 188 extractor_->ExtractMalwareFeatures( 189 browse_info_.get(), 190 request, 191 base::Bind(&BrowserFeatureExtractorTest::ExtractMalwareFeaturesDone, 192 base::Unretained(this))); 193 } 194 195 void GetMalwareUrls( 196 const ClientMalwareRequest& request, 197 std::map<std::string, std::set<std::string> >* urls) { 198 for (int i = 0; i < request.bad_ip_url_info_size(); ++i) { 199 const ClientMalwareRequest::UrlInfo& urlinfo = 200 request.bad_ip_url_info(i); 201 (*urls)[urlinfo.ip()].insert(urlinfo.url()); 202 } 203 } 204 205 int num_pending_; // Number of pending feature extractions. 206 scoped_ptr<BrowserFeatureExtractor> extractor_; 207 std::map<void*, bool> success_; 208 scoped_ptr<BrowseInfo> browse_info_; 209 scoped_ptr<StrictMock<MockClientSideDetectionHost> > host_; 210 scoped_refptr<StrictMock<MockSafeBrowsingDatabaseManager> > db_manager_; 211 212 private: 213 void ExtractFeaturesDone(bool success, ClientPhishingRequest* request) { 214 ASSERT_EQ(0U, success_.count(request)); 215 success_[request] = success; 216 if (--num_pending_ == 0) { 217 base::MessageLoop::current()->Quit(); 218 } 219 } 220 221 void ExtractMalwareFeaturesDone( 222 bool success, 223 scoped_ptr<ClientMalwareRequest> request) { 224 EXPECT_TRUE(BrowserThread::CurrentlyOn(BrowserThread::UI)); 225 ASSERT_EQ(0U, success_.count(request.get())); 226 // The pointer doesn't really belong to us. It belongs to 227 // the test case which passed it to ExtractMalwareFeatures above. 228 success_[request.release()] = success; 229 if (--num_pending_ == 0) { 230 base::MessageLoopForUI::current()->Quit(); 231 } 232 } 233 }; 234 235 TEST_F(BrowserFeatureExtractorTest, UrlNotInHistory) { 236 ClientPhishingRequest request; 237 SimpleNavigateAndCommit(GURL("http://www.google.com")); 238 request.set_url("http://www.google.com/"); 239 request.set_client_score(0.5); 240 EXPECT_FALSE(ExtractFeatures(&request)); 241 } 242 243 TEST_F(BrowserFeatureExtractorTest, RequestNotInitialized) { 244 ClientPhishingRequest request; 245 request.set_url("http://www.google.com/"); 246 // Request is missing the score value. 247 SimpleNavigateAndCommit(GURL("http://www.google.com")); 248 EXPECT_FALSE(ExtractFeatures(&request)); 249 } 250 251 TEST_F(BrowserFeatureExtractorTest, UrlInHistory) { 252 history_service()->AddPage(GURL("http://www.foo.com/bar.html"), 253 base::Time::Now(), 254 history::SOURCE_BROWSED); 255 history_service()->AddPage(GURL("https://www.foo.com/gaa.html"), 256 base::Time::Now(), 257 history::SOURCE_BROWSED); // same host HTTPS. 258 history_service()->AddPage(GURL("http://www.foo.com/gaa.html"), 259 base::Time::Now(), 260 history::SOURCE_BROWSED); // same host HTTP. 261 history_service()->AddPage(GURL("http://bar.foo.com/gaa.html"), 262 base::Time::Now(), 263 history::SOURCE_BROWSED); // different host. 264 history_service()->AddPage(GURL("http://www.foo.com/bar.html?a=b"), 265 base::Time::Now() - base::TimeDelta::FromHours(23), 266 NULL, 0, GURL(), history::RedirectList(), 267 content::PAGE_TRANSITION_LINK, 268 history::SOURCE_BROWSED, false); 269 history_service()->AddPage(GURL("http://www.foo.com/bar.html"), 270 base::Time::Now() - base::TimeDelta::FromHours(25), 271 NULL, 0, GURL(), history::RedirectList(), 272 content::PAGE_TRANSITION_TYPED, 273 history::SOURCE_BROWSED, false); 274 history_service()->AddPage(GURL("https://www.foo.com/goo.html"), 275 base::Time::Now() - base::TimeDelta::FromDays(5), 276 NULL, 0, GURL(), history::RedirectList(), 277 content::PAGE_TRANSITION_TYPED, 278 history::SOURCE_BROWSED, false); 279 280 SimpleNavigateAndCommit(GURL("http://www.foo.com/bar.html")); 281 282 ClientPhishingRequest request; 283 request.set_url("http://www.foo.com/bar.html"); 284 request.set_client_score(0.5); 285 EXPECT_TRUE(ExtractFeatures(&request)); 286 std::map<std::string, double> features; 287 GetFeatureMap(request, &features); 288 289 EXPECT_EQ(12U, features.size()); 290 EXPECT_DOUBLE_EQ(2.0, features[features::kUrlHistoryVisitCount]); 291 EXPECT_DOUBLE_EQ(1.0, 292 features[features::kUrlHistoryVisitCountMoreThan24hAgo]); 293 EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryTypedCount]); 294 EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryLinkCount]); 295 EXPECT_DOUBLE_EQ(4.0, features[features::kHttpHostVisitCount]); 296 EXPECT_DOUBLE_EQ(2.0, features[features::kHttpsHostVisitCount]); 297 EXPECT_DOUBLE_EQ(1.0, features[features::kFirstHttpHostVisitMoreThan24hAgo]); 298 EXPECT_DOUBLE_EQ(1.0, features[features::kFirstHttpsHostVisitMoreThan24hAgo]); 299 300 request.Clear(); 301 request.set_url("http://bar.foo.com/gaa.html"); 302 request.set_client_score(0.5); 303 EXPECT_TRUE(ExtractFeatures(&request)); 304 features.clear(); 305 GetFeatureMap(request, &features); 306 // We have less features because we didn't Navigate to this page, so we don't 307 // have Referrer, IsFirstNavigation, HasSSLReferrer, etc. 308 EXPECT_EQ(7U, features.size()); 309 EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryVisitCount]); 310 EXPECT_DOUBLE_EQ(0.0, 311 features[features::kUrlHistoryVisitCountMoreThan24hAgo]); 312 EXPECT_DOUBLE_EQ(0.0, features[features::kUrlHistoryTypedCount]); 313 EXPECT_DOUBLE_EQ(1.0, features[features::kUrlHistoryLinkCount]); 314 EXPECT_DOUBLE_EQ(1.0, features[features::kHttpHostVisitCount]); 315 EXPECT_DOUBLE_EQ(0.0, features[features::kHttpsHostVisitCount]); 316 EXPECT_DOUBLE_EQ(0.0, features[features::kFirstHttpHostVisitMoreThan24hAgo]); 317 EXPECT_FALSE(features.count(features::kFirstHttpsHostVisitMoreThan24hAgo)); 318 } 319 320 TEST_F(BrowserFeatureExtractorTest, MultipleRequestsAtOnce) { 321 history_service()->AddPage(GURL("http://www.foo.com/bar.html"), 322 base::Time::Now(), 323 history::SOURCE_BROWSED); 324 SimpleNavigateAndCommit(GURL("http:/www.foo.com/bar.html")); 325 ClientPhishingRequest request; 326 request.set_url("http://www.foo.com/bar.html"); 327 request.set_client_score(0.5); 328 StartExtractFeatures(&request); 329 330 SimpleNavigateAndCommit(GURL("http://www.foo.com/goo.html")); 331 ClientPhishingRequest request2; 332 request2.set_url("http://www.foo.com/goo.html"); 333 request2.set_client_score(1.0); 334 StartExtractFeatures(&request2); 335 336 base::MessageLoop::current()->Run(); 337 EXPECT_TRUE(success_[&request]); 338 // Success is false because the second URL is not in the history and we are 339 // not able to distinguish between a missing URL in the history and an error. 340 EXPECT_FALSE(success_[&request2]); 341 } 342 343 TEST_F(BrowserFeatureExtractorTest, BrowseFeatures) { 344 history_service()->AddPage(GURL("http://www.foo.com/"), 345 base::Time::Now(), 346 history::SOURCE_BROWSED); 347 history_service()->AddPage(GURL("http://www.foo.com/page.html"), 348 base::Time::Now(), 349 history::SOURCE_BROWSED); 350 history_service()->AddPage(GURL("http://www.bar.com/"), 351 base::Time::Now(), 352 history::SOURCE_BROWSED); 353 history_service()->AddPage(GURL("http://www.bar.com/other_page.html"), 354 base::Time::Now(), 355 history::SOURCE_BROWSED); 356 history_service()->AddPage(GURL("http://www.baz.com/"), 357 base::Time::Now(), 358 history::SOURCE_BROWSED); 359 360 ClientPhishingRequest request; 361 request.set_url("http://www.foo.com/"); 362 request.set_client_score(0.5); 363 std::vector<GURL> redirect_chain; 364 redirect_chain.push_back(GURL("http://somerandomwebsite.com/")); 365 redirect_chain.push_back(GURL("http://www.foo.com/")); 366 SetRedirectChain(redirect_chain, true); 367 browse_info_->http_status_code = 200; 368 NavigateAndCommit(GURL("http://www.foo.com/"), 369 GURL("http://google.com/"), 370 content::PageTransitionFromInt( 371 content::PAGE_TRANSITION_AUTO_BOOKMARK | 372 content::PAGE_TRANSITION_FORWARD_BACK)); 373 374 EXPECT_TRUE(ExtractFeatures(&request)); 375 std::map<std::string, double> features; 376 GetFeatureMap(request, &features); 377 378 EXPECT_EQ(1.0, 379 features[base::StringPrintf("%s=%s", 380 features::kReferrer, 381 "http://google.com/")]); 382 EXPECT_EQ(1.0, 383 features[base::StringPrintf("%s[0]=%s", 384 features::kRedirect, 385 "http://somerandomwebsite.com/")]); 386 // We shouldn't have a feature for the last redirect in the chain, since it 387 // should always be the URL that we navigated to. 388 EXPECT_EQ(0.0, 389 features[base::StringPrintf("%s[1]=%s", 390 features::kRedirect, 391 "http://foo.com/")]); 392 EXPECT_EQ(0.0, features[features::kHasSSLReferrer]); 393 EXPECT_EQ(2.0, features[features::kPageTransitionType]); 394 EXPECT_EQ(1.0, features[features::kIsFirstNavigation]); 395 EXPECT_EQ(200.0, features[features::kHttpStatusCode]); 396 397 request.Clear(); 398 request.set_url("http://www.foo.com/page.html"); 399 request.set_client_score(0.5); 400 redirect_chain.clear(); 401 redirect_chain.push_back(GURL("http://www.foo.com/redirect")); 402 redirect_chain.push_back(GURL("http://www.foo.com/second_redirect")); 403 redirect_chain.push_back(GURL("http://www.foo.com/page.html")); 404 SetRedirectChain(redirect_chain, false); 405 browse_info_->http_status_code = 404; 406 NavigateAndCommit(GURL("http://www.foo.com/page.html"), 407 GURL("http://www.foo.com"), 408 content::PageTransitionFromInt( 409 content::PAGE_TRANSITION_TYPED | 410 content::PAGE_TRANSITION_CHAIN_START | 411 content::PAGE_TRANSITION_CLIENT_REDIRECT)); 412 413 EXPECT_TRUE(ExtractFeatures(&request)); 414 features.clear(); 415 GetFeatureMap(request, &features); 416 417 EXPECT_EQ(1, 418 features[base::StringPrintf("%s=%s", 419 features::kReferrer, 420 "http://www.foo.com/")]); 421 EXPECT_EQ(1.0, 422 features[base::StringPrintf("%s[0]=%s", 423 features::kRedirect, 424 "http://www.foo.com/redirect")]); 425 EXPECT_EQ(1.0, 426 features[base::StringPrintf("%s[1]=%s", 427 features::kRedirect, 428 "http://www.foo.com/second_redirect")]); 429 EXPECT_EQ(0.0, features[features::kHasSSLReferrer]); 430 EXPECT_EQ(1.0, features[features::kPageTransitionType]); 431 EXPECT_EQ(0.0, features[features::kIsFirstNavigation]); 432 EXPECT_EQ(1.0, 433 features[base::StringPrintf("%s%s=%s", 434 features::kHostPrefix, 435 features::kReferrer, 436 "http://google.com/")]); 437 EXPECT_EQ(1.0, 438 features[base::StringPrintf("%s%s[0]=%s", 439 features::kHostPrefix, 440 features::kRedirect, 441 "http://somerandomwebsite.com/")]); 442 EXPECT_EQ(2.0, 443 features[base::StringPrintf("%s%s", 444 features::kHostPrefix, 445 features::kPageTransitionType)]); 446 EXPECT_EQ(1.0, 447 features[base::StringPrintf("%s%s", 448 features::kHostPrefix, 449 features::kIsFirstNavigation)]); 450 EXPECT_EQ(404.0, features[features::kHttpStatusCode]); 451 452 request.Clear(); 453 request.set_url("http://www.bar.com/"); 454 request.set_client_score(0.5); 455 redirect_chain.clear(); 456 redirect_chain.push_back(GURL("http://www.foo.com/page.html")); 457 redirect_chain.push_back(GURL("http://www.bar.com/")); 458 SetRedirectChain(redirect_chain, true); 459 NavigateAndCommit(GURL("http://www.bar.com/"), 460 GURL("http://www.foo.com/page.html"), 461 content::PageTransitionFromInt( 462 content::PAGE_TRANSITION_LINK | 463 content::PAGE_TRANSITION_CHAIN_END | 464 content::PAGE_TRANSITION_CLIENT_REDIRECT)); 465 466 EXPECT_TRUE(ExtractFeatures(&request)); 467 features.clear(); 468 GetFeatureMap(request, &features); 469 470 EXPECT_EQ(1.0, 471 features[base::StringPrintf("%s=%s", 472 features::kReferrer, 473 "http://www.foo.com/page.html")]); 474 EXPECT_EQ(1.0, 475 features[base::StringPrintf("%s[0]=%s", 476 features::kRedirect, 477 "http://www.foo.com/page.html")]); 478 EXPECT_EQ(0.0, features[features::kHasSSLReferrer]); 479 EXPECT_EQ(0.0, features[features::kPageTransitionType]); 480 EXPECT_EQ(0.0, features[features::kIsFirstNavigation]); 481 482 // Should not have host features. 483 EXPECT_EQ(0U, 484 features.count(base::StringPrintf("%s%s", 485 features::kHostPrefix, 486 features::kPageTransitionType))); 487 EXPECT_EQ(0U, 488 features.count(base::StringPrintf("%s%s", 489 features::kHostPrefix, 490 features::kIsFirstNavigation))); 491 492 request.Clear(); 493 request.set_url("http://www.bar.com/other_page.html"); 494 request.set_client_score(0.5); 495 redirect_chain.clear(); 496 redirect_chain.push_back(GURL("http://www.bar.com/other_page.html")); 497 SetRedirectChain(redirect_chain, false); 498 NavigateAndCommit(GURL("http://www.bar.com/other_page.html"), 499 GURL("http://www.bar.com/"), 500 content::PAGE_TRANSITION_LINK); 501 502 EXPECT_TRUE(ExtractFeatures(&request)); 503 features.clear(); 504 GetFeatureMap(request, &features); 505 506 EXPECT_EQ(1.0, 507 features[base::StringPrintf("%s=%s", 508 features::kReferrer, 509 "http://www.bar.com/")]); 510 EXPECT_EQ(0.0, features[features::kHasSSLReferrer]); 511 EXPECT_EQ(0.0, features[features::kPageTransitionType]); 512 EXPECT_EQ(0.0, features[features::kIsFirstNavigation]); 513 EXPECT_EQ(1.0, 514 features[base::StringPrintf("%s%s=%s", 515 features::kHostPrefix, 516 features::kReferrer, 517 "http://www.foo.com/page.html")]); 518 EXPECT_EQ(1.0, 519 features[base::StringPrintf("%s%s[0]=%s", 520 features::kHostPrefix, 521 features::kRedirect, 522 "http://www.foo.com/page.html")]); 523 EXPECT_EQ(0.0, 524 features[base::StringPrintf("%s%s", 525 features::kHostPrefix, 526 features::kPageTransitionType)]); 527 EXPECT_EQ(0.0, 528 features[base::StringPrintf("%s%s", 529 features::kHostPrefix, 530 features::kIsFirstNavigation)]); 531 request.Clear(); 532 request.set_url("http://www.baz.com/"); 533 request.set_client_score(0.5); 534 redirect_chain.clear(); 535 redirect_chain.push_back(GURL("https://bankofamerica.com")); 536 redirect_chain.push_back(GURL("http://www.baz.com/")); 537 SetRedirectChain(redirect_chain, true); 538 NavigateAndCommit(GURL("http://www.baz.com"), 539 GURL("https://bankofamerica.com"), 540 content::PAGE_TRANSITION_GENERATED); 541 542 EXPECT_TRUE(ExtractFeatures(&request)); 543 features.clear(); 544 GetFeatureMap(request, &features); 545 546 EXPECT_EQ(1.0, 547 features[base::StringPrintf("%s[0]=%s", 548 features::kRedirect, 549 features::kSecureRedirectValue)]); 550 EXPECT_EQ(1.0, features[features::kHasSSLReferrer]); 551 EXPECT_EQ(5.0, features[features::kPageTransitionType]); 552 // Should not have redirect or host features. 553 EXPECT_EQ(0U, 554 features.count(base::StringPrintf("%s%s", 555 features::kHostPrefix, 556 features::kPageTransitionType))); 557 EXPECT_EQ(0U, 558 features.count(base::StringPrintf("%s%s", 559 features::kHostPrefix, 560 features::kIsFirstNavigation))); 561 EXPECT_EQ(5.0, features[features::kPageTransitionType]); 562 } 563 564 TEST_F(BrowserFeatureExtractorTest, SafeBrowsingFeatures) { 565 SimpleNavigateAndCommit(GURL("http://www.foo.com/malware.html")); 566 ClientPhishingRequest request; 567 request.set_url("http://www.foo.com/malware.html"); 568 request.set_client_score(0.5); 569 570 browse_info_->unsafe_resource.reset( 571 new SafeBrowsingUIManager::UnsafeResource); 572 browse_info_->unsafe_resource->url = GURL("http://www.malware.com/"); 573 browse_info_->unsafe_resource->original_url = GURL("http://www.good.com/"); 574 browse_info_->unsafe_resource->is_subresource = true; 575 browse_info_->unsafe_resource->threat_type = SB_THREAT_TYPE_URL_MALWARE; 576 577 ExtractFeatures(&request); 578 std::map<std::string, double> features; 579 GetFeatureMap(request, &features); 580 EXPECT_TRUE(features.count(base::StringPrintf( 581 "%s%s", 582 features::kSafeBrowsingMaliciousUrl, 583 "http://www.malware.com/"))); 584 EXPECT_TRUE(features.count(base::StringPrintf( 585 "%s%s", 586 features::kSafeBrowsingOriginalUrl, 587 "http://www.good.com/"))); 588 EXPECT_DOUBLE_EQ(1.0, features[features::kSafeBrowsingIsSubresource]); 589 EXPECT_DOUBLE_EQ(2.0, features[features::kSafeBrowsingThreatType]); 590 } 591 592 TEST_F(BrowserFeatureExtractorTest, MalwareFeatures) { 593 ClientMalwareRequest request; 594 request.set_url("http://www.foo.com/"); 595 596 std::vector<IPUrlInfo> bad_urls; 597 bad_urls.push_back(IPUrlInfo("http://bad.com", "GET", "", 598 ResourceType::SCRIPT)); 599 bad_urls.push_back(IPUrlInfo("http://evil.com", "GET", "", 600 ResourceType::SCRIPT)); 601 browse_info_->ips.insert(std::make_pair("193.5.163.8", bad_urls)); 602 browse_info_->ips.insert(std::make_pair("92.92.92.92", bad_urls)); 603 std::vector<IPUrlInfo> good_urls; 604 good_urls.push_back(IPUrlInfo("http://ok.com", "GET", "", 605 ResourceType::SCRIPT)); 606 browse_info_->ips.insert(std::make_pair("23.94.78.1", good_urls)); 607 EXPECT_CALL(*db_manager_, MatchMalwareIP("193.5.163.8")) 608 .WillOnce(Return(true)); 609 EXPECT_CALL(*db_manager_, MatchMalwareIP("92.92.92.92")) 610 .WillOnce(Return(true)); 611 EXPECT_CALL(*db_manager_, MatchMalwareIP("23.94.78.1")) 612 .WillOnce(Return(false)); 613 614 ExtractMalwareFeatures(&request); 615 EXPECT_EQ(4, request.bad_ip_url_info_size()); 616 std::map<std::string, std::set<std::string> > result_urls; 617 GetMalwareUrls(request, &result_urls); 618 619 EXPECT_EQ(2U, result_urls.size()); 620 EXPECT_TRUE(result_urls.count("193.5.163.8")); 621 std::set<std::string> urls = result_urls["193.5.163.8"]; 622 EXPECT_EQ(2U, urls.size()); 623 EXPECT_TRUE(urls.find("http://bad.com") != urls.end()); 624 EXPECT_TRUE(urls.find("http://evil.com") != urls.end()); 625 EXPECT_TRUE(result_urls.count("92.92.92.92")); 626 urls = result_urls["92.92.92.92"]; 627 EXPECT_EQ(2U, urls.size()); 628 EXPECT_TRUE(urls.find("http://bad.com") != urls.end()); 629 EXPECT_TRUE(urls.find("http://evil.com") != urls.end()); 630 } 631 632 TEST_F(BrowserFeatureExtractorTest, MalwareFeatures_ExceedLimit) { 633 ClientMalwareRequest request; 634 request.set_url("http://www.foo.com/"); 635 636 std::vector<IPUrlInfo> bad_urls; 637 bad_urls.push_back(IPUrlInfo("http://bad.com", "GET", "", 638 ResourceType::SCRIPT)); 639 std::vector<std::string> ips; 640 for (int i = 0; i < 7; ++i) { // Add 7 ips 641 std::string ip = base::StringPrintf("%d.%d.%d.%d", i, i, i, i); 642 ips.push_back(ip); 643 browse_info_->ips.insert(std::make_pair(ip, bad_urls)); 644 645 // First ip is good but all the others are bad. 646 EXPECT_CALL(*db_manager_, MatchMalwareIP(ip)).WillOnce(Return(i > 0)); 647 } 648 649 ExtractMalwareFeatures(&request); 650 // The number of IP matched url we store is capped at 5 IPs per request. 651 EXPECT_EQ(5, request.bad_ip_url_info_size()); 652 } 653 654 } // namespace safe_browsing 655
