1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import os 6 import re 7 import shutil 8 import sys 9 import StringIO 10 import tempfile 11 import time 12 import unittest 13 14 from telemetry import benchmark 15 from telemetry import story 16 from telemetry.core import exceptions 17 from telemetry.core import util 18 from telemetry import decorators 19 from telemetry.internal.browser import browser_finder 20 from telemetry.internal.browser import user_agent 21 from telemetry.internal.results import results_options 22 from telemetry.internal import story_runner 23 from telemetry.internal.testing.test_page_sets import example_domain 24 from telemetry.internal.util import exception_formatter 25 from telemetry.page import page as page_module 26 from telemetry.page import legacy_page_test 27 from telemetry.page import shared_page_state 28 from telemetry.page import traffic_setting as traffic_setting_module 29 from telemetry.util import image_util 30 from telemetry.testing import fakes 31 from telemetry.testing import options_for_unittests 32 from telemetry.testing import system_stub 33 34 35 SIMPLE_CREDENTIALS_STRING = """ 36 { 37 "test": { 38 "username": "example", 39 "password": "asdf" 40 } 41 } 42 """ 43 44 45 class DummyTest(legacy_page_test.LegacyPageTest): 46 47 def ValidateAndMeasurePage(self, *_): 48 pass 49 50 51 def SetUpStoryRunnerArguments(options): 52 parser = options.CreateParser() 53 story_runner.AddCommandLineArgs(parser) 54 options.MergeDefaultValues(parser.get_default_values()) 55 story_runner.ProcessCommandLineArgs(parser, options) 56 57 58 class EmptyMetadataForTest(benchmark.BenchmarkMetadata): 59 60 def __init__(self): 61 super(EmptyMetadataForTest, self).__init__('') 62 63 64 class StubCredentialsBackend(object): 65 66 def __init__(self, login_return_value): 67 self.did_get_login = False 68 self.did_get_login_no_longer_needed = False 69 self.login_return_value = login_return_value 70 71 @property 72 def credentials_type(self): 73 return 'test' 74 75 def LoginNeeded(self, *_): 76 self.did_get_login = True 77 return self.login_return_value 78 79 def LoginNoLongerNeeded(self, _): 80 self.did_get_login_no_longer_needed = True 81 82 83 def GetSuccessfulPageRuns(results): 84 return [run for run in results.all_page_runs if run.ok or run.skipped] 85 86 87 def CaptureStderr(func, output_buffer): 88 def wrapper(*args, **kwargs): 89 original_stderr, sys.stderr = sys.stderr, output_buffer 90 try: 91 return func(*args, **kwargs) 92 finally: 93 sys.stderr = original_stderr 94 return wrapper 95 96 97 # TODO: remove test cases that use real browsers and replace with a 98 # story_runner or shared_page_state unittest that tests the same logic. 99 class ActualPageRunEndToEndTests(unittest.TestCase): 100 # TODO(nduca): Move the basic "test failed, test succeeded" tests from 101 # page_test_unittest to here. 102 103 def setUp(self): 104 self._story_runner_logging_stub = None 105 self._formatted_exception_buffer = StringIO.StringIO() 106 self._original_formatter = exception_formatter.PrintFormattedException 107 108 def tearDown(self): 109 self.RestoreExceptionFormatter() 110 111 def CaptureFormattedException(self): 112 exception_formatter.PrintFormattedException = CaptureStderr( 113 exception_formatter.PrintFormattedException, 114 self._formatted_exception_buffer) 115 self._story_runner_logging_stub = system_stub.Override( 116 story_runner, ['logging']) 117 118 @property 119 def formatted_exception(self): 120 return self._formatted_exception_buffer.getvalue() 121 122 def RestoreExceptionFormatter(self): 123 exception_formatter.PrintFormattedException = self._original_formatter 124 if self._story_runner_logging_stub: 125 self._story_runner_logging_stub.Restore() 126 self._story_runner_logging_stub = None 127 128 def assertFormattedExceptionIsEmpty(self): 129 self.longMessage = False 130 self.assertEquals( 131 '', self.formatted_exception, 132 msg='Expected empty formatted exception: actual=%s' % '\n > '.join( 133 self.formatted_exception.split('\n'))) 134 135 def assertFormattedExceptionOnlyHas(self, expected_exception_name): 136 self.longMessage = True 137 actual_exception_names = re.findall(r'^Traceback.*?^(\w+)', 138 self.formatted_exception, 139 re.DOTALL | re.MULTILINE) 140 self.assertEquals([expected_exception_name], actual_exception_names, 141 msg='Full formatted exception: %s' % '\n > '.join( 142 self.formatted_exception.split('\n'))) 143 144 def testRaiseBrowserGoneExceptionFromRestartBrowserBeforeEachPage(self): 145 self.CaptureFormattedException() 146 story_set = story.StorySet() 147 story_set.AddStory(page_module.Page( 148 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 149 name='foo')) 150 story_set.AddStory(page_module.Page( 151 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 152 name='bar')) 153 story_set.AddStory(page_module.Page( 154 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 155 name='baz')) 156 157 class Test(legacy_page_test.LegacyPageTest): 158 159 def __init__(self, *args): 160 super(Test, self).__init__( 161 *args, needs_browser_restart_after_each_page=True) 162 self.run_count = 0 163 164 def RestartBrowserBeforeEachPage(self): 165 # This will only be called twice with 3 pages. 166 old_run_count = self.run_count 167 self.run_count += 1 168 if old_run_count == 1: 169 raise exceptions.BrowserGoneException(None) 170 return self._needs_browser_restart_after_each_page 171 172 def ValidateAndMeasurePage(self, page, tab, results): 173 pass 174 175 options = options_for_unittests.GetCopy() 176 options.output_formats = ['none'] 177 options.suppress_gtest_report = True 178 test = Test() 179 SetUpStoryRunnerArguments(options) 180 results = results_options.CreateResults(EmptyMetadataForTest(), options) 181 story_runner.Run(test, story_set, options, results) 182 self.assertEquals(2, test.run_count) 183 self.assertEquals(2, len(GetSuccessfulPageRuns(results))) 184 self.assertEquals(1, len(results.failures)) 185 self.assertFormattedExceptionIsEmpty() 186 187 def testNeedsBrowserRestartAfterEachPage(self): 188 self.CaptureFormattedException() 189 story_set = story.StorySet() 190 story_set.AddStory(page_module.Page( 191 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 192 name='foo')) 193 story_set.AddStory(page_module.Page( 194 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 195 name='bar')) 196 197 class Test(legacy_page_test.LegacyPageTest): 198 199 def __init__(self, *args, **kwargs): 200 super(Test, self).__init__(*args, **kwargs) 201 self.browser_starts = 0 202 203 def DidStartBrowser(self, *args): 204 super(Test, self).DidStartBrowser(*args) 205 self.browser_starts += 1 206 207 def ValidateAndMeasurePage(self, page, tab, results): 208 pass 209 210 options = options_for_unittests.GetCopy() 211 options.output_formats = ['none'] 212 options.suppress_gtest_report = True 213 test = Test(needs_browser_restart_after_each_page=True) 214 SetUpStoryRunnerArguments(options) 215 results = results_options.CreateResults(EmptyMetadataForTest(), options) 216 story_runner.Run(test, story_set, options, results) 217 self.assertEquals(2, len(GetSuccessfulPageRuns(results))) 218 self.assertEquals(2, test.browser_starts) 219 self.assertFormattedExceptionIsEmpty() 220 221 def testCredentialsWhenLoginFails(self): 222 self.CaptureFormattedException() 223 credentials_backend = StubCredentialsBackend(login_return_value=False) 224 did_run = self.runCredentialsTest(credentials_backend) 225 assert credentials_backend.did_get_login 226 assert not credentials_backend.did_get_login_no_longer_needed 227 assert not did_run 228 self.assertFormattedExceptionIsEmpty() 229 230 def testCredentialsWhenLoginSucceeds(self): 231 credentials_backend = StubCredentialsBackend(login_return_value=True) 232 did_run = self.runCredentialsTest(credentials_backend) 233 assert credentials_backend.did_get_login 234 assert credentials_backend.did_get_login_no_longer_needed 235 assert did_run 236 237 def runCredentialsTest(self, credentials_backend): 238 story_set = story.StorySet() 239 did_run = [False] 240 241 try: 242 with tempfile.NamedTemporaryFile(delete=False) as f: 243 page = page_module.Page( 244 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 245 credentials_path=f.name) 246 page.credentials = "test" 247 story_set.AddStory(page) 248 249 f.write(SIMPLE_CREDENTIALS_STRING) 250 251 class TestThatInstallsCredentialsBackend(legacy_page_test.LegacyPageTest): 252 253 def __init__(self, credentials_backend): 254 super(TestThatInstallsCredentialsBackend, self).__init__() 255 self._credentials_backend = credentials_backend 256 257 def DidStartBrowser(self, browser): 258 browser.credentials.AddBackend(self._credentials_backend) 259 260 def ValidateAndMeasurePage(self, *_): 261 did_run[0] = True 262 263 test = TestThatInstallsCredentialsBackend(credentials_backend) 264 options = options_for_unittests.GetCopy() 265 options.output_formats = ['none'] 266 options.suppress_gtest_report = True 267 SetUpStoryRunnerArguments(options) 268 results = results_options.CreateResults(EmptyMetadataForTest(), options) 269 story_runner.Run(test, story_set, options, results) 270 finally: 271 os.remove(f.name) 272 273 return did_run[0] 274 275 @decorators.Disabled('chromeos') # crbug.com/483212 276 def testUserAgent(self): 277 story_set = story.StorySet() 278 page = page_module.Page( 279 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 280 shared_page_state_class=shared_page_state.SharedTabletPageState) 281 story_set.AddStory(page) 282 283 class TestUserAgent(legacy_page_test.LegacyPageTest): 284 def ValidateAndMeasurePage(self, page, tab, results): 285 del page, results # unused 286 actual_user_agent = tab.EvaluateJavaScript( 287 'window.navigator.userAgent') 288 expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] 289 assert actual_user_agent.strip() == expected_user_agent 290 291 # This is so we can check later that the test actually made it into this 292 # function. Previously it was timing out before even getting here, which 293 # should fail, but since it skipped all the asserts, it slipped by. 294 self.hasRun = True # pylint: disable=attribute-defined-outside-init 295 296 test = TestUserAgent() 297 options = options_for_unittests.GetCopy() 298 options.output_formats = ['none'] 299 options.suppress_gtest_report = True 300 SetUpStoryRunnerArguments(options) 301 results = results_options.CreateResults(EmptyMetadataForTest(), options) 302 story_runner.Run(test, story_set, options, results) 303 304 self.assertTrue(hasattr(test, 'hasRun') and test.hasRun) 305 306 # Ensure that story_runner forces exactly 1 tab before running a page. 307 @decorators.Enabled('has tabs') 308 def testOneTab(self): 309 story_set = story.StorySet() 310 page = page_module.Page( 311 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir()) 312 story_set.AddStory(page) 313 314 class TestOneTab(legacy_page_test.LegacyPageTest): 315 316 def DidStartBrowser(self, browser): 317 browser.tabs.New() 318 319 def ValidateAndMeasurePage(self, page, tab, results): 320 del page, results # unused 321 assert len(tab.browser.tabs) == 1 322 323 test = TestOneTab() 324 options = options_for_unittests.GetCopy() 325 options.output_formats = ['none'] 326 options.suppress_gtest_report = True 327 SetUpStoryRunnerArguments(options) 328 results = results_options.CreateResults(EmptyMetadataForTest(), options) 329 story_runner.Run(test, story_set, options, results) 330 331 @decorators.Disabled('chromeos') # crbug.com/652385 332 def testTrafficSettings(self): 333 story_set = story.StorySet() 334 slow_page = page_module.Page( 335 'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(), 336 name='slow', 337 traffic_setting=traffic_setting_module.REGULAR_2G) 338 fast_page = page_module.Page( 339 'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(), 340 name='fast', 341 traffic_setting=traffic_setting_module.WIFI) 342 story_set.AddStory(slow_page) 343 story_set.AddStory(fast_page) 344 345 latencies_by_page_in_ms = {} 346 347 class MeasureLatency(legacy_page_test.LegacyPageTest): 348 def __init__(self): 349 super(MeasureLatency, self).__init__() 350 self._will_navigate_time = None 351 352 def WillNavigateToPage(self, page, tab): 353 del page, tab # unused 354 self._will_navigate_time = time.time() * 1000 355 356 def ValidateAndMeasurePage(self, page, tab, results): 357 del results # unused 358 latencies_by_page_in_ms[page.name] = ( 359 time.time() * 1000 - self._will_navigate_time) 360 361 test = MeasureLatency() 362 options = options_for_unittests.GetCopy() 363 options.output_formats = ['none'] 364 options.suppress_gtest_report = True 365 SetUpStoryRunnerArguments(options) 366 results = results_options.CreateResults(EmptyMetadataForTest(), options) 367 story_runner.Run(test, story_set, options, results) 368 # Slow page should be slower than fast page by at least 300 ms (roundtrip 369 # time of 2G) - 2 ms (roundtrip time of Wifi) 370 self.assertGreater(latencies_by_page_in_ms['slow'], 371 latencies_by_page_in_ms['fast'] + 300 - 2) 372 373 # Ensure that story_runner allows >1 tab for multi-tab test. 374 @decorators.Enabled('has tabs') 375 def testMultipleTabsOkayForMultiTabTest(self): 376 story_set = story.StorySet() 377 page = page_module.Page( 378 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir()) 379 story_set.AddStory(page) 380 381 class TestMultiTabs(legacy_page_test.LegacyPageTest): 382 def TabForPage(self, page, browser): 383 del page # unused 384 return browser.tabs.New() 385 386 def ValidateAndMeasurePage(self, page, tab, results): 387 del page, results # unused 388 assert len(tab.browser.tabs) == 2 389 390 test = TestMultiTabs() 391 options = options_for_unittests.GetCopy() 392 options.output_formats = ['none'] 393 options.suppress_gtest_report = True 394 SetUpStoryRunnerArguments(options) 395 results = results_options.CreateResults(EmptyMetadataForTest(), options) 396 story_runner.Run(test, story_set, options, results) 397 398 # Ensure that story_runner allows the test to customize the browser 399 # before it launches. 400 def testBrowserBeforeLaunch(self): 401 story_set = story.StorySet() 402 page = page_module.Page( 403 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir()) 404 story_set.AddStory(page) 405 406 class TestBeforeLaunch(legacy_page_test.LegacyPageTest): 407 408 def __init__(self): 409 super(TestBeforeLaunch, self).__init__() 410 self._did_call_will_start = False 411 self._did_call_did_start = False 412 413 def WillStartBrowser(self, platform): 414 self._did_call_will_start = True 415 # TODO(simonjam): Test that the profile is available. 416 417 def DidStartBrowser(self, browser): 418 assert self._did_call_will_start 419 self._did_call_did_start = True 420 421 def ValidateAndMeasurePage(self, *_): 422 assert self._did_call_did_start 423 424 test = TestBeforeLaunch() 425 options = options_for_unittests.GetCopy() 426 options.output_formats = ['none'] 427 options.suppress_gtest_report = True 428 SetUpStoryRunnerArguments(options) 429 results = results_options.CreateResults(EmptyMetadataForTest(), options) 430 story_runner.Run(test, story_set, options, results) 431 432 def testRunPageWithStartupUrl(self): 433 num_times_browser_closed = [0] 434 435 class TestSharedState(shared_page_state.SharedPageState): 436 437 def _StopBrowser(self): 438 super(TestSharedState, self)._StopBrowser() 439 num_times_browser_closed[0] += 1 440 441 story_set = story.StorySet() 442 page = page_module.Page( 443 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), 444 startup_url='about:blank', shared_page_state_class=TestSharedState) 445 story_set.AddStory(page) 446 447 class Measurement(legacy_page_test.LegacyPageTest): 448 449 def __init__(self): 450 super(Measurement, self).__init__() 451 452 def ValidateAndMeasurePage(self, page, tab, results): 453 del page, tab, results # not used 454 455 options = options_for_unittests.GetCopy() 456 options.pageset_repeat = 2 457 options.output_formats = ['none'] 458 options.suppress_gtest_report = True 459 if not browser_finder.FindBrowser(options): 460 return 461 test = Measurement() 462 SetUpStoryRunnerArguments(options) 463 results = results_options.CreateResults(EmptyMetadataForTest(), options) 464 story_runner.Run(test, story_set, options, results) 465 self.assertEquals('about:blank', options.browser_options.startup_url) 466 # _StopBrowser should be called 2 times: 467 # 1. browser restarts after page 1 run 468 # 2. in the TearDownState after all the pages have run. 469 self.assertEquals(num_times_browser_closed[0], 2) 470 471 # Ensure that story_runner calls cleanUp when a page run fails. 472 def testCleanUpPage(self): 473 story_set = story.StorySet() 474 page = page_module.Page( 475 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir()) 476 story_set.AddStory(page) 477 478 class Test(legacy_page_test.LegacyPageTest): 479 480 def __init__(self): 481 super(Test, self).__init__() 482 self.did_call_clean_up = False 483 484 def ValidateAndMeasurePage(self, *_): 485 raise legacy_page_test.Failure 486 487 def DidRunPage(self, platform): 488 del platform # unused 489 self.did_call_clean_up = True 490 491 test = Test() 492 options = options_for_unittests.GetCopy() 493 options.output_formats = ['none'] 494 options.suppress_gtest_report = True 495 SetUpStoryRunnerArguments(options) 496 results = results_options.CreateResults(EmptyMetadataForTest(), options) 497 story_runner.Run(test, story_set, options, results) 498 assert test.did_call_clean_up 499 500 # Ensure skipping the test if shared state cannot be run on the browser. 501 def testSharedPageStateCannotRunOnBrowser(self): 502 story_set = story.StorySet() 503 504 class UnrunnableSharedState(shared_page_state.SharedPageState): 505 def CanRunOnBrowser(self, browser_info, page): 506 del browser_info, page # unused 507 return False 508 509 def ValidateAndMeasurePage(self, _): 510 pass 511 512 story_set.AddStory(page_module.Page( 513 url='file://blank.html', page_set=story_set, 514 base_dir=util.GetUnittestDataDir(), 515 shared_page_state_class=UnrunnableSharedState)) 516 517 class Test(legacy_page_test.LegacyPageTest): 518 519 def __init__(self, *args, **kwargs): 520 super(Test, self).__init__(*args, **kwargs) 521 self.will_navigate_to_page_called = False 522 523 def ValidateAndMeasurePage(self, *args): 524 del args # unused 525 raise Exception('Exception should not be thrown') 526 527 def WillNavigateToPage(self, page, tab): 528 del page, tab # unused 529 self.will_navigate_to_page_called = True 530 531 test = Test() 532 options = options_for_unittests.GetCopy() 533 options.output_formats = ['none'] 534 options.suppress_gtest_report = True 535 SetUpStoryRunnerArguments(options) 536 results = results_options.CreateResults(EmptyMetadataForTest(), options) 537 story_runner.Run(test, story_set, options, results) 538 self.assertFalse(test.will_navigate_to_page_called) 539 self.assertEquals(1, len(GetSuccessfulPageRuns(results))) 540 self.assertEquals(1, len(results.skipped_values)) 541 self.assertEquals(0, len(results.failures)) 542 543 def testRunPageWithProfilingFlag(self): 544 story_set = story.StorySet() 545 story_set.AddStory(page_module.Page( 546 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir())) 547 548 class Measurement(legacy_page_test.LegacyPageTest): 549 550 def ValidateAndMeasurePage(self, page, tab, results): 551 pass 552 553 options = options_for_unittests.GetCopy() 554 options.output_formats = ['none'] 555 options.suppress_gtest_report = True 556 options.reset_results = None 557 options.upload_results = None 558 options.results_label = None 559 options.output_dir = tempfile.mkdtemp() 560 options.profiler = 'trace' 561 try: 562 SetUpStoryRunnerArguments(options) 563 results = results_options.CreateResults(EmptyMetadataForTest(), options) 564 story_runner.Run(Measurement(), story_set, options, results) 565 self.assertEquals(1, len(GetSuccessfulPageRuns(results))) 566 self.assertEquals(0, len(results.failures)) 567 self.assertEquals(0, len(results.all_page_specific_values)) 568 self.assertTrue(os.path.isfile( 569 os.path.join(options.output_dir, 'blank_html.html'))) 570 finally: 571 shutil.rmtree(options.output_dir) 572 573 def _RunPageTestThatRaisesAppCrashException(self, test, max_failures): 574 class TestPage(page_module.Page): 575 576 def RunNavigateSteps(self, _): 577 raise exceptions.AppCrashException 578 579 story_set = story.StorySet() 580 for i in range(5): 581 story_set.AddStory( 582 TestPage('file://blank.html', story_set, 583 base_dir=util.GetUnittestDataDir(), name='foo%d' % i)) 584 options = options_for_unittests.GetCopy() 585 options.output_formats = ['none'] 586 options.suppress_gtest_report = True 587 SetUpStoryRunnerArguments(options) 588 results = results_options.CreateResults(EmptyMetadataForTest(), options) 589 story_runner.Run(test, story_set, options, results, 590 max_failures=max_failures) 591 return results 592 593 def testSingleTabMeansCrashWillCauseFailureValue(self): 594 self.CaptureFormattedException() 595 596 class SingleTabTest(legacy_page_test.LegacyPageTest): 597 # Test is not multi-tab because it does not override TabForPage. 598 599 def ValidateAndMeasurePage(self, *_): 600 pass 601 602 test = SingleTabTest() 603 results = self._RunPageTestThatRaisesAppCrashException( 604 test, max_failures=1) 605 self.assertEquals([], GetSuccessfulPageRuns(results)) 606 self.assertEquals(2, len(results.failures)) # max_failures + 1 607 self.assertFormattedExceptionIsEmpty() 608 609 @decorators.Enabled('has tabs') 610 def testMultipleTabsMeansCrashRaises(self): 611 self.CaptureFormattedException() 612 613 class MultipleTabsTest(legacy_page_test.LegacyPageTest): 614 # Test *is* multi-tab because it overrides TabForPage. 615 616 def TabForPage(self, page, browser): 617 return browser.tabs.New() 618 619 def ValidateAndMeasurePage(self, *_): 620 pass 621 622 test = MultipleTabsTest() 623 with self.assertRaises(legacy_page_test.MultiTabTestAppCrashError): 624 self._RunPageTestThatRaisesAppCrashException(test, max_failures=1) 625 self.assertFormattedExceptionOnlyHas('AppCrashException') 626 627 def testWebPageReplay(self): 628 story_set = example_domain.ExampleDomainPageSet() 629 body = [] 630 631 class TestWpr(legacy_page_test.LegacyPageTest): 632 def ValidateAndMeasurePage(self, page, tab, results): 633 del page, results # unused 634 body.append(tab.EvaluateJavaScript('document.body.innerText')) 635 636 def DidRunPage(self, platform): 637 # Force the replay server to restart between pages; this verifies that 638 # the restart mechanism works. 639 platform.network_controller.StopReplay() 640 641 test = TestWpr() 642 options = options_for_unittests.GetCopy() 643 options.output_formats = ['none'] 644 options.suppress_gtest_report = True 645 SetUpStoryRunnerArguments(options) 646 results = results_options.CreateResults(EmptyMetadataForTest(), options) 647 648 story_runner.Run(test, story_set, options, results) 649 650 self.longMessage = True 651 self.assertIn('Example Domain', body[0], 652 msg='URL: %s' % story_set.stories[0].url) 653 self.assertIn('Example Domain', body[1], 654 msg='URL: %s' % story_set.stories[1].url) 655 656 self.assertEquals(2, len(GetSuccessfulPageRuns(results))) 657 self.assertEquals(0, len(results.failures)) 658 659 def testScreenShotTakenForFailedPage(self): 660 self.CaptureFormattedException() 661 platform_screenshot_supported = [False] 662 tab_screenshot_supported = [False] 663 chrome_version_screen_shot = [None] 664 665 class FailingTestPage(page_module.Page): 666 667 def RunNavigateSteps(self, action_runner): 668 action_runner.Navigate(self._url) 669 platform_screenshot_supported[0] = ( 670 action_runner.tab.browser.platform.CanTakeScreenshot) 671 tab_screenshot_supported[0] = action_runner.tab.screenshot_supported 672 if not platform_screenshot_supported[0] and tab_screenshot_supported[0]: 673 chrome_version_screen_shot[0] = action_runner.tab.Screenshot() 674 raise exceptions.AppCrashException 675 676 story_set = story.StorySet() 677 story_set.AddStory(page_module.Page('file://blank.html', story_set)) 678 failing_page = FailingTestPage('chrome://version', story_set) 679 story_set.AddStory(failing_page) 680 options = options_for_unittests.GetCopy() 681 options.output_formats = ['none'] 682 options.browser_options.take_screenshot_for_failed_page = True 683 options.suppress_gtest_report = True 684 SetUpStoryRunnerArguments(options) 685 results = results_options.CreateResults(EmptyMetadataForTest(), options) 686 story_runner.Run(DummyTest(), story_set, options, results, 687 max_failures=2) 688 self.assertEquals(1, len(results.failures)) 689 if not platform_screenshot_supported[0] and tab_screenshot_supported[0]: 690 self.assertEquals(1, len(results.pages_to_profiling_files)) 691 self.assertIn(failing_page, 692 results.pages_to_profiling_files) 693 screenshot_file_path = ( 694 results.pages_to_profiling_files[failing_page][0].GetAbsPath()) 695 try: 696 actual_screenshot = image_util.FromPngFile(screenshot_file_path) 697 self.assertEquals(image_util.Pixels(chrome_version_screen_shot[0]), 698 image_util.Pixels(actual_screenshot)) 699 finally: # Must clean up screenshot file if exists. 700 os.remove(screenshot_file_path) 701 702 def testNoProfilingFilesCreatedForPageByDefault(self): 703 self.CaptureFormattedException() 704 705 class FailingTestPage(page_module.Page): 706 707 def RunNavigateSteps(self, action_runner): 708 action_runner.Navigate(self._url) 709 raise exceptions.AppCrashException 710 711 story_set = story.StorySet() 712 story_set.AddStory(page_module.Page('file://blank.html', story_set)) 713 failing_page = FailingTestPage('chrome://version', story_set) 714 story_set.AddStory(failing_page) 715 options = options_for_unittests.GetCopy() 716 options.output_formats = ['none'] 717 options.suppress_gtest_report = True 718 SetUpStoryRunnerArguments(options) 719 results = results_options.CreateResults(EmptyMetadataForTest(), options) 720 story_runner.Run(DummyTest(), story_set, options, results, 721 max_failures=2) 722 self.assertEquals(1, len(results.failures)) 723 self.assertEquals(0, len(results.pages_to_profiling_files)) 724 725 726 class FakePageRunEndToEndTests(unittest.TestCase): 727 728 def setUp(self): 729 self.options = fakes.CreateBrowserFinderOptions() 730 self.options.output_formats = ['none'] 731 self.options.suppress_gtest_report = True 732 SetUpStoryRunnerArguments(self.options) 733 734 def testNoScreenShotTakenForFailedPageDueToNoSupport(self): 735 self.options.browser_options.take_screenshot_for_failed_page = True 736 737 class FailingTestPage(page_module.Page): 738 739 def RunNavigateSteps(self, action_runner): 740 raise exceptions.AppCrashException 741 742 story_set = story.StorySet() 743 story_set.AddStory(page_module.Page('file://blank.html', story_set)) 744 failing_page = FailingTestPage('chrome://version', story_set) 745 story_set.AddStory(failing_page) 746 results = results_options.CreateResults( 747 EmptyMetadataForTest(), self.options) 748 story_runner.Run(DummyTest(), story_set, self.options, results, 749 max_failures=2) 750 self.assertEquals(1, len(results.failures)) 751 self.assertEquals(0, len(results.pages_to_profiling_files)) 752 753 def testScreenShotTakenForFailedPageOnSupportedPlatform(self): 754 fake_platform = self.options.fake_possible_browser.returned_browser.platform 755 expected_png_base64 = """ 756 iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91 757 JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A 758 T8tgwbJAAAAABJRU5ErkJggg== 759 """ 760 fake_platform.screenshot_png_data = expected_png_base64 761 self.options.browser_options.take_screenshot_for_failed_page = True 762 763 class FailingTestPage(page_module.Page): 764 765 def RunNavigateSteps(self, action_runner): 766 raise exceptions.AppCrashException 767 story_set = story.StorySet() 768 story_set.AddStory(page_module.Page('file://blank.html', story_set)) 769 failing_page = FailingTestPage('chrome://version', story_set) 770 story_set.AddStory(failing_page) 771 772 results = results_options.CreateResults( 773 EmptyMetadataForTest(), self.options) 774 story_runner.Run(DummyTest(), story_set, self.options, results, 775 max_failures=2) 776 self.assertEquals(1, len(results.failures)) 777 self.assertEquals(1, len(results.pages_to_profiling_files)) 778 self.assertIn(failing_page, 779 results.pages_to_profiling_files) 780 screenshot_file_path = ( 781 results.pages_to_profiling_files[failing_page][0].GetAbsPath()) 782 try: 783 actual_screenshot_img = image_util.FromPngFile(screenshot_file_path) 784 self.assertTrue(image_util.AreEqual( 785 image_util.FromBase64Png(expected_png_base64), 786 actual_screenshot_img)) 787 finally: # Must clean up screenshot file if exists. 788 os.remove(screenshot_file_path) 789