1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import math 6 import os 7 import StringIO 8 import sys 9 import unittest 10 11 from py_utils import cloud_storage # pylint: disable=import-error 12 13 from telemetry import benchmark 14 from telemetry.core import exceptions 15 from telemetry.core import util 16 from telemetry import decorators 17 from telemetry.internal.actions import page_action 18 from telemetry.internal.results import page_test_results 19 from telemetry.internal.results import results_options 20 from telemetry.internal import story_runner 21 from telemetry.internal.util import exception_formatter as ex_formatter_module 22 from telemetry.page import page as page_module 23 from telemetry.page import legacy_page_test 24 from telemetry import story as story_module 25 from telemetry.testing import options_for_unittests 26 from telemetry.testing import system_stub 27 import mock 28 from telemetry.value import failure 29 from telemetry.value import improvement_direction 30 from telemetry.value import list_of_scalar_values 31 from telemetry.value import scalar 32 from telemetry.value import skip 33 from telemetry.value import summary as summary_module 34 from telemetry.web_perf import story_test 35 from telemetry.web_perf import timeline_based_measurement 36 from telemetry.wpr import archive_info 37 38 # This linter complains if we define classes nested inside functions. 39 # pylint: disable=bad-super-call 40 41 42 class FakePlatform(object): 43 def CanMonitorThermalThrottling(self): 44 return False 45 46 47 class TestSharedState(story_module.SharedState): 48 49 _platform = FakePlatform() 50 51 @classmethod 52 def SetTestPlatform(cls, platform): 53 cls._platform = platform 54 55 def __init__(self, test, options, story_set): 56 super(TestSharedState, self).__init__( 57 test, options, story_set) 58 self._test = test 59 self._current_story = None 60 61 @property 62 def platform(self): 63 return self._platform 64 65 def WillRunStory(self, story): 66 self._current_story = story 67 68 def CanRunStory(self, story): 69 return True 70 71 def RunStory(self, results): 72 raise NotImplementedError 73 74 def DidRunStory(self, results): 75 pass 76 77 def TearDownState(self): 78 pass 79 80 def DumpStateUponFailure(self, story, results): 81 pass 82 83 84 class TestSharedPageState(TestSharedState): 85 def RunStory(self, results): 86 self._test.RunPage(self._current_story, None, results) 87 88 89 class FooStoryState(TestSharedPageState): 90 pass 91 92 93 class BarStoryState(TestSharedPageState): 94 pass 95 96 97 class DummyTest(legacy_page_test.LegacyPageTest): 98 def RunPage(self, *_): 99 pass 100 101 def ValidateAndMeasurePage(self, page, tab, results): 102 pass 103 104 105 class EmptyMetadataForTest(benchmark.BenchmarkMetadata): 106 def __init__(self): 107 super(EmptyMetadataForTest, self).__init__('') 108 109 110 class DummyLocalStory(story_module.Story): 111 def __init__(self, shared_state_class, name=''): 112 super(DummyLocalStory, self).__init__( 113 shared_state_class, name=name) 114 115 def Run(self, shared_state): 116 pass 117 118 @property 119 def is_local(self): 120 return True 121 122 @property 123 def url(self): 124 return 'data:,' 125 126 127 class MixedStateStorySet(story_module.StorySet): 128 @property 129 def allow_mixed_story_states(self): 130 return True 131 132 def SetupStorySet(allow_multiple_story_states, story_state_list): 133 if allow_multiple_story_states: 134 story_set = MixedStateStorySet() 135 else: 136 story_set = story_module.StorySet() 137 for story_state in story_state_list: 138 story_set.AddStory(DummyLocalStory(story_state)) 139 return story_set 140 141 def _GetOptionForUnittest(): 142 options = options_for_unittests.GetCopy() 143 options.output_formats = ['none'] 144 options.suppress_gtest_report = False 145 parser = options.CreateParser() 146 story_runner.AddCommandLineArgs(parser) 147 options.MergeDefaultValues(parser.get_default_values()) 148 story_runner.ProcessCommandLineArgs(parser, options) 149 return options 150 151 152 class FakeExceptionFormatterModule(object): 153 @staticmethod 154 def PrintFormattedException( 155 exception_class=None, exception=None, tb=None, msg=None): 156 pass 157 158 159 def GetNumberOfSuccessfulPageRuns(results): 160 return len([run for run in results.all_page_runs if run.ok or run.skipped]) 161 162 163 class TestOnlyException(Exception): 164 pass 165 166 167 class FailureValueMatcher(object): 168 169 def __init__(self, expected_exception_message): 170 self._expected_exception_message = expected_exception_message 171 172 def __eq__(self, other): 173 return (isinstance(other, failure.FailureValue) and 174 other.exc_info[1].message == self._expected_exception_message) 175 176 177 class SkipValueMatcher(object): 178 179 def __eq__(self, other): 180 return isinstance(other, skip.SkipValue) 181 182 183 class StoryRunnerTest(unittest.TestCase): 184 185 def setUp(self): 186 self.fake_stdout = StringIO.StringIO() 187 self.actual_stdout = sys.stdout 188 sys.stdout = self.fake_stdout 189 self.options = _GetOptionForUnittest() 190 self.results = results_options.CreateResults( 191 EmptyMetadataForTest(), self.options) 192 self._story_runner_logging_stub = None 193 194 def SuppressExceptionFormatting(self): 195 """Fake out exception formatter to avoid spamming the unittest stdout.""" 196 story_runner.exception_formatter = FakeExceptionFormatterModule 197 self._story_runner_logging_stub = system_stub.Override( 198 story_runner, ['logging']) 199 200 def RestoreExceptionFormatter(self): 201 story_runner.exception_formatter = ex_formatter_module 202 if self._story_runner_logging_stub: 203 self._story_runner_logging_stub.Restore() 204 self._story_runner_logging_stub = None 205 206 def tearDown(self): 207 sys.stdout = self.actual_stdout 208 self.RestoreExceptionFormatter() 209 210 def testStoriesGroupedByStateClass(self): 211 foo_states = [FooStoryState, FooStoryState, FooStoryState, 212 FooStoryState, FooStoryState] 213 mixed_states = [FooStoryState, FooStoryState, FooStoryState, 214 BarStoryState, FooStoryState] 215 # StorySet's are only allowed to have one SharedState. 216 story_set = SetupStorySet(False, foo_states) 217 story_groups = ( 218 story_runner.StoriesGroupedByStateClass( 219 story_set, False)) 220 self.assertEqual(len(story_groups), 1) 221 story_set = SetupStorySet(False, mixed_states) 222 self.assertRaises( 223 ValueError, 224 story_runner.StoriesGroupedByStateClass, 225 story_set, False) 226 # BaseStorySets are allowed to have multiple SharedStates. 227 mixed_story_set = SetupStorySet(True, mixed_states) 228 story_groups = ( 229 story_runner.StoriesGroupedByStateClass( 230 mixed_story_set, True)) 231 self.assertEqual(len(story_groups), 3) 232 self.assertEqual(story_groups[0].shared_state_class, 233 FooStoryState) 234 self.assertEqual(story_groups[1].shared_state_class, 235 BarStoryState) 236 self.assertEqual(story_groups[2].shared_state_class, 237 FooStoryState) 238 239 def RunStoryTest(self, s, expected_successes): 240 test = DummyTest() 241 story_runner.Run( 242 test, s, self.options, self.results) 243 self.assertEquals(0, len(self.results.failures)) 244 self.assertEquals(expected_successes, 245 GetNumberOfSuccessfulPageRuns(self.results)) 246 247 def testRunStoryWithMissingArchiveFile(self): 248 story_set = story_module.StorySet(archive_data_file='data/hi.json') 249 story_set.AddStory(page_module.Page( 250 'http://www.testurl.com', story_set, story_set.base_dir)) 251 test = DummyTest() 252 self.assertRaises(story_runner.ArchiveError, story_runner.Run, test, 253 story_set, self.options, self.results) 254 255 def testStoryTest(self): 256 all_foo = [FooStoryState, FooStoryState, FooStoryState] 257 one_bar = [FooStoryState, FooStoryState, BarStoryState] 258 story_set = SetupStorySet(True, one_bar) 259 self.RunStoryTest(story_set, 3) 260 story_set = SetupStorySet(True, all_foo) 261 self.RunStoryTest(story_set, 6) 262 story_set = SetupStorySet(False, all_foo) 263 self.RunStoryTest(story_set, 9) 264 story_set = SetupStorySet(False, one_bar) 265 test = DummyTest() 266 self.assertRaises(ValueError, story_runner.Run, test, story_set, 267 self.options, self.results) 268 269 def testSuccessfulTimelineBasedMeasurementTest(self): 270 """Check that PageTest is not required for story_runner.Run. 271 272 Any PageTest related calls or attributes need to only be called 273 for PageTest tests. 274 """ 275 class TestSharedTbmState(TestSharedState): 276 def RunStory(self, results): 277 pass 278 279 TEST_WILL_RUN_STORY = 'test.WillRunStory' 280 TEST_MEASURE = 'test.Measure' 281 TEST_DID_RUN_STORY = 'test.DidRunStory' 282 283 EXPECTED_CALLS_IN_ORDER = [TEST_WILL_RUN_STORY, 284 TEST_MEASURE, 285 TEST_DID_RUN_STORY] 286 287 test = timeline_based_measurement.TimelineBasedMeasurement( 288 timeline_based_measurement.Options()) 289 290 manager = mock.MagicMock() 291 test.WillRunStory = mock.MagicMock() 292 test.Measure = mock.MagicMock() 293 test.DidRunStory = mock.MagicMock() 294 manager.attach_mock(test.WillRunStory, TEST_WILL_RUN_STORY) 295 manager.attach_mock(test.Measure, TEST_MEASURE) 296 manager.attach_mock(test.DidRunStory, TEST_DID_RUN_STORY) 297 298 story_set = story_module.StorySet() 299 story_set.AddStory(DummyLocalStory(TestSharedTbmState)) 300 story_set.AddStory(DummyLocalStory(TestSharedTbmState)) 301 story_set.AddStory(DummyLocalStory(TestSharedTbmState)) 302 story_runner.Run( 303 test, story_set, self.options, self.results) 304 self.assertEquals(0, len(self.results.failures)) 305 self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results)) 306 307 self.assertEquals(3*EXPECTED_CALLS_IN_ORDER, 308 [call[0] for call in manager.mock_calls]) 309 310 def testCallOrderBetweenStoryTestAndSharedState(self): 311 """Check that the call order between StoryTest and SharedState is correct. 312 """ 313 TEST_WILL_RUN_STORY = 'test.WillRunStory' 314 TEST_MEASURE = 'test.Measure' 315 TEST_DID_RUN_STORY = 'test.DidRunStory' 316 STATE_WILL_RUN_STORY = 'state.WillRunStory' 317 STATE_RUN_STORY = 'state.RunStory' 318 STATE_DID_RUN_STORY = 'state.DidRunStory' 319 320 EXPECTED_CALLS_IN_ORDER = [TEST_WILL_RUN_STORY, 321 STATE_WILL_RUN_STORY, 322 STATE_RUN_STORY, 323 TEST_MEASURE, 324 STATE_DID_RUN_STORY, 325 TEST_DID_RUN_STORY] 326 327 class TestStoryTest(story_test.StoryTest): 328 def WillRunStory(self, platform): 329 pass 330 331 def Measure(self, platform, results): 332 pass 333 334 def DidRunStory(self, platform): 335 pass 336 337 class TestSharedStateForStoryTest(TestSharedState): 338 def RunStory(self, results): 339 pass 340 341 @mock.patch.object(TestStoryTest, 'WillRunStory') 342 @mock.patch.object(TestStoryTest, 'Measure') 343 @mock.patch.object(TestStoryTest, 'DidRunStory') 344 @mock.patch.object(TestSharedStateForStoryTest, 'WillRunStory') 345 @mock.patch.object(TestSharedStateForStoryTest, 'RunStory') 346 @mock.patch.object(TestSharedStateForStoryTest, 'DidRunStory') 347 def GetCallsInOrder(state_DidRunStory, state_RunStory, state_WillRunStory, 348 test_DidRunStory, test_Measure, test_WillRunStory): 349 manager = mock.MagicMock() 350 manager.attach_mock(test_WillRunStory, TEST_WILL_RUN_STORY) 351 manager.attach_mock(test_Measure, TEST_MEASURE) 352 manager.attach_mock(test_DidRunStory, TEST_DID_RUN_STORY) 353 manager.attach_mock(state_WillRunStory, STATE_WILL_RUN_STORY) 354 manager.attach_mock(state_RunStory, STATE_RUN_STORY) 355 manager.attach_mock(state_DidRunStory, STATE_DID_RUN_STORY) 356 357 test = TestStoryTest() 358 story_set = story_module.StorySet() 359 story_set.AddStory(DummyLocalStory(TestSharedStateForStoryTest)) 360 story_runner.Run(test, story_set, self.options, self.results) 361 return [call[0] for call in manager.mock_calls] 362 363 calls_in_order = GetCallsInOrder() # pylint: disable=no-value-for-parameter 364 self.assertEquals(EXPECTED_CALLS_IN_ORDER, calls_in_order) 365 366 def testTearDownStateAfterEachStoryOrStorySetRun(self): 367 class TestSharedStateForTearDown(TestSharedState): 368 num_of_tear_downs = 0 369 370 def RunStory(self, results): 371 pass 372 373 def TearDownState(self): 374 TestSharedStateForTearDown.num_of_tear_downs += 1 375 376 story_set = story_module.StorySet() 377 story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown)) 378 story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown)) 379 story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown)) 380 381 TestSharedStateForTearDown.num_of_tear_downs = 0 382 story_runner.Run(mock.MagicMock(), story_set, self.options, self.results) 383 self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 1) 384 385 TestSharedStateForTearDown.num_of_tear_downs = 0 386 story_runner.Run(mock.MagicMock(), story_set, self.options, self.results, 387 tear_down_after_story=True) 388 self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 3) 389 390 self.options.pageset_repeat = 5 391 TestSharedStateForTearDown.num_of_tear_downs = 0 392 story_runner.Run(mock.MagicMock(), story_set, self.options, self.results, 393 tear_down_after_story_set=True) 394 self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 5) 395 396 def testTearDownIsCalledOnceForEachStoryGroupWithPageSetRepeat(self): 397 self.options.pageset_repeat = 3 398 fooz_init_call_counter = [0] 399 fooz_tear_down_call_counter = [0] 400 barz_init_call_counter = [0] 401 barz_tear_down_call_counter = [0] 402 class FoozStoryState(FooStoryState): 403 def __init__(self, test, options, storyz): 404 super(FoozStoryState, self).__init__( 405 test, options, storyz) 406 fooz_init_call_counter[0] += 1 407 def TearDownState(self): 408 fooz_tear_down_call_counter[0] += 1 409 410 class BarzStoryState(BarStoryState): 411 def __init__(self, test, options, storyz): 412 super(BarzStoryState, self).__init__( 413 test, options, storyz) 414 barz_init_call_counter[0] += 1 415 def TearDownState(self): 416 barz_tear_down_call_counter[0] += 1 417 def AssertAndCleanUpFoo(): 418 self.assertEquals(1, fooz_init_call_counter[0]) 419 self.assertEquals(1, fooz_tear_down_call_counter[0]) 420 fooz_init_call_counter[0] = 0 421 fooz_tear_down_call_counter[0] = 0 422 423 story_set1_list = [FoozStoryState, FoozStoryState, FoozStoryState, 424 BarzStoryState, BarzStoryState] 425 story_set1 = SetupStorySet(True, story_set1_list) 426 self.RunStoryTest(story_set1, 15) 427 AssertAndCleanUpFoo() 428 self.assertEquals(1, barz_init_call_counter[0]) 429 self.assertEquals(1, barz_tear_down_call_counter[0]) 430 barz_init_call_counter[0] = 0 431 barz_tear_down_call_counter[0] = 0 432 433 story_set2_list = [FoozStoryState, FoozStoryState, FoozStoryState, 434 FoozStoryState] 435 story_set2 = SetupStorySet(False, story_set2_list) 436 self.RunStoryTest(story_set2, 27) 437 AssertAndCleanUpFoo() 438 self.assertEquals(0, barz_init_call_counter[0]) 439 self.assertEquals(0, barz_tear_down_call_counter[0]) 440 441 def testAppCrashExceptionCausesFailureValue(self): 442 self.SuppressExceptionFormatting() 443 story_set = story_module.StorySet() 444 class SharedStoryThatCausesAppCrash(TestSharedPageState): 445 def WillRunStory(self, story): 446 raise exceptions.AppCrashException(msg='App Foo crashes') 447 448 story_set.AddStory(DummyLocalStory( 449 SharedStoryThatCausesAppCrash)) 450 story_runner.Run( 451 DummyTest(), story_set, self.options, self.results) 452 self.assertEquals(1, len(self.results.failures)) 453 self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results)) 454 self.assertIn('App Foo crashes', self.fake_stdout.getvalue()) 455 456 def testExceptionRaisedInSharedStateTearDown(self): 457 self.SuppressExceptionFormatting() 458 story_set = story_module.StorySet() 459 class SharedStoryThatCausesAppCrash(TestSharedPageState): 460 def TearDownState(self): 461 raise TestOnlyException() 462 463 story_set.AddStory(DummyLocalStory( 464 SharedStoryThatCausesAppCrash)) 465 with self.assertRaises(TestOnlyException): 466 story_runner.Run( 467 DummyTest(), story_set, self.options, self.results) 468 469 def testUnknownExceptionIsFatal(self): 470 self.SuppressExceptionFormatting() 471 story_set = story_module.StorySet() 472 473 class UnknownException(Exception): 474 pass 475 476 # This erroneous test is set up to raise exception for the 2nd story 477 # run. 478 class Test(legacy_page_test.LegacyPageTest): 479 def __init__(self, *args): 480 super(Test, self).__init__(*args) 481 self.run_count = 0 482 483 def RunPage(self, *_): 484 old_run_count = self.run_count 485 self.run_count += 1 486 if old_run_count == 1: 487 raise UnknownException('FooBarzException') 488 489 def ValidateAndMeasurePage(self, page, tab, results): 490 pass 491 492 s1 = DummyLocalStory(TestSharedPageState) 493 s2 = DummyLocalStory(TestSharedPageState) 494 story_set.AddStory(s1) 495 story_set.AddStory(s2) 496 test = Test() 497 with self.assertRaises(UnknownException): 498 story_runner.Run( 499 test, story_set, self.options, self.results) 500 self.assertEqual(set([s2]), self.results.pages_that_failed) 501 self.assertEqual(set([s1]), self.results.pages_that_succeeded) 502 self.assertIn('FooBarzException', self.fake_stdout.getvalue()) 503 504 def testRaiseBrowserGoneExceptionFromRunPage(self): 505 self.SuppressExceptionFormatting() 506 story_set = story_module.StorySet() 507 508 class Test(legacy_page_test.LegacyPageTest): 509 def __init__(self, *args): 510 super(Test, self).__init__(*args) 511 self.run_count = 0 512 513 def RunPage(self, *_): 514 old_run_count = self.run_count 515 self.run_count += 1 516 if old_run_count == 0: 517 raise exceptions.BrowserGoneException( 518 None, 'i am a browser crash message') 519 520 def ValidateAndMeasurePage(self, page, tab, results): 521 pass 522 523 story_set.AddStory(DummyLocalStory(TestSharedPageState)) 524 story_set.AddStory(DummyLocalStory(TestSharedPageState)) 525 test = Test() 526 story_runner.Run( 527 test, story_set, self.options, self.results) 528 self.assertEquals(2, test.run_count) 529 self.assertEquals(1, len(self.results.failures)) 530 self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results)) 531 532 def testAppCrashThenRaiseInTearDownFatal(self): 533 self.SuppressExceptionFormatting() 534 story_set = story_module.StorySet() 535 536 unit_test_events = [] # track what was called when 537 class DidRunTestError(Exception): 538 pass 539 540 class TestTearDownSharedState(TestSharedPageState): 541 def TearDownState(self): 542 unit_test_events.append('tear-down-state') 543 raise DidRunTestError 544 545 def DumpStateUponFailure(self, story, results): 546 unit_test_events.append('dump-state') 547 548 549 class Test(legacy_page_test.LegacyPageTest): 550 def __init__(self, *args): 551 super(Test, self).__init__(*args) 552 self.run_count = 0 553 554 def RunPage(self, *_): 555 old_run_count = self.run_count 556 self.run_count += 1 557 if old_run_count == 0: 558 unit_test_events.append('app-crash') 559 raise exceptions.AppCrashException 560 561 def ValidateAndMeasurePage(self, page, tab, results): 562 pass 563 564 story_set.AddStory(DummyLocalStory(TestTearDownSharedState)) 565 story_set.AddStory(DummyLocalStory(TestTearDownSharedState)) 566 test = Test() 567 568 with self.assertRaises(DidRunTestError): 569 story_runner.Run( 570 test, story_set, self.options, self.results) 571 self.assertEqual(['app-crash', 'dump-state', 'tear-down-state'], 572 unit_test_events) 573 # The AppCrashException gets added as a failure. 574 self.assertEquals(1, len(self.results.failures)) 575 576 def testPagesetRepeat(self): 577 story_set = story_module.StorySet() 578 579 # TODO(eakuefner): Factor this out after flattening page ref in Value 580 blank_story = DummyLocalStory(TestSharedPageState, name='blank') 581 green_story = DummyLocalStory(TestSharedPageState, name='green') 582 story_set.AddStory(blank_story) 583 story_set.AddStory(green_story) 584 585 class Measurement(legacy_page_test.LegacyPageTest): 586 i = 0 587 def RunPage(self, page, _, results): 588 self.i += 1 589 results.AddValue(scalar.ScalarValue( 590 page, 'metric', 'unit', self.i, 591 improvement_direction=improvement_direction.UP)) 592 593 def ValidateAndMeasurePage(self, page, tab, results): 594 pass 595 596 self.options.page_repeat = 1 597 self.options.pageset_repeat = 2 598 self.options.output_formats = [] 599 results = results_options.CreateResults( 600 EmptyMetadataForTest(), self.options) 601 story_runner.Run( 602 Measurement(), story_set, self.options, results) 603 summary = summary_module.Summary(results.all_page_specific_values) 604 values = summary.interleaved_computed_per_page_values_and_summaries 605 606 blank_value = list_of_scalar_values.ListOfScalarValues( 607 blank_story, 'metric', 'unit', [1, 3], 608 improvement_direction=improvement_direction.UP) 609 green_value = list_of_scalar_values.ListOfScalarValues( 610 green_story, 'metric', 'unit', [2, 4], 611 improvement_direction=improvement_direction.UP) 612 merged_value = list_of_scalar_values.ListOfScalarValues( 613 None, 'metric', 'unit', 614 [1, 3, 2, 4], std=math.sqrt(2), # Pooled standard deviation. 615 improvement_direction=improvement_direction.UP) 616 617 self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results)) 618 self.assertEquals(0, len(results.failures)) 619 self.assertEquals(3, len(values)) 620 self.assertIn(blank_value, values) 621 self.assertIn(green_value, values) 622 self.assertIn(merged_value, values) 623 624 @decorators.Disabled('chromeos') # crbug.com/483212 625 def testUpdateAndCheckArchives(self): 626 usr_stub = system_stub.Override(story_runner, ['cloud_storage']) 627 wpr_stub = system_stub.Override(archive_info, ['cloud_storage']) 628 archive_data_dir = os.path.join( 629 util.GetTelemetryDir(), 630 'telemetry', 'internal', 'testing', 'archive_files') 631 try: 632 story_set = story_module.StorySet() 633 story_set.AddStory(page_module.Page( 634 'http://www.testurl.com', story_set, story_set.base_dir)) 635 # Page set missing archive_data_file. 636 self.assertRaises( 637 story_runner.ArchiveError, 638 story_runner._UpdateAndCheckArchives, 639 story_set.archive_data_file, 640 story_set.wpr_archive_info, 641 story_set.stories) 642 643 story_set = story_module.StorySet( 644 archive_data_file='missing_archive_data_file.json') 645 story_set.AddStory(page_module.Page( 646 'http://www.testurl.com', story_set, story_set.base_dir)) 647 # Page set missing json file specified in archive_data_file. 648 self.assertRaises( 649 story_runner.ArchiveError, 650 story_runner._UpdateAndCheckArchives, 651 story_set.archive_data_file, 652 story_set.wpr_archive_info, 653 story_set.stories) 654 655 story_set = story_module.StorySet( 656 archive_data_file=os.path.join(archive_data_dir, 'test.json'), 657 cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET) 658 story_set.AddStory(page_module.Page( 659 'http://www.testurl.com', story_set, story_set.base_dir)) 660 # Page set with valid archive_data_file. 661 self.assertTrue(story_runner._UpdateAndCheckArchives( 662 story_set.archive_data_file, story_set.wpr_archive_info, 663 story_set.stories)) 664 story_set.AddStory(page_module.Page( 665 'http://www.google.com', story_set, story_set.base_dir)) 666 # Page set with an archive_data_file which exists but is missing a page. 667 self.assertRaises( 668 story_runner.ArchiveError, 669 story_runner._UpdateAndCheckArchives, 670 story_set.archive_data_file, 671 story_set.wpr_archive_info, 672 story_set.stories) 673 674 story_set = story_module.StorySet( 675 archive_data_file= 676 os.path.join(archive_data_dir, 'test_missing_wpr_file.json'), 677 cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET) 678 story_set.AddStory(page_module.Page( 679 'http://www.testurl.com', story_set, story_set.base_dir)) 680 story_set.AddStory(page_module.Page( 681 'http://www.google.com', story_set, story_set.base_dir)) 682 # Page set with an archive_data_file which exists and contains all pages 683 # but fails to find a wpr file. 684 self.assertRaises( 685 story_runner.ArchiveError, 686 story_runner._UpdateAndCheckArchives, 687 story_set.archive_data_file, 688 story_set.wpr_archive_info, 689 story_set.stories) 690 finally: 691 usr_stub.Restore() 692 wpr_stub.Restore() 693 694 695 def _testMaxFailuresOptionIsRespectedAndOverridable( 696 self, num_failing_stories, runner_max_failures, options_max_failures, 697 expected_num_failures): 698 class SimpleSharedState(story_module.SharedState): 699 _fake_platform = FakePlatform() 700 _current_story = None 701 702 @property 703 def platform(self): 704 return self._fake_platform 705 706 def WillRunStory(self, story): 707 self._current_story = story 708 709 def RunStory(self, results): 710 self._current_story.Run(self) 711 712 def DidRunStory(self, results): 713 pass 714 715 def CanRunStory(self, story): 716 return True 717 718 def TearDownState(self): 719 pass 720 721 def DumpStateUponFailure(self, story, results): 722 pass 723 724 class FailingStory(story_module.Story): 725 def __init__(self): 726 super(FailingStory, self).__init__( 727 shared_state_class=SimpleSharedState, 728 is_local=True) 729 self.was_run = False 730 731 def Run(self, shared_state): 732 self.was_run = True 733 raise legacy_page_test.Failure 734 735 @property 736 def url(self): 737 return 'data:,' 738 739 self.SuppressExceptionFormatting() 740 741 story_set = story_module.StorySet() 742 for _ in range(num_failing_stories): 743 story_set.AddStory(FailingStory()) 744 745 options = _GetOptionForUnittest() 746 options.output_formats = ['none'] 747 options.suppress_gtest_report = True 748 if options_max_failures: 749 options.max_failures = options_max_failures 750 751 results = results_options.CreateResults(EmptyMetadataForTest(), options) 752 story_runner.Run( 753 DummyTest(), story_set, options, 754 results, max_failures=runner_max_failures) 755 self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) 756 self.assertEquals(expected_num_failures, len(results.failures)) 757 for ii, story in enumerate(story_set.stories): 758 self.assertEqual(story.was_run, ii < expected_num_failures) 759 760 def testMaxFailuresNotSpecified(self): 761 self._testMaxFailuresOptionIsRespectedAndOverridable( 762 num_failing_stories=5, runner_max_failures=None, 763 options_max_failures=None, expected_num_failures=5) 764 765 def testMaxFailuresSpecifiedToRun(self): 766 # Runs up to max_failures+1 failing tests before stopping, since 767 # every tests after max_failures failures have been encountered 768 # may all be passing. 769 self._testMaxFailuresOptionIsRespectedAndOverridable( 770 num_failing_stories=5, runner_max_failures=3, 771 options_max_failures=None, expected_num_failures=4) 772 773 def testMaxFailuresOption(self): 774 # Runs up to max_failures+1 failing tests before stopping, since 775 # every tests after max_failures failures have been encountered 776 # may all be passing. 777 self._testMaxFailuresOptionIsRespectedAndOverridable( 778 num_failing_stories=5, runner_max_failures=3, 779 options_max_failures=1, expected_num_failures=2) 780 781 def _CreateErrorProcessingMock(self, method_exceptions=None, 782 legacy_test=False): 783 if legacy_test: 784 test_class = legacy_page_test.LegacyPageTest 785 else: 786 test_class = story_test.StoryTest 787 788 root_mock = mock.NonCallableMock( 789 story=mock.NonCallableMagicMock(story_module.Story), 790 results=mock.NonCallableMagicMock(page_test_results.PageTestResults), 791 test=mock.NonCallableMagicMock(test_class), 792 state=mock.NonCallableMagicMock( 793 story_module.SharedState, 794 CanRunStory=mock.Mock(return_value=True))) 795 796 if method_exceptions: 797 root_mock.configure_mock(**{ 798 path + '.side_effect': exception 799 for path, exception in method_exceptions.iteritems()}) 800 801 return root_mock 802 803 def testRunStoryAndProcessErrorIfNeeded_success(self): 804 root_mock = self._CreateErrorProcessingMock() 805 806 story_runner._RunStoryAndProcessErrorIfNeeded( 807 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 808 809 self.assertEquals(root_mock.method_calls, [ 810 mock.call.test.WillRunStory(root_mock.state.platform), 811 mock.call.state.WillRunStory(root_mock.story), 812 mock.call.state.CanRunStory(root_mock.story), 813 mock.call.state.RunStory(root_mock.results), 814 mock.call.test.Measure(root_mock.state.platform, root_mock.results), 815 mock.call.state.DidRunStory(root_mock.results), 816 mock.call.test.DidRunStory(root_mock.state.platform) 817 ]) 818 819 def testRunStoryAndProcessErrorIfNeeded_successLegacy(self): 820 root_mock = self._CreateErrorProcessingMock(legacy_test=True) 821 822 story_runner._RunStoryAndProcessErrorIfNeeded( 823 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 824 825 self.assertEquals(root_mock.method_calls, [ 826 mock.call.state.WillRunStory(root_mock.story), 827 mock.call.state.CanRunStory(root_mock.story), 828 mock.call.state.RunStory(root_mock.results), 829 mock.call.state.DidRunStory(root_mock.results), 830 mock.call.test.DidRunPage(root_mock.state.platform) 831 ]) 832 833 def testRunStoryAndProcessErrorIfNeeded_tryTimeout(self): 834 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 835 'state.WillRunStory': exceptions.TimeoutException('foo') 836 }) 837 838 story_runner._RunStoryAndProcessErrorIfNeeded( 839 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 840 841 self.assertEquals(root_mock.method_calls, [ 842 mock.call.test.WillRunStory(root_mock.state.platform), 843 mock.call.state.WillRunStory(root_mock.story), 844 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results), 845 mock.call.results.AddValue(FailureValueMatcher('foo')), 846 mock.call.state.DidRunStory(root_mock.results), 847 mock.call.test.DidRunStory(root_mock.state.platform) 848 ]) 849 850 def testRunStoryAndProcessErrorIfNeeded_tryError(self): 851 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 852 'state.CanRunStory': exceptions.Error('foo') 853 }) 854 855 with self.assertRaisesRegexp(exceptions.Error, 'foo'): 856 story_runner._RunStoryAndProcessErrorIfNeeded( 857 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 858 859 self.assertEquals(root_mock.method_calls, [ 860 mock.call.test.WillRunStory(root_mock.state.platform), 861 mock.call.state.WillRunStory(root_mock.story), 862 mock.call.state.CanRunStory(root_mock.story), 863 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results), 864 mock.call.results.AddValue(FailureValueMatcher('foo')), 865 mock.call.state.DidRunStory(root_mock.results), 866 mock.call.test.DidRunStory(root_mock.state.platform) 867 ]) 868 869 def testRunStoryAndProcessErrorIfNeeded_tryUnsupportedAction(self): 870 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 871 'state.RunStory': page_action.PageActionNotSupported('foo') 872 }) 873 874 story_runner._RunStoryAndProcessErrorIfNeeded( 875 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 876 877 self.assertEquals(root_mock.method_calls, [ 878 mock.call.test.WillRunStory(root_mock.state.platform), 879 mock.call.state.WillRunStory(root_mock.story), 880 mock.call.state.CanRunStory(root_mock.story), 881 mock.call.state.RunStory(root_mock.results), 882 mock.call.results.AddValue(SkipValueMatcher()), 883 mock.call.state.DidRunStory(root_mock.results), 884 mock.call.test.DidRunStory(root_mock.state.platform) 885 ]) 886 887 def testRunStoryAndProcessErrorIfNeeded_tryUnhandlable(self): 888 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 889 'test.WillRunStory': Exception('foo') 890 }) 891 892 with self.assertRaisesRegexp(Exception, 'foo'): 893 story_runner._RunStoryAndProcessErrorIfNeeded( 894 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 895 896 self.assertEquals(root_mock.method_calls, [ 897 mock.call.test.WillRunStory(root_mock.state.platform), 898 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results), 899 mock.call.results.AddValue(FailureValueMatcher('foo')), 900 mock.call.state.DidRunStory(root_mock.results), 901 mock.call.test.DidRunStory(root_mock.state.platform) 902 ]) 903 904 def testRunStoryAndProcessErrorIfNeeded_finallyException(self): 905 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 906 'state.DidRunStory': Exception('bar') 907 }) 908 909 with self.assertRaisesRegexp(Exception, 'bar'): 910 story_runner._RunStoryAndProcessErrorIfNeeded( 911 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 912 913 self.assertEquals(root_mock.method_calls, [ 914 mock.call.test.WillRunStory(root_mock.state.platform), 915 mock.call.state.WillRunStory(root_mock.story), 916 mock.call.state.CanRunStory(root_mock.story), 917 mock.call.state.RunStory(root_mock.results), 918 mock.call.test.Measure(root_mock.state.platform, root_mock.results), 919 mock.call.state.DidRunStory(root_mock.results), 920 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results) 921 ]) 922 923 def testRunStoryAndProcessErrorIfNeeded_tryTimeout_finallyException(self): 924 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 925 'state.RunStory': exceptions.TimeoutException('foo'), 926 'state.DidRunStory': Exception('bar') 927 }) 928 929 story_runner._RunStoryAndProcessErrorIfNeeded( 930 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 931 932 self.assertEquals(root_mock.method_calls, [ 933 mock.call.test.WillRunStory(root_mock.state.platform), 934 mock.call.state.WillRunStory(root_mock.story), 935 mock.call.state.CanRunStory(root_mock.story), 936 mock.call.state.RunStory(root_mock.results), 937 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results), 938 mock.call.results.AddValue(FailureValueMatcher('foo')), 939 mock.call.state.DidRunStory(root_mock.results) 940 ]) 941 942 def testRunStoryAndProcessErrorIfNeeded_tryError_finallyException(self): 943 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 944 'state.WillRunStory': exceptions.Error('foo'), 945 'test.DidRunStory': Exception('bar') 946 }) 947 948 with self.assertRaisesRegexp(exceptions.Error, 'foo'): 949 story_runner._RunStoryAndProcessErrorIfNeeded( 950 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 951 952 self.assertEquals(root_mock.method_calls, [ 953 mock.call.test.WillRunStory(root_mock.state.platform), 954 mock.call.state.WillRunStory(root_mock.story), 955 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results), 956 mock.call.results.AddValue(FailureValueMatcher('foo')), 957 mock.call.state.DidRunStory(root_mock.results), 958 mock.call.test.DidRunStory(root_mock.state.platform) 959 ]) 960 961 def testRunStoryAndProcessErrorIfNeeded_tryUnsupportedAction_finallyException( 962 self): 963 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 964 'test.WillRunStory': page_action.PageActionNotSupported('foo'), 965 'state.DidRunStory': Exception('bar') 966 }) 967 968 story_runner._RunStoryAndProcessErrorIfNeeded( 969 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 970 971 self.assertEquals(root_mock.method_calls, [ 972 mock.call.test.WillRunStory(root_mock.state.platform), 973 mock.call.results.AddValue(SkipValueMatcher()), 974 mock.call.state.DidRunStory(root_mock.results) 975 ]) 976 977 def testRunStoryAndProcessErrorIfNeeded_tryUnhandlable_finallyException(self): 978 root_mock = self._CreateErrorProcessingMock(method_exceptions={ 979 'test.Measure': Exception('foo'), 980 'test.DidRunStory': Exception('bar') 981 }) 982 983 with self.assertRaisesRegexp(Exception, 'foo'): 984 story_runner._RunStoryAndProcessErrorIfNeeded( 985 root_mock.story, root_mock.results, root_mock.state, root_mock.test) 986 987 self.assertEquals(root_mock.method_calls, [ 988 mock.call.test.WillRunStory(root_mock.state.platform), 989 mock.call.state.WillRunStory(root_mock.story), 990 mock.call.state.CanRunStory(root_mock.story), 991 mock.call.state.RunStory(root_mock.results), 992 mock.call.test.Measure(root_mock.state.platform, root_mock.results), 993 mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results), 994 mock.call.results.AddValue(FailureValueMatcher('foo')), 995 mock.call.state.DidRunStory(root_mock.results), 996 mock.call.test.DidRunStory(root_mock.state.platform) 997 ]) 998
