1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Run the first page of every benchmark that has a composable measurement. 6 7 Ideally this test would be comprehensive, but the above serves as a 8 kind of smoke test. 9 """ 10 11 import os 12 import unittest 13 14 from telemetry import benchmark as benchmark_module 15 from telemetry.core import discover 16 from telemetry.page import page_test 17 from telemetry.unittest import options_for_unittests 18 from telemetry.unittest import progress_reporter 19 20 21 def SmokeTestGenerator(benchmark): 22 # NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST. 23 # 24 # This smoke test dynamically tests all benchmarks. So disabling it for one 25 # failing or flaky benchmark would disable a much wider swath of coverage 26 # than is usally intended. Instead, if a particular benchmark is failing, 27 # disable it in tools/perf/benchmarks/*. 28 @benchmark_module.Disabled('chromeos') # crbug.com/351114 29 def BenchmarkSmokeTest(self): 30 # Only measure a single page so that this test cycles reasonably quickly. 31 benchmark.options['pageset_repeat'] = 1 32 benchmark.options['page_repeat'] = 1 33 34 class SinglePageBenchmark(benchmark): # pylint: disable=W0232 35 def CreatePageSet(self, options): 36 # pylint: disable=E1002 37 ps = super(SinglePageBenchmark, self).CreatePageSet(options) 38 for p in ps.pages: 39 if not p.disabled: 40 p.skip_waits = True 41 ps.pages = [p] 42 break 43 return ps 44 45 # Set the benchmark's default arguments. 46 options = options_for_unittests.GetCopy() 47 options.output_format = 'none' 48 options.suppress_gtest_report = True 49 parser = options.CreateParser() 50 51 benchmark.AddCommandLineArgs(parser) 52 benchmark_module.AddCommandLineArgs(parser) 53 benchmark.SetArgumentDefaults(parser) 54 options.MergeDefaultValues(parser.get_default_values()) 55 56 benchmark.ProcessCommandLineArgs(None, options) 57 benchmark_module.ProcessCommandLineArgs(None, options) 58 59 self.assertEqual(0, SinglePageBenchmark().Run(options), 60 msg='Failed: %s' % benchmark) 61 62 return BenchmarkSmokeTest 63 64 65 def load_tests(_, _2, _3): 66 suite = progress_reporter.TestSuite() 67 68 benchmarks_dir = os.path.dirname(__file__) 69 top_level_dir = os.path.dirname(benchmarks_dir) 70 measurements_dir = os.path.join(top_level_dir, 'measurements') 71 72 all_measurements = discover.DiscoverClasses( 73 measurements_dir, top_level_dir, page_test.PageTest, 74 pattern='*.py').values() 75 all_benchmarks = discover.DiscoverClasses( 76 benchmarks_dir, top_level_dir, benchmark_module.Benchmark, 77 pattern='*.py').values() 78 for benchmark in all_benchmarks: 79 if benchmark.PageTestClass() not in all_measurements: 80 # If the benchmark is not in measurements, then it is not composable. 81 # Ideally we'd like to test these as well, but the non-composable 82 # benchmarks are usually long-running benchmarks. 83 continue 84 85 # TODO(tonyg): Smoke doesn't work with session_restore yet. 86 if benchmark.Name().startswith('session_restore'): 87 continue 88 89 if hasattr(benchmark, 'generated_profile_archive'): 90 # We'd like to test these, but don't know how yet. 91 continue 92 93 class BenchmarkSmokeTest(unittest.TestCase): 94 pass 95 setattr(BenchmarkSmokeTest, benchmark.Name(), SmokeTestGenerator(benchmark)) 96 suite.addTest(BenchmarkSmokeTest(benchmark.Name())) 97 98 return suite 99
