1 #!/usr/bin/env python2 2 3 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 """Unit test for experiment_factory.py""" 7 8 from __future__ import print_function 9 10 import StringIO 11 import socket 12 import mock 13 import unittest 14 15 from cros_utils.file_utils import FileUtils 16 17 from experiment_factory import ExperimentFactory 18 from experiment_file import ExperimentFile 19 import test_flag 20 import benchmark 21 import experiment_factory 22 import settings_factory 23 24 EXPERIMENT_FILE_1 = """ 25 board: x86-alex 26 remote: chromeos-alex3 27 28 benchmark: PageCycler { 29 iterations: 3 30 } 31 32 image1 { 33 chromeos_image: /usr/local/google/cros_image1.bin 34 } 35 36 image2 { 37 chromeos_image: /usr/local/google/cros_image2.bin 38 } 39 """ 40 41 # pylint: disable=too-many-function-args 42 43 44 class ExperimentFactoryTest(unittest.TestCase): 45 """Class for running experiment factory unittests.""" 46 47 def setUp(self): 48 self.append_benchmark_call_args = [] 49 50 def testLoadExperimentFile1(self): 51 experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) 52 exp = ExperimentFactory().GetExperiment( 53 experiment_file, working_directory='', log_dir='') 54 self.assertEqual(exp.remote, ['chromeos-alex3']) 55 56 self.assertEqual(len(exp.benchmarks), 1) 57 self.assertEqual(exp.benchmarks[0].name, 'PageCycler') 58 self.assertEqual(exp.benchmarks[0].test_name, 'PageCycler') 59 self.assertEqual(exp.benchmarks[0].iterations, 3) 60 61 self.assertEqual(len(exp.labels), 2) 62 self.assertEqual(exp.labels[0].chromeos_image, 63 '/usr/local/google/cros_image1.bin') 64 self.assertEqual(exp.labels[0].board, 'x86-alex') 65 66 def test_append_benchmark_set(self): 67 ef = ExperimentFactory() 68 69 bench_list = [] 70 ef.AppendBenchmarkSet(bench_list, experiment_factory.telemetry_perfv2_tests, 71 '', 1, False, '', 'telemetry_Crosperf', False, 0, 72 False) 73 self.assertEqual( 74 len(bench_list), len(experiment_factory.telemetry_perfv2_tests)) 75 self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) 76 77 bench_list = [] 78 ef.AppendBenchmarkSet(bench_list, 79 experiment_factory.telemetry_pagecycler_tests, '', 1, 80 False, '', 'telemetry_Crosperf', False, 0, False) 81 self.assertEqual( 82 len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) 83 self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) 84 85 bench_list = [] 86 ef.AppendBenchmarkSet(bench_list, 87 experiment_factory.telemetry_toolchain_perf_tests, '', 88 1, False, '', 'telemetry_Crosperf', False, 0, False) 89 self.assertEqual( 90 len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests)) 91 self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) 92 93 @mock.patch.object(socket, 'gethostname') 94 def test_get_experiment(self, mock_socket): 95 96 test_flag.SetTestMode(False) 97 self.append_benchmark_call_args = [] 98 99 def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch, 100 perf_args, suite, show_all): 101 'Helper function for test_get_experiment' 102 arg_list = [ 103 bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all 104 ] 105 self.append_benchmark_call_args.append(arg_list) 106 107 def FakeGetDefaultRemotes(board): 108 if not board: 109 return [] 110 return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'] 111 112 def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level): 113 autotest_path = autotest_dir 114 if not autotest_path: 115 autotest_path = 'fake_autotest_path' 116 if not build or not board or not chroot or not log_level: 117 return '', autotest_path 118 return 'fake_image_path', autotest_path 119 120 ef = ExperimentFactory() 121 ef.AppendBenchmarkSet = FakeAppendBenchmarkSet 122 ef.GetDefaultRemotes = FakeGetDefaultRemotes 123 124 label_settings = settings_factory.LabelSettings('image_label') 125 benchmark_settings = settings_factory.BenchmarkSettings('bench_test') 126 global_settings = settings_factory.GlobalSettings('test_name') 127 128 label_settings.GetXbuddyPath = FakeGetXbuddyPath 129 130 mock_experiment_file = ExperimentFile(StringIO.StringIO('')) 131 mock_experiment_file.all_settings = [] 132 133 test_flag.SetTestMode(True) 134 # Basic test. 135 global_settings.SetField('name', 'unittest_test') 136 global_settings.SetField('board', 'lumpy') 137 global_settings.SetField('remote', '123.45.67.89 123.45.76.80') 138 benchmark_settings.SetField('test_name', 'kraken') 139 benchmark_settings.SetField('suite', 'telemetry_Crosperf') 140 benchmark_settings.SetField('iterations', 1) 141 label_settings.SetField( 142 'chromeos_image', 143 'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin') 144 label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top') 145 label_settings.SetField('autotest_path', '/tmp/autotest') 146 147 mock_experiment_file.global_settings = global_settings 148 mock_experiment_file.all_settings.append(label_settings) 149 mock_experiment_file.all_settings.append(benchmark_settings) 150 mock_experiment_file.all_settings.append(global_settings) 151 152 mock_socket.return_value = '' 153 154 # First test. General test. 155 exp = ef.GetExperiment(mock_experiment_file, '', '') 156 self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80']) 157 self.assertEqual(exp.cache_conditions, [0, 2, 1]) 158 self.assertEqual(exp.log_level, 'average') 159 160 self.assertEqual(len(exp.benchmarks), 1) 161 self.assertEqual(exp.benchmarks[0].name, 'kraken') 162 self.assertEqual(exp.benchmarks[0].test_name, 'kraken') 163 self.assertEqual(exp.benchmarks[0].iterations, 1) 164 self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf') 165 self.assertFalse(exp.benchmarks[0].show_all_results) 166 167 self.assertEqual(len(exp.labels), 1) 168 self.assertEqual(exp.labels[0].chromeos_image, 169 'chromeos/src/build/images/lumpy/latest/' 170 'chromiumos_test_image.bin') 171 self.assertEqual(exp.labels[0].autotest_path, '/tmp/autotest') 172 self.assertEqual(exp.labels[0].board, 'lumpy') 173 174 # Second test: Remotes listed in labels. 175 test_flag.SetTestMode(True) 176 label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros') 177 exp = ef.GetExperiment(mock_experiment_file, '', '') 178 self.assertEqual( 179 exp.remote, 180 ['chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80']) 181 182 # Third test: Automatic fixing of bad logging_level param: 183 global_settings.SetField('logging_level', 'really loud!') 184 exp = ef.GetExperiment(mock_experiment_file, '', '') 185 self.assertEqual(exp.log_level, 'verbose') 186 187 # Fourth test: Setting cache conditions; only 1 remote with "same_machine" 188 global_settings.SetField('rerun_if_failed', 'true') 189 global_settings.SetField('rerun', 'true') 190 global_settings.SetField('same_machine', 'true') 191 global_settings.SetField('same_specs', 'true') 192 193 self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '') 194 label_settings.SetField('remote', '') 195 global_settings.SetField('remote', '123.45.67.89') 196 exp = ef.GetExperiment(mock_experiment_file, '', '') 197 self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) 198 199 # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all 200 # remotes (Call GetDefaultRemotes). 201 mock_socket.return_value = 'test.corp.google.com' 202 global_settings.SetField('remote', '') 203 global_settings.SetField('same_machine', 'false') 204 205 label_settings_2 = settings_factory.LabelSettings('official_image_label') 206 label_settings_2.SetField('chromeos_root', 'chromeos') 207 label_settings_2.SetField('build', 'official-dev') 208 label_settings_2.SetField('autotest_path', '') 209 label_settings_2.GetXbuddyPath = FakeGetXbuddyPath 210 211 mock_experiment_file.all_settings.append(label_settings_2) 212 exp = ef.GetExperiment(mock_experiment_file, '', '') 213 self.assertEqual(len(exp.labels), 2) 214 self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path') 215 self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path') 216 self.assertEqual( 217 exp.remote, 218 ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']) 219 220 def test_get_default_remotes(self): 221 board_list = [ 222 'x86-alex', 'lumpy', 'elm', 'parrot', 'daisy', 'peach_pit', 'peppy', 223 'squawks' 224 ] 225 226 ef = ExperimentFactory() 227 self.assertRaises(Exception, ef.GetDefaultRemotes, 'bad-board') 228 229 # Verify that we have entries for every board, and that we get three 230 # machines back for each board. 231 for b in board_list: 232 remotes = ef.GetDefaultRemotes(b) 233 if b == 'elm': 234 self.assertEqual(len(remotes), 2) 235 else: 236 self.assertEqual(len(remotes), 3) 237 238 239 if __name__ == '__main__': 240 FileUtils.Configure(True) 241 test_flag.SetTestMode(True) 242 unittest.main() 243
