1 #!/usr/bin/env python2 2 3 # Copyright 2015 Google Inc. All Rights Reserved. 4 """This contains the unit tests for the new Crosperf task scheduler.""" 5 6 from __future__ import print_function 7 8 import mock 9 import unittest 10 import StringIO 11 12 import benchmark_run 13 import test_flag 14 from experiment_factory import ExperimentFactory 15 from experiment_file import ExperimentFile 16 from cros_utils.command_executer import CommandExecuter 17 from experiment_runner_unittest import FakeLogger 18 from schedv2 import Schedv2 19 20 EXPERIMENT_FILE_1 = """\ 21 board: daisy 22 remote: chromeos-daisy1.cros chromeos-daisy2.cros 23 24 benchmark: kraken { 25 suite: telemetry_Crosperf 26 iterations: 3 27 } 28 29 image1 { 30 chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin 31 remote: chromeos-daisy3.cros 32 } 33 34 image2 { 35 chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin 36 remote: chromeos-daisy4.cros chromeos-daisy5.cros 37 } 38 """ 39 40 EXPERIMENT_FILE_WITH_FORMAT = """\ 41 board: daisy 42 remote: chromeos-daisy1.cros chromeos-daisy2.cros 43 44 benchmark: kraken {{ 45 suite: telemetry_Crosperf 46 iterations: {kraken_iterations} 47 }} 48 49 image1 {{ 50 chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin 51 remote: chromeos-daisy3.cros 52 }} 53 54 image2 {{ 55 chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin 56 remote: chromeos-daisy4.cros chromeos-daisy5.cros 57 }} 58 """ 59 60 61 class Schedv2Test(unittest.TestCase): 62 """Class for setting up and running the unit tests.""" 63 64 def setUp(self): 65 self.exp = None 66 67 mock_logger = FakeLogger() 68 mock_cmd_exec = mock.Mock(spec=CommandExecuter) 69 70 @mock.patch('benchmark_run.BenchmarkRun', new=benchmark_run.MockBenchmarkRun) 71 def _make_fake_experiment(self, expstr): 72 """Create fake experiment from string. 73 74 Note - we mock out BenchmarkRun in this step. 75 """ 76 experiment_file = ExperimentFile(StringIO.StringIO(expstr)) 77 experiment = ExperimentFactory().GetExperiment(experiment_file, 78 working_directory='', 79 log_dir='') 80 return experiment 81 82 def test_remote(self): 83 """Test that remotes in labels are aggregated into experiment.remote.""" 84 85 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) 86 self.exp.log_level = 'verbose' 87 my_schedv2 = Schedv2(self.exp) 88 self.assertFalse(my_schedv2.is_complete()) 89 self.assertIn('chromeos-daisy1.cros', self.exp.remote) 90 self.assertIn('chromeos-daisy2.cros', self.exp.remote) 91 self.assertIn('chromeos-daisy3.cros', self.exp.remote) 92 self.assertIn('chromeos-daisy4.cros', self.exp.remote) 93 self.assertIn('chromeos-daisy5.cros', self.exp.remote) 94 95 def test_unreachable_remote(self): 96 """Test unreachable remotes are removed from experiment and label.""" 97 98 def MockIsReachable(cm): 99 return (cm.name != 'chromeos-daisy3.cros' and 100 cm.name != 'chromeos-daisy5.cros') 101 102 with mock.patch('machine_manager.MockCrosMachine.IsReachable', 103 new=MockIsReachable): 104 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) 105 self.assertIn('chromeos-daisy1.cros', self.exp.remote) 106 self.assertIn('chromeos-daisy2.cros', self.exp.remote) 107 self.assertNotIn('chromeos-daisy3.cros', self.exp.remote) 108 self.assertIn('chromeos-daisy4.cros', self.exp.remote) 109 self.assertNotIn('chromeos-daisy5.cros', self.exp.remote) 110 111 for l in self.exp.labels: 112 if l.name == 'image2': 113 self.assertNotIn('chromeos-daisy5.cros', l.remote) 114 self.assertIn('chromeos-daisy4.cros', l.remote) 115 elif l.name == 'image1': 116 self.assertNotIn('chromeos-daisy3.cros', l.remote) 117 118 @mock.patch('schedv2.BenchmarkRunCacheReader') 119 def test_BenchmarkRunCacheReader_1(self, reader): 120 """Test benchmarkrun set is split into 5 segments.""" 121 122 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format( 123 kraken_iterations=9)) 124 my_schedv2 = Schedv2(self.exp) 125 self.assertFalse(my_schedv2.is_complete()) 126 # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4, 127 # 4, 2 brs respectively. 128 # Assert that BenchmarkRunCacheReader() is called 5 times. 129 self.assertEquals(reader.call_count, 5) 130 # reader.call_args_list[n] - nth call. 131 # reader.call_args_list[n][0] - positioned args in nth call. 132 # reader.call_args_list[n][0][1] - the 2nd arg in nth call, 133 # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'. 134 self.assertEquals(len(reader.call_args_list[0][0][1]), 4) 135 self.assertEquals(len(reader.call_args_list[1][0][1]), 4) 136 self.assertEquals(len(reader.call_args_list[2][0][1]), 4) 137 self.assertEquals(len(reader.call_args_list[3][0][1]), 4) 138 self.assertEquals(len(reader.call_args_list[4][0][1]), 2) 139 140 @mock.patch('schedv2.BenchmarkRunCacheReader') 141 def test_BenchmarkRunCacheReader_2(self, reader): 142 """Test benchmarkrun set is split into 4 segments.""" 143 144 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format( 145 kraken_iterations=8)) 146 my_schedv2 = Schedv2(self.exp) 147 self.assertFalse(my_schedv2.is_complete()) 148 # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs. 149 self.assertEquals(reader.call_count, 4) 150 self.assertEquals(len(reader.call_args_list[0][0][1]), 4) 151 self.assertEquals(len(reader.call_args_list[1][0][1]), 4) 152 self.assertEquals(len(reader.call_args_list[2][0][1]), 4) 153 self.assertEquals(len(reader.call_args_list[3][0][1]), 4) 154 155 @mock.patch('schedv2.BenchmarkRunCacheReader') 156 def test_BenchmarkRunCacheReader_3(self, reader): 157 """Test benchmarkrun set is split into 2 segments.""" 158 159 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format( 160 kraken_iterations=3)) 161 my_schedv2 = Schedv2(self.exp) 162 self.assertFalse(my_schedv2.is_complete()) 163 # We have 3 * 2 == 6 brs, we use 2 threads. 164 self.assertEquals(reader.call_count, 2) 165 self.assertEquals(len(reader.call_args_list[0][0][1]), 3) 166 self.assertEquals(len(reader.call_args_list[1][0][1]), 3) 167 168 @mock.patch('schedv2.BenchmarkRunCacheReader') 169 def test_BenchmarkRunCacheReader_4(self, reader): 170 """Test benchmarkrun set is not splitted.""" 171 172 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format( 173 kraken_iterations=1)) 174 my_schedv2 = Schedv2(self.exp) 175 self.assertFalse(my_schedv2.is_complete()) 176 # We have 1 * 2 == 2 br, so only 1 instance. 177 self.assertEquals(reader.call_count, 1) 178 self.assertEquals(len(reader.call_args_list[0][0][1]), 2) 179 180 def test_cachehit(self): 181 """Test cache-hit and none-cache-hit brs are properly organized.""" 182 183 def MockReadCache(br): 184 br.cache_hit = (br.label.name == 'image2') 185 186 with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache', 187 new=MockReadCache): 188 # We have 2 * 30 brs, half of which are put into _cached_br_list. 189 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format( 190 kraken_iterations=30)) 191 my_schedv2 = Schedv2(self.exp) 192 self.assertEquals(len(my_schedv2.get_cached_run_list()), 30) 193 # The non-cache-hit brs are put into Schedv2._label_brl_map. 194 self.assertEquals( 195 reduce(lambda a, x: a + len(x[1]), 196 my_schedv2.get_label_map().iteritems(), 197 0), 30) 198 199 def test_nocachehit(self): 200 """Test no cache-hit.""" 201 202 def MockReadCache(br): 203 br.cache_hit = False 204 205 with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache', 206 new=MockReadCache): 207 # We have 2 * 30 brs, none of which are put into _cached_br_list. 208 self.exp = self._make_fake_experiment(EXPERIMENT_FILE_WITH_FORMAT.format( 209 kraken_iterations=30)) 210 my_schedv2 = Schedv2(self.exp) 211 self.assertEquals(len(my_schedv2.get_cached_run_list()), 0) 212 # The non-cache-hit brs are put into Schedv2._label_brl_map. 213 self.assertEquals( 214 reduce(lambda a, x: a + len(x[1]), 215 my_schedv2.get_label_map().iteritems(), 216 0), 60) 217 218 219 if __name__ == '__main__': 220 test_flag.SetTestMode(True) 221 unittest.main() 222
