1 #!/usr/bin/env python2 2 3 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 """Module of result cache unittest.""" 7 8 from __future__ import print_function 9 10 import mock 11 import os 12 import tempfile 13 import unittest 14 15 import image_checksummer 16 import machine_manager 17 import test_flag 18 19 from label import MockLabel 20 from results_cache import CacheConditions 21 from results_cache import Result 22 from results_cache import ResultsCache 23 from results_cache import TelemetryResult 24 from cros_utils import command_executer 25 from cros_utils import logger 26 from cros_utils import misc 27 28 OUTPUT = """CMD (True): ./test_that.sh\ 29 --remote=172.17.128.241 --board=lumpy LibCBench 30 CMD (None): cd /usr/local/google/home/yunlian/gd/src/build/images/lumpy/latest/../../../../..; cros_sdk -- ./in_chroot_cmd6X7Cxu.sh 31 Identity added: /tmp/test_that.PO1234567/autotest_key (/tmp/test_that.PO1234567/autotest_key) 32 INFO : Using emerged autotests already installed at /build/lumpy/usr/local/autotest. 33 34 INFO : Running the following control files 1 times: 35 INFO : * 'client/site_tests/platform_LibCBench/control' 36 37 INFO : Running client test client/site_tests/platform_LibCBench/control 38 ./server/autoserv -m 172.17.128.241 --ssh-port 22 -c client/site_tests/platform_LibCBench/control -r /tmp/test_that.PO1234567/platform_LibCBench --test-retry=0 --args 39 ERROR:root:import statsd failed, no stats will be reported. 40 14:20:22 INFO | Results placed in /tmp/test_that.PO1234567/platform_LibCBench 41 14:20:22 INFO | Processing control file 42 14:20:23 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_VIIP67ssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 43 14:20:23 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. 44 14:20:23 INFO | INFO ---- ---- kernel=3.8.11 localtime=May 22 14:20:23 timestamp=1369257623 45 14:20:23 INFO | Installing autotest on 172.17.128.241 46 14:20:23 INFO | Using installation dir /usr/local/autotest 47 14:20:23 WARNI| No job_repo_url for <remote host: 172.17.128.241> 48 14:20:23 INFO | Could not install autotest using the packaging system: No repos to install an autotest client from. Trying other methods 49 14:20:23 INFO | Installation of autotest completed 50 14:20:24 WARNI| No job_repo_url for <remote host: 172.17.128.241> 51 14:20:24 INFO | Executing /usr/local/autotest/bin/autotest /usr/local/autotest/control phase 0 52 14:20:24 INFO | Entered autotestd_monitor. 53 14:20:24 INFO | Finished launching tail subprocesses. 54 14:20:24 INFO | Finished waiting on autotestd to start. 55 14:20:26 INFO | START ---- ---- timestamp=1369257625 localtime=May 22 14:20:25 56 14:20:26 INFO | START platform_LibCBench platform_LibCBench timestamp=1369257625 localtime=May 22 14:20:25 57 14:20:30 INFO | GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 completed successfully 58 14:20:30 INFO | END GOOD platform_LibCBench platform_LibCBench timestamp=1369257630 localtime=May 22 14:20:30 59 14:20:31 INFO | END GOOD ---- ---- timestamp=1369257630 localtime=May 22 14:20:30 60 14:20:31 INFO | Got lock of exit_code_file. 61 14:20:31 INFO | Released lock of exit_code_file and closed it. 62 OUTPUT: ============================== 63 OUTPUT: Current time: 2013-05-22 14:20:32.818831 Elapsed: 0:01:30 ETA: Unknown 64 Done: 0% [ ] 65 OUTPUT: Thread Status: 66 RUNNING: 1 ('ttt: LibCBench (1)' 0:01:21) 67 Machine Status: 68 Machine Thread Lock Status Checksum 69 172.17.128.241 ttt: LibCBench (1) True RUNNING 3ba9f2ecbb222f20887daea5583d86ba 70 71 OUTPUT: ============================== 72 14:20:33 INFO | Killing child processes. 73 14:20:33 INFO | Client complete 74 14:20:33 INFO | Finished processing control file 75 14:20:33 INFO | Starting master ssh connection '/usr/bin/ssh -a -x -N -o ControlMaster=yes -o ControlPath=/tmp/_autotmp_aVJUgmssh-master/socket -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=180 -o ServerAliveCountMax=3 -o ConnectionAttempts=4 -o Protocol=2 -l root -p 22 172.17.128.241' 76 14:20:33 ERROR| [stderr] Warning: Permanently added '172.17.128.241' (RSA) to the list of known hosts. 77 78 INFO : Test results: 79 ------------------------------------------------------------------- 80 platform_LibCBench [ PASSED ] 81 platform_LibCBench/platform_LibCBench [ PASSED ] 82 platform_LibCBench/platform_LibCBench b_malloc_big1__0_ 0.00375231466667 83 platform_LibCBench/platform_LibCBench b_malloc_big2__0_ 0.002951359 84 platform_LibCBench/platform_LibCBench b_malloc_bubble__0_ 0.015066374 85 platform_LibCBench/platform_LibCBench b_malloc_sparse__0_ 0.015053784 86 platform_LibCBench/platform_LibCBench b_malloc_thread_local__0_ 0.01138439 87 platform_LibCBench/platform_LibCBench b_malloc_thread_stress__0_ 0.0367894733333 88 platform_LibCBench/platform_LibCBench b_malloc_tiny1__0_ 0.000768474333333 89 platform_LibCBench/platform_LibCBench b_malloc_tiny2__0_ 0.000581407333333 90 platform_LibCBench/platform_LibCBench b_pthread_create_serial1__0_ 0.0291785246667 91 platform_LibCBench/platform_LibCBench b_pthread_createjoin_serial1__0_ 0.031907936 92 platform_LibCBench/platform_LibCBench b_pthread_createjoin_serial2__0_ 0.043485347 93 platform_LibCBench/platform_LibCBench b_pthread_uselesslock__0_ 0.0294113346667 94 platform_LibCBench/platform_LibCBench b_regex_compile____a_b_c__d_b__ 0.00529833933333 95 platform_LibCBench/platform_LibCBench b_regex_search____a_b_c__d_b__ 0.00165455066667 96 platform_LibCBench/platform_LibCBench b_regex_search___a_25_b__ 0.0496191923333 97 platform_LibCBench/platform_LibCBench b_stdio_putcgetc__0_ 0.100005711667 98 platform_LibCBench/platform_LibCBench b_stdio_putcgetc_unlocked__0_ 0.0371443833333 99 platform_LibCBench/platform_LibCBench b_string_memset__0_ 0.00275405066667 100 platform_LibCBench/platform_LibCBench b_string_strchr__0_ 0.00456903 101 platform_LibCBench/platform_LibCBench b_string_strlen__0_ 0.044893587 102 platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac__ 0.118360778 103 platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaac__ 0.068957325 104 platform_LibCBench/platform_LibCBench b_string_strstr___aaaaaaaaaaaaaacccccccccccc__ 0.0135694476667 105 platform_LibCBench/platform_LibCBench b_string_strstr___abcdefghijklmnopqrstuvwxyz__ 0.0134553343333 106 platform_LibCBench/platform_LibCBench b_string_strstr___azbycxdwevfugthsirjqkplomn__ 0.0133123556667 107 platform_LibCBench/platform_LibCBench b_utf8_bigbuf__0_ 0.0473772253333 108 platform_LibCBench/platform_LibCBench b_utf8_onebyone__0_ 0.130938538333 109 ------------------------------------------------------------------- 110 Total PASS: 2/2 (100%) 111 112 INFO : Elapsed time: 0m16s 113 """ 114 115 error = """ 116 ERROR: Identity added: /tmp/test_that.Z4Ld/autotest_key (/tmp/test_that.Z4Ld/autotest_key) 117 INFO : Using emerged autotests already installed at /build/lumpy/usr/local/autotest. 118 INFO : Running the following control files 1 times: 119 INFO : * 'client/site_tests/platform_LibCBench/control' 120 INFO : Running client test client/site_tests/platform_LibCBench/control 121 INFO : Test results: 122 INFO : Elapsed time: 0m18s 123 """ 124 125 keyvals = { 126 '': 'PASS', 127 'b_stdio_putcgetc__0_': '0.100005711667', 128 'b_string_strstr___azbycxdwevfugthsirjqkplomn__': '0.0133123556667', 129 'b_malloc_thread_local__0_': '0.01138439', 130 'b_string_strlen__0_': '0.044893587', 131 'b_malloc_sparse__0_': '0.015053784', 132 'b_string_memset__0_': '0.00275405066667', 133 'platform_LibCBench': 'PASS', 134 'b_pthread_uselesslock__0_': '0.0294113346667', 135 'b_string_strchr__0_': '0.00456903', 136 'b_pthread_create_serial1__0_': '0.0291785246667', 137 'b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac__': '0.118360778', 138 'b_string_strstr___aaaaaaaaaaaaaacccccccccccc__': '0.0135694476667', 139 'b_pthread_createjoin_serial1__0_': '0.031907936', 140 'b_malloc_thread_stress__0_': '0.0367894733333', 141 'b_regex_search____a_b_c__d_b__': '0.00165455066667', 142 'b_malloc_bubble__0_': '0.015066374', 143 'b_malloc_big2__0_': '0.002951359', 144 'b_stdio_putcgetc_unlocked__0_': '0.0371443833333', 145 'b_pthread_createjoin_serial2__0_': '0.043485347', 146 'b_regex_search___a_25_b__': '0.0496191923333', 147 'b_utf8_bigbuf__0_': '0.0473772253333', 148 'b_malloc_big1__0_': '0.00375231466667', 149 'b_regex_compile____a_b_c__d_b__': '0.00529833933333', 150 'b_string_strstr___aaaaaaaaaaaaaaaaaaaaaaaaac__': '0.068957325', 151 'b_malloc_tiny2__0_': '0.000581407333333', 152 'b_utf8_onebyone__0_': '0.130938538333', 153 'b_malloc_tiny1__0_': '0.000768474333333', 154 'b_string_strstr___abcdefghijklmnopqrstuvwxyz__': '0.0134553343333' 155 } 156 157 TMP_DIR1 = '/tmp/tmpAbcXyz' 158 159 160 class MockResult(Result): 161 """Mock result class.""" 162 163 def __init__(self, mylogger, label, logging_level, machine): 164 super(MockResult, self).__init__(mylogger, label, logging_level, machine) 165 166 def FindFilesInResultsDir(self, find_args): 167 return '' 168 169 # pylint: disable=arguments-differ 170 def GetKeyvals(self, temp=False): 171 if temp: 172 pass 173 return keyvals 174 175 176 class ResultTest(unittest.TestCase): 177 """Result test class.""" 178 179 def __init__(self, *args, **kwargs): 180 super(ResultTest, self).__init__(*args, **kwargs) 181 self.callFakeProcessResults = False 182 self.fakeCacheReturnResult = None 183 self.callGetResultsDir = False 184 self.callProcessResults = False 185 self.callGetPerfReportFiles = False 186 self.kv_dict = None 187 self.tmpdir = '' 188 self.callGetNewKeyvals = False 189 self.callGetResultsFile = False 190 self.callGetPerfDataFiles = False 191 self.args = None 192 self.callGatherPerfResults = False 193 self.mock_logger = mock.Mock(spec=logger.Logger) 194 self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 195 self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', 196 '/tmp', 'lumpy', 'remote', 'image_args', 197 'cache_dir', 'average', 'gcc', None) 198 199 def testCreateFromRun(self): 200 result = MockResult.CreateFromRun(logger.GetLogger(), 'average', 201 self.mock_label, 'remote1', OUTPUT, error, 202 0, True, 0) 203 self.assertEqual(result.keyvals, keyvals) 204 self.assertEqual(result.chroot_results_dir, 205 '/tmp/test_that.PO1234567/platform_LibCBench') 206 self.assertEqual(result.results_dir, 207 '/tmp/chroot/tmp/test_that.PO1234567/platform_LibCBench') 208 self.assertEqual(result.retval, 0) 209 210 def setUp(self): 211 self.result = Result(self.mock_logger, self.mock_label, 'average', 212 self.mock_cmd_exec) 213 214 @mock.patch.object(os.path, 'isdir') 215 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 216 @mock.patch.object(command_executer.CommandExecuter, 'CopyFiles') 217 def test_copy_files_to(self, mock_copyfiles, mock_runcmd, mock_isdir): 218 219 files = ['src_file_1', 'src_file_2', 'src_file_3'] 220 dest_dir = '/tmp/test' 221 self.mock_cmd_exec.RunCommand = mock_runcmd 222 self.mock_cmd_exec.CopyFiles = mock_copyfiles 223 224 mock_copyfiles.return_value = 0 225 226 #test 1. dest_dir exists; CopyFiles returns 0. 227 mock_isdir.return_value = True 228 self.result.CopyFilesTo(dest_dir, files) 229 self.assertEqual(mock_runcmd.call_count, 0) 230 self.assertEqual(mock_copyfiles.call_count, 3) 231 first_args = mock_copyfiles.call_args_list[0][0] 232 second_args = mock_copyfiles.call_args_list[1][0] 233 third_args = mock_copyfiles.call_args_list[2][0] 234 self.assertEqual(first_args, ('src_file_1', '/tmp/test/src_file_1.0')) 235 self.assertEqual(second_args, ('src_file_2', '/tmp/test/src_file_2.0')) 236 self.assertEqual(third_args, ('src_file_3', '/tmp/test/src_file_3.0')) 237 238 mock_runcmd.reset_mock() 239 mock_copyfiles.reset_mock() 240 #test 2. dest_dir does not exist; CopyFiles returns 0. 241 mock_isdir.return_value = False 242 self.result.CopyFilesTo(dest_dir, files) 243 self.assertEqual(mock_runcmd.call_count, 3) 244 self.assertEqual(mock_copyfiles.call_count, 3) 245 self.assertEqual(mock_runcmd.call_args_list[0], 246 mock_runcmd.call_args_list[1]) 247 self.assertEqual(mock_runcmd.call_args_list[0], 248 mock_runcmd.call_args_list[2]) 249 self.assertEqual(mock_runcmd.call_args_list[0][0], ('mkdir -p /tmp/test',)) 250 251 #test 3. CopyFiles returns 1 (fails). 252 mock_copyfiles.return_value = 1 253 self.assertRaises(Exception, self.result.CopyFilesTo, dest_dir, files) 254 255 @mock.patch.object(Result, 'CopyFilesTo') 256 def test_copy_results_to(self, mockCopyFilesTo): 257 perf_data_files = [ 258 '/tmp/perf.data.0', '/tmp/perf.data.1', '/tmp/perf.data.2' 259 ] 260 perf_report_files = [ 261 '/tmp/perf.report.0', '/tmp/perf.report.1', '/tmp/perf.report.2' 262 ] 263 264 self.result.perf_data_files = perf_data_files 265 self.result.perf_report_files = perf_report_files 266 267 self.result.CopyFilesTo = mockCopyFilesTo 268 self.result.CopyResultsTo('/tmp/results/') 269 self.assertEqual(mockCopyFilesTo.call_count, 2) 270 self.assertEqual(len(mockCopyFilesTo.call_args_list), 2) 271 self.assertEqual(mockCopyFilesTo.call_args_list[0][0], ('/tmp/results/', 272 perf_data_files)) 273 self.assertEqual(mockCopyFilesTo.call_args_list[1][0], ('/tmp/results/', 274 perf_report_files)) 275 276 def test_get_new_keyvals(self): 277 kv_dict = {} 278 279 def FakeGetDataMeasurementsFiles(): 280 filename = os.path.join(os.getcwd(), 'unittest_keyval_file.txt') 281 return [filename] 282 283 self.result.GetDataMeasurementsFiles = FakeGetDataMeasurementsFiles 284 kv_dict2, udict = self.result.GetNewKeyvals(kv_dict) 285 self.assertEqual(kv_dict2, { 286 u'Box2D__Box2D': 4775, 287 u'Mandreel__Mandreel': 6620, 288 u'Gameboy__Gameboy': 9901, 289 u'Crypto__Crypto': 8737, 290 u'telemetry_page_measurement_results__num_errored': 0, 291 u'telemetry_page_measurement_results__num_failed': 0, 292 u'PdfJS__PdfJS': 6455, 293 u'Total__Score': 7918, 294 u'EarleyBoyer__EarleyBoyer': 14340, 295 u'MandreelLatency__MandreelLatency': 5188, 296 u'CodeLoad__CodeLoad': 6271, 297 u'DeltaBlue__DeltaBlue': 14401, 298 u'Typescript__Typescript': 9815, 299 u'SplayLatency__SplayLatency': 7653, 300 u'zlib__zlib': 16094, 301 u'Richards__Richards': 10358, 302 u'RegExp__RegExp': 1765, 303 u'NavierStokes__NavierStokes': 9815, 304 u'Splay__Splay': 4425, 305 u'RayTrace__RayTrace': 16600 306 }) 307 self.assertEqual(udict, { 308 u'Box2D__Box2D': u'score', 309 u'Mandreel__Mandreel': u'score', 310 u'Gameboy__Gameboy': u'score', 311 u'Crypto__Crypto': u'score', 312 u'telemetry_page_measurement_results__num_errored': u'count', 313 u'telemetry_page_measurement_results__num_failed': u'count', 314 u'PdfJS__PdfJS': u'score', 315 u'Total__Score': u'score', 316 u'EarleyBoyer__EarleyBoyer': u'score', 317 u'MandreelLatency__MandreelLatency': u'score', 318 u'CodeLoad__CodeLoad': u'score', 319 u'DeltaBlue__DeltaBlue': u'score', 320 u'Typescript__Typescript': u'score', 321 u'SplayLatency__SplayLatency': u'score', 322 u'zlib__zlib': u'score', 323 u'Richards__Richards': u'score', 324 u'RegExp__RegExp': u'score', 325 u'NavierStokes__NavierStokes': u'score', 326 u'Splay__Splay': u'score', 327 u'RayTrace__RayTrace': u'score' 328 }) 329 330 def test_append_telemetry_units(self): 331 kv_dict = { 332 u'Box2D__Box2D': 4775, 333 u'Mandreel__Mandreel': 6620, 334 u'Gameboy__Gameboy': 9901, 335 u'Crypto__Crypto': 8737, 336 u'PdfJS__PdfJS': 6455, 337 u'Total__Score': 7918, 338 u'EarleyBoyer__EarleyBoyer': 14340, 339 u'MandreelLatency__MandreelLatency': 5188, 340 u'CodeLoad__CodeLoad': 6271, 341 u'DeltaBlue__DeltaBlue': 14401, 342 u'Typescript__Typescript': 9815, 343 u'SplayLatency__SplayLatency': 7653, 344 u'zlib__zlib': 16094, 345 u'Richards__Richards': 10358, 346 u'RegExp__RegExp': 1765, 347 u'NavierStokes__NavierStokes': 9815, 348 u'Splay__Splay': 4425, 349 u'RayTrace__RayTrace': 16600 350 } 351 units_dict = { 352 u'Box2D__Box2D': u'score', 353 u'Mandreel__Mandreel': u'score', 354 u'Gameboy__Gameboy': u'score', 355 u'Crypto__Crypto': u'score', 356 u'PdfJS__PdfJS': u'score', 357 u'Total__Score': u'score', 358 u'EarleyBoyer__EarleyBoyer': u'score', 359 u'MandreelLatency__MandreelLatency': u'score', 360 u'CodeLoad__CodeLoad': u'score', 361 u'DeltaBlue__DeltaBlue': u'score', 362 u'Typescript__Typescript': u'score', 363 u'SplayLatency__SplayLatency': u'score', 364 u'zlib__zlib': u'score', 365 u'Richards__Richards': u'score', 366 u'RegExp__RegExp': u'score', 367 u'NavierStokes__NavierStokes': u'score', 368 u'Splay__Splay': u'score', 369 u'RayTrace__RayTrace': u'score' 370 } 371 372 results_dict = self.result.AppendTelemetryUnits(kv_dict, units_dict) 373 self.assertEqual(results_dict, { 374 u'Box2D__Box2D': [4775, u'score'], 375 u'Splay__Splay': [4425, u'score'], 376 u'Gameboy__Gameboy': [9901, u'score'], 377 u'Crypto__Crypto': [8737, u'score'], 378 u'PdfJS__PdfJS': [6455, u'score'], 379 u'Total__Score': [7918, u'score'], 380 u'EarleyBoyer__EarleyBoyer': [14340, u'score'], 381 u'MandreelLatency__MandreelLatency': [5188, u'score'], 382 u'DeltaBlue__DeltaBlue': [14401, u'score'], 383 u'SplayLatency__SplayLatency': [7653, u'score'], 384 u'Mandreel__Mandreel': [6620, u'score'], 385 u'Richards__Richards': [10358, u'score'], 386 u'zlib__zlib': [16094, u'score'], 387 u'CodeLoad__CodeLoad': [6271, u'score'], 388 u'Typescript__Typescript': [9815, u'score'], 389 u'RegExp__RegExp': [1765, u'score'], 390 u'RayTrace__RayTrace': [16600, u'score'], 391 u'NavierStokes__NavierStokes': [9815, u'score'] 392 }) 393 394 @mock.patch.object(misc, 'GetInsideChrootPath') 395 @mock.patch.object(tempfile, 'mkdtemp') 396 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 397 @mock.patch.object(command_executer.CommandExecuter, 398 'ChrootRunCommandWOutput') 399 def test_get_keyvals(self, mock_chrootruncmd, mock_runcmd, mock_mkdtemp, 400 mock_getpath): 401 402 self.kv_dict = {} 403 self.callGetNewKeyvals = False 404 405 def reset(): 406 self.kv_dict = {} 407 self.callGetNewKeyvals = False 408 mock_chrootruncmd.reset_mock() 409 mock_runcmd.reset_mock() 410 mock_mkdtemp.reset_mock() 411 mock_getpath.reset_mock() 412 413 def FakeGetNewKeyvals(kv_dict): 414 self.kv_dict = kv_dict 415 self.callGetNewKeyvals = True 416 return_kvdict = {'first_time': 680, 'Total': 10} 417 return_udict = {'first_time': 'ms', 'Total': 'score'} 418 return return_kvdict, return_udict 419 420 mock_mkdtemp.return_value = TMP_DIR1 421 mock_chrootruncmd.return_value = [ 422 '', ('%s,PASS\n%s/telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), '' 423 ] 424 mock_getpath.return_value = TMP_DIR1 425 self.result.ce.ChrootRunCommandWOutput = mock_chrootruncmd 426 self.result.ce.RunCommand = mock_runcmd 427 self.result.GetNewKeyvals = FakeGetNewKeyvals 428 self.result.suite = 'telemetry_Crosperf' 429 self.result.results_dir = '/tmp/test_that_resultsNmq' 430 431 # Test 1. no self.temp_dir. 432 res = self.result.GetKeyvals() 433 self.assertTrue(self.callGetNewKeyvals) 434 self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) 435 self.assertEqual(mock_runcmd.call_count, 1) 436 self.assertEqual(mock_runcmd.call_args_list[0][0], 437 ('cp -r /tmp/test_that_resultsNmq/* %s' % TMP_DIR1,)) 438 self.assertEqual(mock_chrootruncmd.call_count, 1) 439 self.assertEqual( 440 mock_chrootruncmd.call_args_list[0][0], 441 ('/tmp', 442 ('python generate_test_report --no-color --csv %s') % TMP_DIR1)) 443 self.assertEqual(mock_getpath.call_count, 1) 444 self.assertEqual(mock_mkdtemp.call_count, 1) 445 self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) 446 447 # Test 2. self.temp_dir 448 reset() 449 mock_chrootruncmd.return_value = [ 450 '', ('/tmp/tmpJCajRG,PASS\n/tmp/tmpJCajRG/' 451 'telemetry_Crosperf,PASS\n'), '' 452 ] 453 mock_getpath.return_value = '/tmp/tmpJCajRG' 454 self.result.temp_dir = '/tmp/tmpJCajRG' 455 res = self.result.GetKeyvals() 456 self.assertEqual(mock_runcmd.call_count, 0) 457 self.assertEqual(mock_mkdtemp.call_count, 0) 458 self.assertEqual(mock_chrootruncmd.call_count, 1) 459 self.assertTrue(self.callGetNewKeyvals) 460 self.assertEqual(self.kv_dict, {'': 'PASS', 'telemetry_Crosperf': 'PASS'}) 461 self.assertEqual(res, {'Total': [10, 'score'], 'first_time': [680, 'ms']}) 462 463 # Test 3. suite != telemetry_Crosperf. Normally this would be for 464 # running non-Telemetry autotests, such as BootPerfServer. In this test 465 # case, the keyvals we have set up were returned from a Telemetry test run; 466 # so this pass is basically testing that we don't append the units to the 467 # test results (which we do for Telemetry autotest runs). 468 reset() 469 self.result.suite = '' 470 res = self.result.GetKeyvals() 471 self.assertEqual(res, {'Total': 10, 'first_time': 680}) 472 473 def test_get_results_dir(self): 474 475 self.result.out = '' 476 self.assertRaises(Exception, self.result.GetResultsDir) 477 478 self.result.out = OUTPUT 479 resdir = self.result.GetResultsDir() 480 self.assertEqual(resdir, '/tmp/test_that.PO1234567/platform_LibCBench') 481 482 @mock.patch.object(command_executer.CommandExecuter, 'RunCommandGeneric') 483 def test_find_files_in_results_dir(self, mock_runcmd): 484 485 self.result.results_dir = None 486 res = self.result.FindFilesInResultsDir('-name perf.data') 487 self.assertIsNone(res) 488 489 self.result.ce.RunCommand = mock_runcmd 490 self.result.results_dir = '/tmp/test_results' 491 mock_runcmd.return_value = [0, '/tmp/test_results/perf.data', ''] 492 res = self.result.FindFilesInResultsDir('-name perf.data') 493 self.assertEqual(mock_runcmd.call_count, 1) 494 self.assertEqual(mock_runcmd.call_args_list[0][0], 495 ('find /tmp/test_results -name perf.data',)) 496 self.assertEqual(res, '/tmp/test_results/perf.data') 497 498 mock_runcmd.reset_mock() 499 mock_runcmd.return_value = [1, '', ''] 500 self.assertRaises(Exception, self.result.FindFilesInResultsDir, 501 '-name perf.data') 502 503 @mock.patch.object(Result, 'FindFilesInResultsDir') 504 def test_get_perf_data_files(self, mock_findfiles): 505 self.args = None 506 507 mock_findfiles.return_value = 'line1\nline1\n' 508 self.result.FindFilesInResultsDir = mock_findfiles 509 res = self.result.GetPerfDataFiles() 510 self.assertEqual(res, ['line1', 'line1']) 511 self.assertEqual(mock_findfiles.call_args_list[0][0], ('-name perf.data',)) 512 513 def test_get_perf_report_files(self): 514 self.args = None 515 516 def FakeFindFiles(find_args): 517 self.args = find_args 518 return 'line1\nline1\n' 519 520 self.result.FindFilesInResultsDir = FakeFindFiles 521 res = self.result.GetPerfReportFiles() 522 self.assertEqual(res, ['line1', 'line1']) 523 self.assertEqual(self.args, '-name perf.data.report') 524 525 def test_get_data_measurement_files(self): 526 self.args = None 527 528 def FakeFindFiles(find_args): 529 self.args = find_args 530 return 'line1\nline1\n' 531 532 self.result.FindFilesInResultsDir = FakeFindFiles 533 res = self.result.GetDataMeasurementsFiles() 534 self.assertEqual(res, ['line1', 'line1']) 535 self.assertEqual(self.args, '-name perf_measurements') 536 537 @mock.patch.object(misc, 'GetInsideChrootPath') 538 @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') 539 def test_generate_perf_report_files(self, mock_chrootruncmd, mock_getpath): 540 fake_file = '/usr/chromeos/chroot/tmp/results/fake_file' 541 self.result.perf_data_files = ['/tmp/results/perf.data'] 542 self.result.board = 'lumpy' 543 mock_getpath.return_value = fake_file 544 self.result.ce.ChrootRunCommand = mock_chrootruncmd 545 tmp = self.result.GeneratePerfReportFiles() 546 self.assertEqual(tmp, ['/tmp/chroot%s' % fake_file]) 547 self.assertEqual(mock_chrootruncmd.call_args_list[0][0], 548 ('/tmp', 549 ('/usr/sbin/perf report -n --symfs /build/lumpy ' 550 '--vmlinux /build/lumpy/usr/lib/debug/boot/vmlinux ' 551 '--kallsyms /build/lumpy/boot/System.map-* -i ' 552 '%s --stdio > %s') % (fake_file, fake_file))) 553 554 @mock.patch.object(misc, 'GetOutsideChrootPath') 555 def test_populate_from_run(self, mock_getpath): 556 557 def FakeGetResultsDir(): 558 self.callGetResultsDir = True 559 return '/tmp/results_dir' 560 561 def FakeGetResultsFile(): 562 self.callGetResultsFile = True 563 return [] 564 565 def FakeGetPerfDataFiles(): 566 self.callGetPerfDataFiles = True 567 return [] 568 569 def FakeGetPerfReportFiles(): 570 self.callGetPerfReportFiles = True 571 return [] 572 573 def FakeProcessResults(show_results=False): 574 if show_results: 575 pass 576 self.callProcessResults = True 577 578 if mock_getpath: 579 pass 580 mock.get_path = '/tmp/chromeos/tmp/results_dir' 581 self.result.chromeos_root = '/tmp/chromeos' 582 583 self.callGetResultsDir = False 584 self.callGetResultsFile = False 585 self.callGetPerfDataFiles = False 586 self.callGetPerfReportFiles = False 587 self.callProcessResults = False 588 589 self.result.GetResultsDir = FakeGetResultsDir 590 self.result.GetResultsFile = FakeGetResultsFile 591 self.result.GetPerfDataFiles = FakeGetPerfDataFiles 592 self.result.GeneratePerfReportFiles = FakeGetPerfReportFiles 593 self.result.ProcessResults = FakeProcessResults 594 595 self.result.PopulateFromRun(OUTPUT, '', 0, 'test', 'telemetry_Crosperf') 596 self.assertTrue(self.callGetResultsDir) 597 self.assertTrue(self.callGetResultsFile) 598 self.assertTrue(self.callGetPerfDataFiles) 599 self.assertTrue(self.callGetPerfReportFiles) 600 self.assertTrue(self.callProcessResults) 601 602 def test_process_results(self): 603 604 def FakeGetKeyvals(show_all=False): 605 if show_all: 606 return {'first_time': 680, 'Total': 10} 607 else: 608 return {'Total': 10} 609 610 def FakeGatherPerfResults(): 611 self.callGatherPerfResults = True 612 613 self.callGatherPerfResults = False 614 615 self.result.GetKeyvals = FakeGetKeyvals 616 self.result.GatherPerfResults = FakeGatherPerfResults 617 618 self.result.retval = 0 619 self.result.ProcessResults() 620 self.assertTrue(self.callGatherPerfResults) 621 self.assertEqual(len(self.result.keyvals), 2) 622 self.assertEqual(self.result.keyvals, {'Total': 10, 'retval': 0}) 623 624 self.result.retval = 1 625 self.result.ProcessResults() 626 self.assertEqual(len(self.result.keyvals), 2) 627 self.assertEqual(self.result.keyvals, {'Total': 10, 'retval': 1}) 628 629 @mock.patch.object(misc, 'GetInsideChrootPath') 630 @mock.patch.object(command_executer.CommandExecuter, 631 'ChrootRunCommandWOutput') 632 def test_populate_from_cache_dir(self, mock_runchrootcmd, mock_getpath): 633 634 # pylint: disable=redefined-builtin 635 def FakeMkdtemp(dir=None): 636 if dir: 637 pass 638 return self.tmpdir 639 640 current_path = os.getcwd() 641 cache_dir = os.path.join(current_path, 'test_cache/test_input') 642 self.result.ce = command_executer.GetCommandExecuter(log_level='average') 643 self.result.ce.ChrootRunCommandWOutput = mock_runchrootcmd 644 mock_runchrootcmd.return_value = [ 645 '', ('%s,PASS\n%s/\telemetry_Crosperf,PASS\n') % (TMP_DIR1, TMP_DIR1), 646 '' 647 ] 648 mock_getpath.return_value = TMP_DIR1 649 self.tmpdir = tempfile.mkdtemp() 650 save_real_mkdtemp = tempfile.mkdtemp 651 tempfile.mkdtemp = FakeMkdtemp 652 653 self.result.PopulateFromCacheDir(cache_dir, 'sunspider', 654 'telemetry_Crosperf') 655 self.assertEqual(self.result.keyvals, { 656 u'Total__Total': [444.0, u'ms'], 657 u'regexp-dna__regexp-dna': [16.2, u'ms'], 658 u'telemetry_page_measurement_results__num_failed': [0, u'count'], 659 u'telemetry_page_measurement_results__num_errored': [0, u'count'], 660 u'string-fasta__string-fasta': [23.2, u'ms'], 661 u'crypto-sha1__crypto-sha1': [11.6, u'ms'], 662 u'bitops-3bit-bits-in-byte__bitops-3bit-bits-in-byte': [3.2, u'ms'], 663 u'access-nsieve__access-nsieve': [7.9, u'ms'], 664 u'bitops-nsieve-bits__bitops-nsieve-bits': [9.4, u'ms'], 665 u'string-validate-input__string-validate-input': [19.3, u'ms'], 666 u'3d-raytrace__3d-raytrace': [24.7, u'ms'], 667 u'3d-cube__3d-cube': [28.0, u'ms'], 668 u'string-unpack-code__string-unpack-code': [46.7, u'ms'], 669 u'date-format-tofte__date-format-tofte': [26.3, u'ms'], 670 u'math-partial-sums__math-partial-sums': [22.0, u'ms'], 671 '\telemetry_Crosperf': ['PASS', ''], 672 u'crypto-aes__crypto-aes': [15.2, u'ms'], 673 u'bitops-bitwise-and__bitops-bitwise-and': [8.4, u'ms'], 674 u'crypto-md5__crypto-md5': [10.5, u'ms'], 675 u'string-tagcloud__string-tagcloud': [52.8, u'ms'], 676 u'access-nbody__access-nbody': [8.5, u'ms'], 677 'retval': 0, 678 u'math-spectral-norm__math-spectral-norm': [6.6, u'ms'], 679 u'math-cordic__math-cordic': [8.7, u'ms'], 680 u'access-binary-trees__access-binary-trees': [4.5, u'ms'], 681 u'controlflow-recursive__controlflow-recursive': [4.4, u'ms'], 682 u'access-fannkuch__access-fannkuch': [17.8, u'ms'], 683 u'string-base64__string-base64': [16.0, u'ms'], 684 u'date-format-xparb__date-format-xparb': [20.9, u'ms'], 685 u'3d-morph__3d-morph': [22.1, u'ms'], 686 u'bitops-bits-in-byte__bitops-bits-in-byte': [9.1, u'ms'] 687 }) 688 689 # Clean up after test. 690 tempfile.mkdtemp = save_real_mkdtemp 691 command = 'rm -Rf %s' % self.tmpdir 692 self.result.ce.RunCommand(command) 693 694 @mock.patch.object(misc, 'GetRoot') 695 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 696 def test_cleanup(self, mock_runcmd, mock_getroot): 697 698 # Test 1. 'rm_chroot_tmp' is True; self.results_dir exists; 699 # self.temp_dir exists; results_dir name contains 'test_that_results_'. 700 mock_getroot.return_value = ['/tmp/tmp_AbcXyz', 'test_that_results_fake'] 701 self.result.ce.RunCommand = mock_runcmd 702 self.result.results_dir = 'test_results_dir' 703 self.result.temp_dir = 'testtemp_dir' 704 self.result.CleanUp(True) 705 self.assertEqual(mock_getroot.call_count, 1) 706 self.assertEqual(mock_runcmd.call_count, 2) 707 self.assertEqual(mock_runcmd.call_args_list[0][0], 708 ('rm -rf test_results_dir',)) 709 self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) 710 711 # Test 2. Same, except ath results_dir name does not contain 712 # 'test_that_results_' 713 mock_getroot.reset_mock() 714 mock_runcmd.reset_mock() 715 mock_getroot.return_value = ['/tmp/tmp_AbcXyz', 'other_results_fake'] 716 self.result.ce.RunCommand = mock_runcmd 717 self.result.results_dir = 'test_results_dir' 718 self.result.temp_dir = 'testtemp_dir' 719 self.result.CleanUp(True) 720 self.assertEqual(mock_getroot.call_count, 1) 721 self.assertEqual(mock_runcmd.call_count, 2) 722 self.assertEqual(mock_runcmd.call_args_list[0][0], 723 ('rm -rf /tmp/tmp_AbcXyz',)) 724 self.assertEqual(mock_runcmd.call_args_list[1][0], ('rm -rf testtemp_dir',)) 725 726 # Test 3. mock_getroot returns nothing; 'rm_chroot_tmp' is False. 727 mock_getroot.reset_mock() 728 mock_runcmd.reset_mock() 729 self.result.CleanUp(False) 730 self.assertEqual(mock_getroot.call_count, 0) 731 self.assertEqual(mock_runcmd.call_count, 1) 732 self.assertEqual(mock_runcmd.call_args_list[0][0], ('rm -rf testtemp_dir',)) 733 734 # Test 4. 'rm_chroot_tmp' is True, but result_dir & temp_dir are None. 735 mock_getroot.reset_mock() 736 mock_runcmd.reset_mock() 737 self.result.results_dir = None 738 self.result.temp_dir = None 739 self.result.CleanUp(True) 740 self.assertEqual(mock_getroot.call_count, 0) 741 self.assertEqual(mock_runcmd.call_count, 0) 742 743 @mock.patch.object(misc, 'GetInsideChrootPath') 744 @mock.patch.object(command_executer.CommandExecuter, 'ChrootRunCommand') 745 def test_store_to_cache_dir(self, mock_chrootruncmd, mock_getpath): 746 747 def FakeMkdtemp(directory=''): 748 if directory: 749 pass 750 return self.tmpdir 751 752 if mock_chrootruncmd or mock_getpath: 753 pass 754 current_path = os.getcwd() 755 cache_dir = os.path.join(current_path, 'test_cache/test_output') 756 757 self.result.ce = command_executer.GetCommandExecuter(log_level='average') 758 self.result.out = OUTPUT 759 self.result.err = error 760 self.result.retval = 0 761 self.tmpdir = tempfile.mkdtemp() 762 if not os.path.exists(self.tmpdir): 763 os.makedirs(self.tmpdir) 764 self.result.results_dir = os.path.join(os.getcwd(), 'test_cache') 765 save_real_mkdtemp = tempfile.mkdtemp 766 tempfile.mkdtemp = FakeMkdtemp 767 768 mock_mm = machine_manager.MockMachineManager('/tmp/chromeos_root', 0, 769 'average', '') 770 mock_mm.machine_checksum_string['mock_label'] = 'fake_machine_checksum123' 771 772 mock_keylist = ['key1', 'key2', 'key3'] 773 test_flag.SetTestMode(True) 774 self.result.StoreToCacheDir(cache_dir, mock_mm, mock_keylist) 775 776 # Check that the correct things were written to the 'cache'. 777 test_dir = os.path.join(os.getcwd(), 'test_cache/test_output') 778 base_dir = os.path.join(os.getcwd(), 'test_cache/compare_output') 779 self.assertTrue(os.path.exists(os.path.join(test_dir, 'autotest.tbz2'))) 780 self.assertTrue(os.path.exists(os.path.join(test_dir, 'machine.txt'))) 781 self.assertTrue(os.path.exists(os.path.join(test_dir, 'results.txt'))) 782 783 f1 = os.path.join(test_dir, 'machine.txt') 784 f2 = os.path.join(base_dir, 'machine.txt') 785 cmd = 'diff %s %s' % (f1, f2) 786 [_, out, _] = self.result.ce.RunCommandWOutput(cmd) 787 self.assertEqual(len(out), 0) 788 789 f1 = os.path.join(test_dir, 'results.txt') 790 f2 = os.path.join(base_dir, 'results.txt') 791 cmd = 'diff %s %s' % (f1, f2) 792 [_, out, _] = self.result.ce.RunCommandWOutput(cmd) 793 self.assertEqual(len(out), 0) 794 795 # Clean up after test. 796 tempfile.mkdtemp = save_real_mkdtemp 797 command = 'rm %s/*' % test_dir 798 self.result.ce.RunCommand(command) 799 800 801 TELEMETRY_RESULT_KEYVALS = { 802 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 803 'math-cordic (ms)': 804 '11.4', 805 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 806 'access-nbody (ms)': 807 '6.9', 808 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 809 'access-fannkuch (ms)': 810 '26.3', 811 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 812 'math-spectral-norm (ms)': 813 '6.3', 814 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 815 'bitops-nsieve-bits (ms)': 816 '9.3', 817 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 818 'math-partial-sums (ms)': 819 '32.8', 820 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 821 'regexp-dna (ms)': 822 '16.1', 823 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 824 '3d-cube (ms)': 825 '42.7', 826 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 827 'crypto-md5 (ms)': 828 '10.8', 829 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 830 'crypto-sha1 (ms)': 831 '12.4', 832 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 833 'string-tagcloud (ms)': 834 '47.2', 835 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 836 'string-fasta (ms)': 837 '36.3', 838 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 839 'access-binary-trees (ms)': 840 '7.3', 841 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 842 'date-format-xparb (ms)': 843 '138.1', 844 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 845 'crypto-aes (ms)': 846 '19.2', 847 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 848 'Total (ms)': 849 '656.5', 850 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 851 'string-base64 (ms)': 852 '17.5', 853 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 854 'string-validate-input (ms)': 855 '24.8', 856 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 857 '3d-raytrace (ms)': 858 '28.7', 859 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 860 'controlflow-recursive (ms)': 861 '5.3', 862 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 863 'bitops-bits-in-byte (ms)': 864 '9.8', 865 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 866 '3d-morph (ms)': 867 '50.2', 868 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 869 'bitops-bitwise-and (ms)': 870 '8.8', 871 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 872 'access-nsieve (ms)': 873 '8.6', 874 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 875 'date-format-tofte (ms)': 876 '31.2', 877 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 878 'bitops-3bit-bits-in-byte (ms)': 879 '3.5', 880 'retval': 881 0, 882 'http://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html ' 883 'string-unpack-code (ms)': 884 '45.0' 885 } 886 887 PURE_TELEMETRY_OUTPUT = """ 888 page_name,3d-cube (ms),3d-morph (ms),3d-raytrace (ms),Total (ms),access-binary-trees (ms),access-fannkuch (ms),access-nbody (ms),access-nsieve (ms),bitops-3bit-bits-in-byte (ms),bitops-bits-in-byte (ms),bitops-bitwise-and (ms),bitops-nsieve-bits (ms),controlflow-recursive (ms),crypto-aes (ms),crypto-md5 (ms),crypto-sha1 (ms),date-format-tofte (ms),date-format-xparb (ms),math-cordic (ms),math-partial-sums (ms),math-spectral-norm (ms),regexp-dna (ms),string-base64 (ms),string-fasta (ms),string-tagcloud (ms),string-unpack-code (ms),string-validate-input (ms)\r\nhttp://www.webkit.org/perf/sunspider-1.0.2/sunspider-1.0.2/driver.html,42.7,50.2,28.7,656.5,7.3,26.3,6.9,8.6,3.5,9.8,8.8,9.3,5.3,19.2,10.8,12.4,31.2,138.1,11.4,32.8,6.3,16.1,17.5,36.3,47.2,45.0,24.8\r 889 """ 890 891 892 class TelemetryResultTest(unittest.TestCase): 893 """Telemetry result test.""" 894 895 def __init__(self, *args, **kwargs): 896 super(TelemetryResultTest, self).__init__(*args, **kwargs) 897 self.callFakeProcessResults = False 898 self.result = None 899 self.mock_logger = mock.Mock(spec=logger.Logger) 900 self.mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 901 self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', 902 '/tmp', 'lumpy', 'remote', 'image_args', 903 'cache_dir', 'average', 'gcc', None) 904 self.mock_machine = machine_manager.MockCrosMachine( 905 'falco.cros', '/tmp/chromeos', 'average') 906 907 def test_populate_from_run(self): 908 909 def FakeProcessResults(): 910 self.callFakeProcessResults = True 911 912 self.callFakeProcessResults = False 913 self.result = TelemetryResult(self.mock_logger, self.mock_label, 'average', 914 self.mock_cmd_exec) 915 self.result.ProcessResults = FakeProcessResults 916 self.result.PopulateFromRun(OUTPUT, error, 3, 'fake_test', 917 'telemetry_Crosperf') 918 self.assertTrue(self.callFakeProcessResults) 919 self.assertEqual(self.result.out, OUTPUT) 920 self.assertEqual(self.result.err, error) 921 self.assertEqual(self.result.retval, 3) 922 923 def test_populate_from_cache_dir_and_process_results(self): 924 925 self.result = TelemetryResult(self.mock_logger, self.mock_label, 'average', 926 self.mock_machine) 927 current_path = os.getcwd() 928 cache_dir = os.path.join(current_path, 929 'test_cache/test_puretelemetry_input') 930 self.result.PopulateFromCacheDir(cache_dir, '', '') 931 self.assertEqual(self.result.out.strip(), PURE_TELEMETRY_OUTPUT.strip()) 932 self.assertEqual(self.result.err, '') 933 self.assertEqual(self.result.retval, 0) 934 self.assertEqual(self.result.keyvals, TELEMETRY_RESULT_KEYVALS) 935 936 937 class ResultsCacheTest(unittest.TestCase): 938 """Resultcache test class.""" 939 940 def __init__(self, *args, **kwargs): 941 super(ResultsCacheTest, self).__init__(*args, **kwargs) 942 self.fakeCacheReturnResult = None 943 self.mock_logger = mock.Mock(spec=logger.Logger) 944 self.mock_label = MockLabel('mock_label', 'chromeos_image', 'autotest_dir', 945 '/tmp', 'lumpy', 'remote', 'image_args', 946 'cache_dir', 'average', 'gcc', None) 947 948 def setUp(self): 949 self.results_cache = ResultsCache() 950 951 mock_machine = machine_manager.MockCrosMachine('falco.cros', 952 '/tmp/chromeos', 'average') 953 954 mock_mm = machine_manager.MockMachineManager('/tmp/chromeos_root', 0, 955 'average', '') 956 mock_mm.machine_checksum_string['mock_label'] = 'fake_machine_checksum123' 957 958 self.results_cache.Init( 959 self.mock_label.chromeos_image, 960 self.mock_label.chromeos_root, 961 'sunspider', 962 1, # benchmark_run.iteration, 963 '', # benchmark_run.test_args, 964 '', # benchmark_run.profiler_args, 965 mock_mm, 966 mock_machine, 967 self.mock_label.board, 968 [CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH], 969 self.mock_logger, 970 'average', 971 self.mock_label, 972 '', # benchmark_run.share_cache 973 'telemetry_Crosperf', 974 True, # benchmark_run.show_all_results 975 False) # benchmark_run.run_local 976 977 @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') 978 def test_get_cache_dir_for_write(self, mock_checksum): 979 980 def FakeGetMachines(label): 981 if label: 982 pass 983 m1 = machine_manager.MockCrosMachine( 984 'lumpy1.cros', self.results_cache.chromeos_root, 'average') 985 m2 = machine_manager.MockCrosMachine( 986 'lumpy2.cros', self.results_cache.chromeos_root, 'average') 987 return [m1, m2] 988 989 mock_checksum.return_value = 'FakeImageChecksumabc123' 990 self.results_cache.machine_manager.GetMachines = FakeGetMachines 991 self.results_cache.machine_manager.machine_checksum['mock_label'] = \ 992 'FakeMachineChecksumabc987' 993 # Based on the label, benchmark and machines, get the directory in which 994 # to store the cache information for this test run. 995 result_path = self.results_cache.GetCacheDirForWrite() 996 # Verify that the returned directory is correct (since the label 997 # contained a cache_dir, named 'cache_dir', that's what is expected in 998 # the result, rather than '~/cros_scratch'). 999 comp_path = os.path.join(os.getcwd(), 1000 'cache_dir/54524606abaae4fdf7b02f49f7ae7127_' 1001 'sunspider_1_fda29412ceccb72977516c4785d08e2c_' 1002 'FakeImageChecksumabc123_FakeMachineChecksum' 1003 'abc987__6') 1004 self.assertEqual(result_path, comp_path) 1005 1006 def test_form_cache_dir(self): 1007 # This is very similar to the previous test (FormCacheDir is called 1008 # from GetCacheDirForWrite). 1009 cache_key_list = ('54524606abaae4fdf7b02f49f7ae7127', 'sunspider', '1', 1010 '7215ee9c7d9dc229d2921a40e899ec5f', 1011 'FakeImageChecksumabc123', '*', '*', '6') 1012 path = self.results_cache.FormCacheDir(cache_key_list) 1013 self.assertEqual(len(path), 1) 1014 path1 = path[0] 1015 test_dirname = ('54524606abaae4fdf7b02f49f7ae7127_sunspider_1_7215ee9' 1016 'c7d9dc229d2921a40e899ec5f_FakeImageChecksumabc123_*_*_6') 1017 comp_path = os.path.join(os.getcwd(), 'cache_dir', test_dirname) 1018 self.assertEqual(path1, comp_path) 1019 1020 @mock.patch.object(image_checksummer.ImageChecksummer, 'Checksum') 1021 def test_get_cache_key_list(self, mock_checksum): 1022 # This tests the mechanism that generates the various pieces of the 1023 # cache directory name, based on various conditions. 1024 1025 def FakeGetMachines(label): 1026 if label: 1027 pass 1028 m1 = machine_manager.MockCrosMachine( 1029 'lumpy1.cros', self.results_cache.chromeos_root, 'average') 1030 m2 = machine_manager.MockCrosMachine( 1031 'lumpy2.cros', self.results_cache.chromeos_root, 'average') 1032 return [m1, m2] 1033 1034 mock_checksum.return_value = 'FakeImageChecksumabc123' 1035 self.results_cache.machine_manager.GetMachines = FakeGetMachines 1036 self.results_cache.machine_manager.machine_checksum['mock_label'] = \ 1037 'FakeMachineChecksumabc987' 1038 1039 # Test 1. Generating cache name for reading (not writing). 1040 key_list = self.results_cache.GetCacheKeyList(True) 1041 self.assertEqual(key_list[0], '*') # Machine checksum value, for read. 1042 self.assertEqual(key_list[1], 'sunspider') 1043 self.assertEqual(key_list[2], '1') 1044 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1045 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1046 self.assertEqual(key_list[5], '*') 1047 self.assertEqual(key_list[6], '*') 1048 self.assertEqual(key_list[7], '6') 1049 1050 # Test 2. Generating cache name for writing, with local image type. 1051 key_list = self.results_cache.GetCacheKeyList(False) 1052 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1053 self.assertEqual(key_list[1], 'sunspider') 1054 self.assertEqual(key_list[2], '1') 1055 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1056 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1057 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1058 self.assertEqual(key_list[6], '') 1059 self.assertEqual(key_list[7], '6') 1060 1061 # Test 3. Generating cache name for writing, with trybot image type. 1062 self.results_cache.label.image_type = 'trybot' 1063 key_list = self.results_cache.GetCacheKeyList(False) 1064 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1065 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1066 self.assertEqual(key_list[4], '54524606abaae4fdf7b02f49f7ae7127') 1067 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1068 1069 # Test 4. Generating cache name for writing, with official image type. 1070 self.results_cache.label.image_type = 'official' 1071 key_list = self.results_cache.GetCacheKeyList(False) 1072 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1073 self.assertEqual(key_list[1], 'sunspider') 1074 self.assertEqual(key_list[2], '1') 1075 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1076 self.assertEqual(key_list[4], '*') 1077 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1078 self.assertEqual(key_list[6], '') 1079 self.assertEqual(key_list[7], '6') 1080 1081 # Test 5. Generating cache name for writing, with local image type, and 1082 # specifying that the image path must match the cached image path. 1083 self.results_cache.label.image_type = 'local' 1084 self.results_cache.cache_conditions.append(CacheConditions.IMAGE_PATH_MATCH) 1085 key_list = self.results_cache.GetCacheKeyList(False) 1086 self.assertEqual(key_list[0], '54524606abaae4fdf7b02f49f7ae7127') 1087 self.assertEqual(key_list[3], 'fda29412ceccb72977516c4785d08e2c') 1088 self.assertEqual(key_list[4], 'FakeImageChecksumabc123') 1089 self.assertEqual(key_list[5], 'FakeMachineChecksumabc987') 1090 1091 @mock.patch.object(command_executer.CommandExecuter, 'RunCommand') 1092 @mock.patch.object(os.path, 'isdir') 1093 @mock.patch.object(Result, 'CreateFromCacheHit') 1094 def test_read_result(self, mock_create, mock_isdir, mock_runcmd): 1095 1096 self.fakeCacheReturnResult = None 1097 1098 def FakeGetCacheDirForRead(): 1099 return self.fakeCacheReturnResult 1100 1101 def FakeGetCacheDirForWrite(): 1102 return self.fakeCacheReturnResult 1103 1104 mock_cmd_exec = mock.Mock(spec=command_executer.CommandExecuter) 1105 fake_result = Result(self.mock_logger, self.mock_label, 'average', 1106 mock_cmd_exec) 1107 fake_result.retval = 0 1108 1109 # Set up results_cache _GetCacheDirFor{Read,Write} to return 1110 # self.fakeCacheReturnResult, which is initially None (see above). 1111 # So initially, no cache dir is returned. 1112 self.results_cache.GetCacheDirForRead = FakeGetCacheDirForRead 1113 self.results_cache.GetCacheDirForWrite = FakeGetCacheDirForWrite 1114 1115 mock_isdir.return_value = True 1116 save_cc = [ 1117 CacheConditions.CACHE_FILE_EXISTS, CacheConditions.CHECKSUMS_MATCH 1118 ] 1119 self.results_cache.cache_conditions.append(CacheConditions.FALSE) 1120 1121 # Test 1. CacheCondition.FALSE, which means do not read from the cache. 1122 # (force re-running of test). Result should be None. 1123 res = self.results_cache.ReadResult() 1124 self.assertIsNone(res) 1125 self.assertEqual(mock_runcmd.call_count, 1) 1126 1127 # Test 2. Remove CacheCondition.FALSE. Result should still be None, 1128 # because GetCacheDirForRead is returning None at the moment. 1129 mock_runcmd.reset_mock() 1130 self.results_cache.cache_conditions = save_cc 1131 res = self.results_cache.ReadResult() 1132 self.assertIsNone(res) 1133 self.assertEqual(mock_runcmd.call_count, 0) 1134 1135 # Test 3. Now set up cache dir to be returned by GetCacheDirForRead. 1136 # Since cache_dir is found, will call Result.CreateFromCacheHit, which 1137 # which will actually all our mock_create and should return fake_result. 1138 self.fakeCacheReturnResult = 'fake/cache/dir' 1139 mock_create.return_value = fake_result 1140 res = self.results_cache.ReadResult() 1141 self.assertEqual(mock_runcmd.call_count, 0) 1142 self.assertEqual(res, fake_result) 1143 1144 # Test 4. os.path.isdir(cache_dir) will now return false, so result 1145 # should be None again (no cache found). 1146 mock_isdir.return_value = False 1147 res = self.results_cache.ReadResult() 1148 self.assertEqual(mock_runcmd.call_count, 0) 1149 self.assertIsNone(res) 1150 1151 # Test 5. os.path.isdir returns true, but mock_create now returns None 1152 # (the call to CreateFromCacheHit returns None), so overal result is None. 1153 mock_isdir.return_value = True 1154 mock_create.return_value = None 1155 res = self.results_cache.ReadResult() 1156 self.assertEqual(mock_runcmd.call_count, 0) 1157 self.assertIsNone(res) 1158 1159 # Test 6. Everything works 'as expected', result should be fake_result. 1160 mock_create.return_value = fake_result 1161 res = self.results_cache.ReadResult() 1162 self.assertEqual(mock_runcmd.call_count, 0) 1163 self.assertEqual(res, fake_result) 1164 1165 # Test 7. The run failed; result should be None. 1166 mock_create.return_value = fake_result 1167 fake_result.retval = 1 1168 self.results_cache.cache_conditions.append(CacheConditions.RUN_SUCCEEDED) 1169 res = self.results_cache.ReadResult() 1170 self.assertEqual(mock_runcmd.call_count, 0) 1171 self.assertIsNone(res) 1172 1173 1174 if __name__ == '__main__': 1175 unittest.main() 1176
