1 #!/usr/bin/env python2 2 # 3 # Copyright 2016 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 """Test for generate_report.py.""" 7 8 from __future__ import division 9 from __future__ import print_function 10 11 from StringIO import StringIO 12 13 import copy 14 import json 15 import mock 16 import test_flag 17 import unittest 18 19 import generate_report 20 import results_report 21 22 23 class _ContextualStringIO(StringIO): 24 """StringIO that can be used in `with` statements.""" 25 26 def __init__(self, *args): 27 StringIO.__init__(self, *args) 28 29 def __enter__(self): 30 return self 31 32 def __exit__(self, _type, _value, _traceback): 33 pass 34 35 36 class GenerateReportTests(unittest.TestCase): 37 """Tests for generate_report.py.""" 38 39 def testCountBenchmarks(self): 40 runs = { 41 'foo': [[{}, {}, {}], [{}, {}, {}, {}]], 42 'bar': [], 43 'baz': [[], [{}], [{}, {}, {}]] 44 } 45 results = generate_report.CountBenchmarks(runs) 46 expected_results = [('foo', 4), ('bar', 0), ('baz', 3)] 47 self.assertItemsEqual(expected_results, results) 48 49 def testCutResultsInPlace(self): 50 bench_data = { 51 'foo': [[{ 52 'a': 1, 53 'b': 2, 54 'c': 3 55 }, { 56 'a': 3, 57 'b': 2.5, 58 'c': 1 59 }]], 60 'bar': [[{ 61 'd': 11, 62 'e': 12, 63 'f': 13 64 }]], 65 'baz': [[{ 66 'g': 12, 67 'h': 13 68 }]], 69 'qux': [[{ 70 'i': 11 71 }]], 72 } 73 original_bench_data = copy.deepcopy(bench_data) 74 75 max_keys = 2 76 results = generate_report.CutResultsInPlace( 77 bench_data, max_keys=max_keys, complain_on_update=False) 78 # Cuts should be in-place. 79 self.assertIs(results, bench_data) 80 self.assertItemsEqual(original_bench_data.keys(), bench_data.keys()) 81 for bench_name, original_runs in original_bench_data.iteritems(): 82 bench_runs = bench_data[bench_name] 83 self.assertEquals(len(original_runs), len(bench_runs)) 84 # Order of these sub-lists shouldn't have changed. 85 for original_list, new_list in zip(original_runs, bench_runs): 86 self.assertEqual(len(original_list), len(new_list)) 87 for original_keyvals, sub_keyvals in zip(original_list, new_list): 88 # sub_keyvals must be a subset of original_keyvals 89 self.assertDictContainsSubset(sub_keyvals, original_keyvals) 90 91 def testCutResultsInPlaceLeavesRetval(self): 92 bench_data = { 93 'foo': [[{ 94 'retval': 0, 95 'a': 1 96 }]], 97 'bar': [[{ 98 'retval': 1 99 }]], 100 'baz': [[{ 101 'RETVAL': 1 102 }]], 103 } 104 results = generate_report.CutResultsInPlace( 105 bench_data, max_keys=0, complain_on_update=False) 106 # Just reach into results assuming we know it otherwise outputs things 107 # sanely. If it doesn't, testCutResultsInPlace should give an indication as 108 # to what, exactly, is broken. 109 self.assertEqual(results['foo'][0][0].items(), [('retval', 0)]) 110 self.assertEqual(results['bar'][0][0].items(), [('retval', 1)]) 111 self.assertEqual(results['baz'][0][0].items(), []) 112 113 def _RunMainWithInput(self, args, input_obj): 114 assert '-i' not in args 115 args += ['-i', '-'] 116 input_buf = _ContextualStringIO(json.dumps(input_obj)) 117 with mock.patch('generate_report.PickInputFile', return_value=input_buf) \ 118 as patched_pick: 119 result = generate_report.Main(args) 120 patched_pick.assert_called_once_with('-') 121 return result 122 123 @mock.patch('generate_report.RunActions') 124 def testMain(self, mock_run_actions): 125 # Email is left out because it's a bit more difficult to test, and it'll be 126 # mildly obvious if it's failing. 127 args = ['--json', '--html', '--text'] 128 return_code = self._RunMainWithInput(args, {'platforms': [], 'data': {}}) 129 self.assertEqual(0, return_code) 130 self.assertEqual(mock_run_actions.call_count, 1) 131 ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] 132 self.assertItemsEqual(ctors, [ 133 results_report.JSONResultsReport, 134 results_report.TextResultsReport, 135 results_report.HTMLResultsReport, 136 ]) 137 138 @mock.patch('generate_report.RunActions') 139 def testMainSelectsHTMLIfNoReportsGiven(self, mock_run_actions): 140 args = [] 141 return_code = self._RunMainWithInput(args, {'platforms': [], 'data': {}}) 142 self.assertEqual(0, return_code) 143 self.assertEqual(mock_run_actions.call_count, 1) 144 ctors = [ctor for ctor, _ in mock_run_actions.call_args[0][0]] 145 self.assertItemsEqual(ctors, [results_report.HTMLResultsReport]) 146 147 # We only mock print_exc so we don't have exception info printed to stdout. 148 @mock.patch('generate_report.WriteFile', side_effect=ValueError('Oh noo')) 149 @mock.patch('traceback.print_exc') 150 def testRunActionsRunsAllActionsRegardlessOfExceptions( 151 self, mock_print_exc, mock_write_file): 152 actions = [(None, 'json'), (None, 'html'), (None, 'text'), (None, 'email')] 153 output_prefix = '-' 154 ok = generate_report.RunActions( 155 actions, {}, output_prefix, overwrite=False, verbose=False) 156 self.assertFalse(ok) 157 self.assertEqual(mock_write_file.call_count, len(actions)) 158 self.assertEqual(mock_print_exc.call_count, len(actions)) 159 160 @mock.patch('generate_report.WriteFile') 161 def testRunActionsReturnsTrueIfAllActionsSucceed(self, mock_write_file): 162 actions = [(None, 'json'), (None, 'html'), (None, 'text')] 163 output_prefix = '-' 164 ok = generate_report.RunActions( 165 actions, {}, output_prefix, overwrite=False, verbose=False) 166 self.assertEqual(mock_write_file.call_count, len(actions)) 167 self.assertTrue(ok) 168 169 170 if __name__ == '__main__': 171 test_flag.SetTestMode(True) 172 unittest.main() 173
