1 #!/usr/bin/python 2 # 3 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Unit tests for server/cros/dynamic_suite/telemetry_runner.py.""" 8 import mox 9 import unittest 10 11 import common 12 13 from autotest_lib.server.cros import telemetry_runner 14 15 16 class TelemetryResultTest(mox.MoxTestBase): 17 """Unit tests for telemetry_runner.TelemetryResult.""" 18 19 SAMPLE_RESULT_LINES = ( 20 'RESULT average_commit_time_by_url: http___www.ebay.com= 8.86528 ms\n' 21 'RESULT CodeLoad: CodeLoad= 6343 score (bigger is better)\n' 22 'RESULT ai-astar: ai-astar= ' 23 '[614,527,523,471,530,523,577,625,614,538] ms\n' 24 'RESULT graph_name: test_name= {3.14, 0.98} units') 25 26 EXPECTED_PERF_DATA = [ 27 {'graph': 'average_commit_time_by_url', 'trace': 'http___www.ebay.com', 28 'units': 'ms', 'value': 8.86528}, 29 {'graph': 'CodeLoad', 'trace': 'CodeLoad', 30 'units': 'score__bigger_is_better_', 'value': 6343}, 31 {'graph': 'ai-astar', 'trace': 'ai-astar', 32 'units': 'ms', 'value': 554.2}, 33 {'graph': 'graph_name', 'trace': 'test_name', 34 'units': 'units', 'value': 3.14}] 35 36 37 def testEmptyStdout(self): 38 """Test when the test exits with 0 but there is no output.""" 39 result = telemetry_runner.TelemetryResult() 40 result.parse_benchmark_results() 41 self.assertEquals(result.status, telemetry_runner.FAILED_STATUS) 42 43 44 def testOnlyResultLines(self): 45 """Test when the stdout is only Result lines.""" 46 result = telemetry_runner.TelemetryResult( 47 exit_code=0, stdout=self.SAMPLE_RESULT_LINES) 48 result.parse_benchmark_results() 49 self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) 50 self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) 51 52 53 def testOnlyResultLinesWithWarnings(self): 54 """Test when the stderr has Warnings.""" 55 stdout = self.SAMPLE_RESULT_LINES 56 stderr = ('WARNING: Page failed to load http://www.facebook.com\n' 57 'WARNING: Page failed to load http://www.yahoo.com\n') 58 59 result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout, 60 stderr=stderr) 61 result.parse_benchmark_results() 62 self.assertEquals(result.status, telemetry_runner.WARNING_STATUS) 63 self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) 64 65 66 def testOnlyResultLinesWithWarningsAndTraceback(self): 67 """Test when the stderr has Warnings and Traceback.""" 68 stdout = self.SAMPLE_RESULT_LINES 69 stderr = ('WARNING: Page failed to load http://www.facebook.com\n' 70 'WARNING: Page failed to load http://www.yahoo.com\n' 71 'Traceback (most recent call last):\n' 72 'File "../../utils/unittest_suite.py", line 238, in ' 73 '<module>\n' 74 'main()') 75 76 result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout, 77 stderr=stderr) 78 result.parse_benchmark_results() 79 self.assertEquals(result.status, telemetry_runner.FAILED_STATUS) 80 self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) 81 82 83 def testInfoBeforeResultLines(self): 84 """Test when there is info before the Result lines.""" 85 stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' + 86 self.SAMPLE_RESULT_LINES) 87 stderr = 'WARNING: Page failed to load http://www.facebook.com\n' 88 89 result = telemetry_runner.TelemetryResult(exit_code=1, stdout=stdout, 90 stderr=stderr) 91 result.parse_benchmark_results() 92 self.assertEquals(result.status, telemetry_runner.WARNING_STATUS) 93 self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) 94 95 96 def testInfoAfterResultLines(self): 97 """Test when there is info after the Result lines.""" 98 stdout = (self.SAMPLE_RESULT_LINES + '\n' 99 'stderr:WARNING:root:Found (system), but you do not have ' 100 'a DISPLAY environment set.\n\n' 101 '04/16 12:51:23.312 DEBUG|telemetry_:0139|') 102 103 result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, 104 stderr='') 105 result.parse_benchmark_results() 106 self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) 107 self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) 108 109 110 def testInfoBeforeAndAfterResultLines(self): 111 """Test when there is info before and after the Result lines.""" 112 stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' + 113 self.SAMPLE_RESULT_LINES + '\n' 114 'stderr:WARNING:root:Found (system), but you do not have ' 115 'a DISPLAY environment set.\n\n' 116 '04/16 12:51:23.312 DEBUG|telemetry_:0139|') 117 118 result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, 119 stderr='') 120 result.parse_benchmark_results() 121 self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) 122 self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data) 123 124 125 def testNoResultLines(self): 126 """Test when Result lines are missing from stdout.""" 127 stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' 128 'stderr:WARNING:root:Found (system), but you do not have ' 129 'a DISPLAY environment set.\n\n' 130 '04/16 12:51:23.312 DEBUG|telemetry_:0139|') 131 132 result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, 133 stderr='') 134 result.parse_benchmark_results() 135 self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) 136 self.assertEquals([], result.perf_data) 137 138 139 def testBadCharactersInResultStringComponents(self): 140 """Test bad characters are cleaned up in RESULT string components.""" 141 stdout = ( 142 'RESULT average_commit_time_by_url!: ' 143 'http___www.^^ebay.com= 8.86528 ms\n' 144 'RESULT CodeLoad*: CodeLoad= 6343 score\n' 145 'RESULT ai-astar: ai-astar= ' 146 '[614,527,523,471,530,523,577,625,614,538] ~~ms\n' 147 'RESULT !!graph_name: &&test_name= {3.14, 0.98} units!') 148 expected_perf_data = [ 149 {'graph': 'average_commit_time_by_url_', 150 'trace': 'http___www.__ebay.com', 151 'units': 'ms', 'value': 8.86528}, 152 {'graph': 'CodeLoad_', 'trace': 'CodeLoad', 153 'units': 'score', 'value': 6343}, 154 {'graph': 'ai-astar', 'trace': 'ai-astar', 155 'units': '__ms', 'value': 554.2}, 156 {'graph': '__graph_name', 'trace': '__test_name', 157 'units': 'units_', 'value': 3.14}] 158 159 result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout, 160 stderr='') 161 result.parse_benchmark_results() 162 self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS) 163 self.assertEquals(expected_perf_data, result.perf_data) 164 165 166 def testCleanupUnitsString(self): 167 """Test that special characters in units strings are cleaned up.""" 168 result = telemetry_runner.TelemetryResult() 169 self.assertEquals(result._cleanup_units_string('score/unit'), 170 'score_per_unit') 171 self.assertEquals(result._cleanup_units_string('score / unit'), 172 'score__per__unit') 173 self.assertEquals(result._cleanup_units_string('%'), 174 'percent') 175 self.assertEquals(result._cleanup_units_string('unit%'), 176 'unitpercent') 177 self.assertEquals(result._cleanup_units_string('^^un!ts##'), 178 '__un_ts__') 179 self.assertEquals(result._cleanup_units_string('^^un!ts##/time %'), 180 '__un_ts___per_time_percent') 181 182 183 if __name__ == '__main__': 184 unittest.main() 185
