1 #!/usr/bin/env python 2 # Copyright 2014 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 7 """Unit tests for instrumentation.TestRunner.""" 8 9 # pylint: disable=W0212 10 11 import os 12 import sys 13 import unittest 14 15 from pylib import constants 16 from pylib.base import base_test_result 17 from pylib.instrumentation import test_runner 18 19 sys.path.append(os.path.join( 20 constants.DIR_SOURCE_ROOT, 'third_party', 'pymock')) 21 import mock # pylint: disable=F0401 22 23 24 class InstrumentationTestRunnerTest(unittest.TestCase): 25 26 def setUp(self): 27 options = mock.Mock() 28 options.tool = '' 29 package = mock.Mock() 30 self.instance = test_runner.TestRunner(options, None, 0, package) 31 32 def testParseAmInstrumentRawOutput_nothing(self): 33 code, result, statuses = ( 34 test_runner.TestRunner._ParseAmInstrumentRawOutput([''])) 35 self.assertEqual(None, code) 36 self.assertEqual([], result) 37 self.assertEqual([], statuses) 38 39 def testParseAmInstrumentRawOutput_noMatchingStarts(self): 40 raw_output = [ 41 '', 42 'this.is.a.test.package.TestClass:.', 43 'Test result for =.', 44 'Time: 1.234', 45 '', 46 'OK (1 test)', 47 ] 48 49 code, result, statuses = ( 50 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) 51 self.assertEqual(None, code) 52 self.assertEqual([], result) 53 self.assertEqual([], statuses) 54 55 def testParseAmInstrumentRawOutput_resultAndCode(self): 56 raw_output = [ 57 'INSTRUMENTATION_RESULT: foo', 58 'bar', 59 'INSTRUMENTATION_CODE: -1', 60 ] 61 62 code, result, _ = ( 63 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) 64 self.assertEqual(-1, code) 65 self.assertEqual(['foo', 'bar'], result) 66 67 def testParseAmInstrumentRawOutput_oneStatus(self): 68 raw_output = [ 69 'INSTRUMENTATION_STATUS: foo=1', 70 'INSTRUMENTATION_STATUS: bar=hello', 71 'INSTRUMENTATION_STATUS: world=false', 72 'INSTRUMENTATION_STATUS: class=this.is.a.test.package.TestClass', 73 'INSTRUMENTATION_STATUS: test=testMethod', 74 'INSTRUMENTATION_STATUS_CODE: 0', 75 ] 76 77 _, _, statuses = ( 78 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) 79 80 expected = [ 81 (0, { 82 'foo': ['1'], 83 'bar': ['hello'], 84 'world': ['false'], 85 'class': ['this.is.a.test.package.TestClass'], 86 'test': ['testMethod'], 87 }) 88 ] 89 self.assertEqual(expected, statuses) 90 91 def testParseAmInstrumentRawOutput_multiStatus(self): 92 raw_output = [ 93 'INSTRUMENTATION_STATUS: class=foo', 94 'INSTRUMENTATION_STATUS: test=bar', 95 'INSTRUMENTATION_STATUS_CODE: 1', 96 'INSTRUMENTATION_STATUS: test_skipped=true', 97 'INSTRUMENTATION_STATUS_CODE: 0', 98 'INSTRUMENTATION_STATUS: class=hello', 99 'INSTRUMENTATION_STATUS: test=world', 100 'INSTRUMENTATION_STATUS: stack=', 101 'foo/bar.py (27)', 102 'hello/world.py (42)', 103 'test/file.py (1)', 104 'INSTRUMENTATION_STATUS_CODE: -1', 105 ] 106 107 _, _, statuses = ( 108 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) 109 110 expected = [ 111 (1, {'class': ['foo'], 'test': ['bar'],}), 112 (0, {'test_skipped': ['true']}), 113 (-1, { 114 'class': ['hello'], 115 'test': ['world'], 116 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)', 117 'test/file.py (1)'], 118 }), 119 ] 120 self.assertEqual(expected, statuses) 121 122 def testParseAmInstrumentRawOutput_statusResultAndCode(self): 123 raw_output = [ 124 'INSTRUMENTATION_STATUS: class=foo', 125 'INSTRUMENTATION_STATUS: test=bar', 126 'INSTRUMENTATION_STATUS_CODE: 1', 127 'INSTRUMENTATION_RESULT: hello', 128 'world', 129 '', 130 '', 131 'INSTRUMENTATION_CODE: 0', 132 ] 133 134 code, result, statuses = ( 135 test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output)) 136 137 self.assertEqual(0, code) 138 self.assertEqual(['hello', 'world', '', ''], result) 139 self.assertEqual([(1, {'class': ['foo'], 'test': ['bar']})], statuses) 140 141 def testGenerateTestResult_noStatus(self): 142 result = self.instance._GenerateTestResult( 143 'test.package.TestClass#testMethod', [], 0, 1000) 144 self.assertEqual('test.package.TestClass#testMethod', result.GetName()) 145 self.assertEqual(base_test_result.ResultType.UNKNOWN, result.GetType()) 146 self.assertEqual('', result.GetLog()) 147 self.assertEqual(1000, result.GetDur()) 148 149 def testGenerateTestResult_testPassed(self): 150 statuses = [ 151 (1, { 152 'class': ['test.package.TestClass'], 153 'test': ['testMethod'], 154 }), 155 (0, { 156 'class': ['test.package.TestClass'], 157 'test': ['testMethod'], 158 }), 159 ] 160 result = self.instance._GenerateTestResult( 161 'test.package.TestClass#testMethod', statuses, 0, 1000) 162 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) 163 164 def testGenerateTestResult_testSkipped_first(self): 165 statuses = [ 166 (0, { 167 'test_skipped': ['true'], 168 }), 169 (1, { 170 'class': ['test.package.TestClass'], 171 'test': ['testMethod'], 172 }), 173 (0, { 174 'class': ['test.package.TestClass'], 175 'test': ['testMethod'], 176 }), 177 ] 178 result = self.instance._GenerateTestResult( 179 'test.package.TestClass#testMethod', statuses, 0, 1000) 180 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) 181 182 def testGenerateTestResult_testSkipped_last(self): 183 statuses = [ 184 (1, { 185 'class': ['test.package.TestClass'], 186 'test': ['testMethod'], 187 }), 188 (0, { 189 'class': ['test.package.TestClass'], 190 'test': ['testMethod'], 191 }), 192 (0, { 193 'test_skipped': ['true'], 194 }), 195 ] 196 result = self.instance._GenerateTestResult( 197 'test.package.TestClass#testMethod', statuses, 0, 1000) 198 self.assertEqual(base_test_result.ResultType.SKIP, result.GetType()) 199 200 def testGenerateTestResult_testSkipped_false(self): 201 statuses = [ 202 (0, { 203 'test_skipped': ['false'], 204 }), 205 (1, { 206 'class': ['test.package.TestClass'], 207 'test': ['testMethod'], 208 }), 209 (0, { 210 'class': ['test.package.TestClass'], 211 'test': ['testMethod'], 212 }), 213 ] 214 result = self.instance._GenerateTestResult( 215 'test.package.TestClass#testMethod', statuses, 0, 1000) 216 self.assertEqual(base_test_result.ResultType.PASS, result.GetType()) 217 218 def testGenerateTestResult_testFailed(self): 219 statuses = [ 220 (1, { 221 'class': ['test.package.TestClass'], 222 'test': ['testMethod'], 223 }), 224 (-2, { 225 'class': ['test.package.TestClass'], 226 'test': ['testMethod'], 227 }), 228 ] 229 result = self.instance._GenerateTestResult( 230 'test.package.TestClass#testMethod', statuses, 0, 1000) 231 self.assertEqual(base_test_result.ResultType.FAIL, result.GetType()) 232 233 def testGenerateTestResult_testCrashed(self): 234 self.instance.test_pkg.GetPackageName = mock.Mock( 235 return_value='generate.test.result.test.package') 236 self.instance.device.old_interface.DismissCrashDialogIfNeeded = mock.Mock( 237 return_value='generate.test.result.test.package') 238 statuses = [ 239 (1, { 240 'class': ['test.package.TestClass'], 241 'test': ['testMethod'], 242 }), 243 (-1, { 244 'class': ['test.package.TestClass'], 245 'test': ['testMethod'], 246 'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)'], 247 }), 248 ] 249 result = self.instance._GenerateTestResult( 250 'test.package.TestClass#testMethod', statuses, 0, 1000) 251 self.assertEqual(base_test_result.ResultType.CRASH, result.GetType()) 252 self.assertEqual('\nfoo/bar.py (27)\nhello/world.py (42)', result.GetLog()) 253 254 def test_RunTest_verifyAdbShellCommand(self): 255 self.instance.options.test_runner = 'MyTestRunner' 256 self.instance.device.RunShellCommand = mock.Mock() 257 self.instance.test_pkg.GetPackageName = mock.Mock( 258 return_value='test.package') 259 self.instance._GetInstrumentationArgs = mock.Mock( 260 return_value={'test_arg_key': 'test_arg_value'}) 261 self.instance._RunTest('test.package.TestClass#testMethod', 100) 262 self.instance.device.RunShellCommand.assert_called_with( 263 ['am', 'instrument', '-r', 264 '-e', 'test_arg_key', "'test_arg_value'", 265 '-e', 'class', "'test.package.TestClass#testMethod'", 266 '-w', 'test.package/MyTestRunner'], 267 timeout=100, retries=0) 268 269 if __name__ == '__main__': 270 unittest.main(verbosity=2) 271 272
