1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.ddmlib.testrunner.TestIdentifier; 20 import com.android.tradefed.config.ConfigurationException; 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.result.ITestInvocationListener; 23 import com.android.tradefed.util.CommandResult; 24 import com.android.tradefed.util.CommandStatus; 25 import com.android.tradefed.util.IRunUtil; 26 27 import junit.framework.TestCase; 28 29 import org.easymock.EasyMock; 30 31 import java.util.Map; 32 33 public class PythonUnitTestRunnerTest extends TestCase { 34 35 private static final String[] TEST_PASS_STDERR = { 36 "b (a) ... ok", "", PythonUnitTestResultParser.LINE, "Ran 1 tests in 1s", "", "OK", 37 }; 38 39 private static final String[] TEST_FAIL_STDERR = { 40 "b (a) ... ERROR", 41 "", 42 PythonUnitTestResultParser.EQLINE, 43 "ERROR: b (a)", 44 PythonUnitTestResultParser.LINE, 45 "Traceback (most recent call last):", 46 " File \"test_rangelib.py\", line 129, in test_reallyfail", 47 " raise ValueError()", 48 "ValueError", 49 "", 50 PythonUnitTestResultParser.LINE, 51 "Ran 1 tests in 1s", 52 "", 53 "FAILED (errors=1)", 54 }; 55 56 private static final String[] TEST_EXECUTION_FAIL_STDERR = { 57 "Traceback (most recent call last):", 58 " File \"/usr/lib/python2.7/runpy.py\", line 162, in _run_module_as_main", 59 " \"__main__\", fname, loader, pkg_name)", 60 " File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code", 61 " exec code in run_globals", 62 " File \"/usr/lib/python2.7/unittest/__main__.py\", line 12, in <module>", 63 " main(module=None)", 64 " File \"/usr/lib/python2.7/unittest/main.py\", line 94, in __init__", 65 " self.parseArgs(argv)", 66 " File \"/usr/lib/python2.7/unittest/main.py\", line 149, in parseArgs", 67 " self.createTests()", 68 " File \"/usr/lib/python2.7/unittest/main.py\", line 158, in createTests", 69 " self.module)", 70 " File \"/usr/lib/python2.7/unittest/loader.py\", line 130, in loadTestsFromNames", 71 " suites = [self.loadTestsFromName(name, module) for name in names]", 72 " File \"/usr/lib/python2.7/unittest/loader.py\", line 91, in loadTestsFromName", 73 " module = __import__('.'.join(parts_copy))", 74 "ImportError: No module named stub", 75 }; 76 77 private enum UnitTestResult { 78 PASS, 79 FAIL, 80 EXECUTION_FAIL, 81 TIMEOUT; 82 83 private String getStderr() { 84 switch (this) { 85 case PASS: 86 return String.join("\n", TEST_PASS_STDERR); 87 case FAIL: 88 return String.join("\n", TEST_FAIL_STDERR); 89 case EXECUTION_FAIL: 90 return String.join("\n", TEST_EXECUTION_FAIL_STDERR); 91 case TIMEOUT: 92 return null; 93 } 94 return null; 95 } 96 97 private CommandStatus getStatus() { 98 switch (this) { 99 case PASS: 100 return CommandStatus.SUCCESS; 101 case FAIL: 102 return CommandStatus.FAILED; 103 case EXECUTION_FAIL: 104 // UnitTest runner returns with exit code 1 if execution failed. 105 return CommandStatus.FAILED; 106 case TIMEOUT: 107 return CommandStatus.TIMED_OUT; 108 } 109 return null; 110 } 111 112 public CommandResult getCommandResult() { 113 CommandResult cr = new CommandResult(); 114 cr.setStderr(this.getStderr()); 115 cr.setStatus(this.getStatus()); 116 return cr; 117 } 118 } 119 120 private PythonUnitTestRunner mRunner; 121 private ITestInvocationListener mMockListener; 122 123 @Override 124 public void setUp() throws Exception { 125 super.setUp(); 126 mRunner = new PythonUnitTestRunner(); 127 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 128 } 129 130 public void testCheckPythonVersion_276given270min() { 131 CommandResult c = new CommandResult(); 132 c.setStderr("Python 2.7.6"); 133 mRunner.checkPythonVersion(c); 134 } 135 136 public void testCheckPythonVersion_276given331min() { 137 CommandResult c = new CommandResult(); 138 c.setStderr("Python 2.7.6"); 139 mRunner.setMinPythonVersion("3.3.1"); 140 try { 141 mRunner.checkPythonVersion(c); 142 fail("Detected 2.7.6 >= 3.3.1"); 143 } catch (AssertionError e) { 144 return; 145 } 146 } 147 148 public void testCheckPythonVersion_300given276min() { 149 CommandResult c = new CommandResult(); 150 c.setStderr("Python 3.0.0"); 151 mRunner.setMinPythonVersion("2.7.6"); 152 mRunner.checkPythonVersion(c); 153 } 154 155 private IRunUtil getMockRunUtil(UnitTestResult testResult) { 156 CommandResult expectedResult = testResult.getCommandResult(); 157 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 158 // EasyMock checks the number of arguments when verifying method call. 159 // The actual runTimedCmd() expected here looks like: 160 // runTimedCmd(300000, null, "-m", "unittest", "-v", "") 161 EasyMock.expect( 162 mockRunUtil.runTimedCmd( 163 EasyMock.anyLong(), 164 (String) EasyMock.anyObject(), 165 (String) EasyMock.anyObject(), 166 (String) EasyMock.anyObject(), 167 (String) EasyMock.anyObject(), 168 (String) EasyMock.anyObject())) 169 .andReturn(expectedResult) 170 .times(1); 171 return mockRunUtil; 172 } 173 174 private void setMockListenerExpectTestPass(boolean testPass) { 175 mMockListener.testRunStarted((String) EasyMock.anyObject(), EasyMock.anyInt()); 176 EasyMock.expectLastCall().times(1); 177 mMockListener.testStarted((TestIdentifier) EasyMock.anyObject()); 178 EasyMock.expectLastCall().times(1); 179 if (!testPass) { 180 mMockListener.testFailed( 181 (TestIdentifier) EasyMock.anyObject(), (String) EasyMock.anyObject()); 182 EasyMock.expectLastCall().times(1); 183 } 184 mMockListener.testEnded( 185 (TestIdentifier) EasyMock.anyObject(), (Map<String, String>) EasyMock.anyObject()); 186 EasyMock.expectLastCall().times(1); 187 if (!testPass) { 188 mMockListener.testRunFailed((String) EasyMock.anyObject()); 189 EasyMock.expectLastCall().times(1); 190 } 191 mMockListener.testRunEnded(EasyMock.anyLong(), (Map<String, String>) EasyMock.anyObject()); 192 EasyMock.expectLastCall().times(1); 193 } 194 195 /** Test execution succeeds and all test cases pass. */ 196 public void testRunPass() throws DeviceNotAvailableException, ConfigurationException { 197 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.PASS); 198 setMockListenerExpectTestPass(true); 199 EasyMock.replay(mMockListener, mockRunUtil); 200 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 201 EasyMock.verify(mMockListener, mockRunUtil); 202 } 203 204 /** Test execution succeeds and some test cases fail. */ 205 public void testRunFail() throws DeviceNotAvailableException, ConfigurationException { 206 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.FAIL); 207 setMockListenerExpectTestPass(false); 208 EasyMock.replay(mMockListener, mockRunUtil); 209 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 210 EasyMock.verify(mMockListener, mockRunUtil); 211 } 212 213 /** Test execution fails. */ 214 public void testRunExecutionFail() throws DeviceNotAvailableException, ConfigurationException { 215 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.EXECUTION_FAIL); 216 EasyMock.replay(mockRunUtil); 217 try { 218 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 219 fail("Should not reach here."); 220 } catch (RuntimeException e) { 221 assertEquals("Failed to parse python-unittest", e.getMessage()); 222 } 223 EasyMock.verify(mockRunUtil); 224 } 225 226 /** Test execution times out. */ 227 public void testRunTimeout() throws DeviceNotAvailableException, ConfigurationException { 228 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.TIMEOUT); 229 EasyMock.replay(mockRunUtil); 230 try { 231 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 232 fail("Should not reach here."); 233 } catch (RuntimeException e) { 234 assertTrue(e.getMessage().startsWith("Python unit test timed out after")); 235 } 236 EasyMock.verify(mockRunUtil); 237 } 238 }
