1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.ddmlib.testrunner.TestIdentifier; 20 import com.android.tradefed.result.ITestInvocationListener; 21 import com.android.tradefed.util.CommandResult; 22 import com.android.tradefed.util.CommandStatus; 23 import com.android.tradefed.util.IRunUtil; 24 25 import junit.framework.TestCase; 26 27 import org.easymock.EasyMock; 28 29 /** Unit tests for {@link PythonUnitTestRunner}. */ 30 public class PythonUnitTestRunnerTest extends TestCase { 31 32 private static final String[] TEST_PASS_STDERR = { 33 "b (a) ... ok", "", PythonUnitTestResultParser.LINE, "Ran 1 tests in 1s", "", "OK", 34 }; 35 36 private static final String[] TEST_FAIL_STDERR = { 37 "b (a) ... ERROR", 38 "", 39 PythonUnitTestResultParser.EQLINE, 40 "ERROR: b (a)", 41 PythonUnitTestResultParser.LINE, 42 "Traceback (most recent call last):", 43 " File \"test_rangelib.py\", line 129, in test_reallyfail", 44 " raise ValueError()", 45 "ValueError", 46 "", 47 PythonUnitTestResultParser.LINE, 48 "Ran 1 tests in 1s", 49 "", 50 "FAILED (errors=1)", 51 }; 52 53 private static final String[] TEST_EXECUTION_FAIL_STDERR = { 54 "Traceback (most recent call last):", 55 " File \"/usr/lib/python2.7/runpy.py\", line 162, in _run_module_as_main", 56 " \"__main__\", fname, loader, pkg_name)", 57 " File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code", 58 " exec code in run_globals", 59 " File \"/usr/lib/python2.7/unittest/__main__.py\", line 12, in <module>", 60 " main(module=None)", 61 " File \"/usr/lib/python2.7/unittest/main.py\", line 94, in __init__", 62 " self.parseArgs(argv)", 63 " File \"/usr/lib/python2.7/unittest/main.py\", line 149, in parseArgs", 64 " self.createTests()", 65 " File \"/usr/lib/python2.7/unittest/main.py\", line 158, in createTests", 66 " self.module)", 67 " File \"/usr/lib/python2.7/unittest/loader.py\", line 130, in loadTestsFromNames", 68 " suites = [self.loadTestsFromName(name, module) for name in names]", 69 " File \"/usr/lib/python2.7/unittest/loader.py\", line 91, in loadTestsFromName", 70 " module = __import__('.'.join(parts_copy))", 71 "ImportError: No module named stub", 72 }; 73 74 private enum UnitTestResult { 75 PASS, 76 FAIL, 77 EXECUTION_FAIL, 78 TIMEOUT; 79 80 private String getStderr() { 81 switch (this) { 82 case PASS: 83 return String.join("\n", TEST_PASS_STDERR); 84 case FAIL: 85 return String.join("\n", TEST_FAIL_STDERR); 86 case EXECUTION_FAIL: 87 return String.join("\n", TEST_EXECUTION_FAIL_STDERR); 88 case TIMEOUT: 89 return null; 90 } 91 return null; 92 } 93 94 private CommandStatus getStatus() { 95 switch (this) { 96 case PASS: 97 return CommandStatus.SUCCESS; 98 case FAIL: 99 return CommandStatus.FAILED; 100 case EXECUTION_FAIL: 101 // UnitTest runner returns with exit code 1 if execution failed. 102 return CommandStatus.FAILED; 103 case TIMEOUT: 104 return CommandStatus.TIMED_OUT; 105 } 106 return null; 107 } 108 109 public CommandResult getCommandResult() { 110 CommandResult cr = new CommandResult(); 111 cr.setStderr(this.getStderr()); 112 cr.setStatus(this.getStatus()); 113 return cr; 114 } 115 } 116 117 private PythonUnitTestRunner mRunner; 118 private ITestInvocationListener mMockListener; 119 120 @Override 121 public void setUp() throws Exception { 122 super.setUp(); 123 mRunner = new PythonUnitTestRunner(); 124 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 125 } 126 127 public void testCheckPythonVersion_276given270min() { 128 CommandResult c = new CommandResult(); 129 c.setStderr("Python 2.7.6"); 130 mRunner.checkPythonVersion(c); 131 } 132 133 public void testCheckPythonVersion_276given331min() { 134 CommandResult c = new CommandResult(); 135 c.setStderr("Python 2.7.6"); 136 mRunner.setMinPythonVersion("3.3.1"); 137 try { 138 mRunner.checkPythonVersion(c); 139 fail("Detected 2.7.6 >= 3.3.1"); 140 } catch (AssertionError e) { 141 return; 142 } 143 } 144 145 public void testCheckPythonVersion_300given276min() { 146 CommandResult c = new CommandResult(); 147 c.setStderr("Python 3.0.0"); 148 mRunner.setMinPythonVersion("2.7.6"); 149 mRunner.checkPythonVersion(c); 150 } 151 152 private IRunUtil getMockRunUtil(UnitTestResult testResult) { 153 CommandResult expectedResult = testResult.getCommandResult(); 154 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 155 // EasyMock checks the number of arguments when verifying method call. 156 // The actual runTimedCmd() expected here looks like: 157 // runTimedCmd(300000, null, "-m", "unittest", "-v", "") 158 EasyMock.expect( 159 mockRunUtil.runTimedCmd( 160 EasyMock.anyLong(), 161 (String) EasyMock.anyObject(), 162 (String) EasyMock.anyObject(), 163 (String) EasyMock.anyObject(), 164 (String) EasyMock.anyObject(), 165 (String) EasyMock.anyObject())) 166 .andReturn(expectedResult) 167 .times(1); 168 return mockRunUtil; 169 } 170 171 private void setMockListenerExpectTestPass(boolean testPass) { 172 mMockListener.testRunStarted((String) EasyMock.anyObject(), EasyMock.anyInt()); 173 EasyMock.expectLastCall().times(1); 174 mMockListener.testStarted((TestIdentifier) EasyMock.anyObject()); 175 EasyMock.expectLastCall().times(1); 176 if (!testPass) { 177 mMockListener.testFailed( 178 (TestIdentifier) EasyMock.anyObject(), (String) EasyMock.anyObject()); 179 EasyMock.expectLastCall().times(1); 180 } 181 mMockListener.testEnded((TestIdentifier) EasyMock.anyObject(), EasyMock.anyObject()); 182 EasyMock.expectLastCall().times(1); 183 if (!testPass) { 184 mMockListener.testRunFailed((String) EasyMock.anyObject()); 185 EasyMock.expectLastCall().times(1); 186 } 187 mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.anyObject()); 188 EasyMock.expectLastCall().times(1); 189 } 190 191 /** Test execution succeeds and all test cases pass. */ 192 public void testRunPass() { 193 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.PASS); 194 setMockListenerExpectTestPass(true); 195 EasyMock.replay(mMockListener, mockRunUtil); 196 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 197 EasyMock.verify(mMockListener, mockRunUtil); 198 } 199 200 /** Test execution succeeds and some test cases fail. */ 201 public void testRunFail() { 202 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.FAIL); 203 setMockListenerExpectTestPass(false); 204 EasyMock.replay(mMockListener, mockRunUtil); 205 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 206 EasyMock.verify(mMockListener, mockRunUtil); 207 } 208 209 /** Test execution fails. */ 210 public void testRunExecutionFail() { 211 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.EXECUTION_FAIL); 212 EasyMock.replay(mockRunUtil); 213 try { 214 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 215 fail("Should not reach here."); 216 } catch (RuntimeException e) { 217 assertEquals("Failed to parse python-unittest", e.getMessage()); 218 } 219 EasyMock.verify(mockRunUtil); 220 } 221 222 /** Test execution times out. */ 223 public void testRunTimeout() { 224 IRunUtil mockRunUtil = getMockRunUtil(UnitTestResult.TIMEOUT); 225 EasyMock.replay(mockRunUtil); 226 try { 227 mRunner.doRunTest(mMockListener, mockRunUtil, ""); 228 fail("Should not reach here."); 229 } catch (RuntimeException e) { 230 assertTrue(e.getMessage().startsWith("Python unit test timed out after")); 231 } 232 EasyMock.verify(mockRunUtil); 233 } 234 }
