1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.tradefed.device.CollectingOutputReceiver; 19 import com.android.tradefed.log.LogUtil.CLog; 20 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 21 import com.android.tradefed.result.ITestInvocationListener; 22 import com.android.tradefed.result.TestDescription; 23 import com.android.tradefed.util.FileUtil; 24 25 import junit.framework.TestCase; 26 27 import org.easymock.EasyMock; 28 29 import java.io.File; 30 import java.io.FileOutputStream; 31 import java.io.IOException; 32 import java.io.InputStream; 33 import java.util.HashMap; 34 35 /** Unit tests for {@link GTestXmlResultParser} */ 36 public class GTestXmlResultParserTest extends TestCase { 37 private static final String TEST_TYPE_DIR = "testtype"; 38 private static final String TEST_MODULE_NAME = "module"; 39 private static final String GTEST_OUTPUT_FILE_1 = "gtest_output1.xml"; 40 private static final String GTEST_OUTPUT_FILE_2 = "gtest_output2.xml"; 41 private static final String GTEST_OUTPUT_FILE_3 = "gtest_output3.xml"; 42 private static final String GTEST_OUTPUT_FILE_4 = "gtest_output4.xml"; 43 private static final String GTEST_OUTPUT_FILE_5 = "gtest_output5.xml"; 44 45 /** 46 * Helper to read a file from the res/testtype directory and return the associated {@link File} 47 * 48 * @param filename the name of the file (without the extension) in the res/testtype directory 49 * @return a File to the output 50 */ 51 private File readInFile(String filename) { 52 InputStream gtest_output = null; 53 File res = null; 54 FileOutputStream out = null; 55 try { 56 gtest_output = getClass().getResourceAsStream(File.separator + 57 TEST_TYPE_DIR + File.separator + filename); 58 res = FileUtil.createTempFile("unit_gtest_", ".xml") ; 59 out = new FileOutputStream(res); 60 byte[] buffer = new byte[1024]; 61 int byteRead; 62 while ((byteRead = gtest_output.read(buffer)) != -1) { 63 out.write(buffer, 0, byteRead); 64 } 65 out.close(); 66 } 67 catch (NullPointerException | IOException e) { 68 CLog.e("Gest output file does not exist: " + filename); 69 } 70 return res; 71 } 72 73 /** 74 * Tests the parser for a simple test run output with 6 tests. 75 */ 76 @SuppressWarnings("unchecked") 77 public void testParseSimpleFile() throws Exception { 78 File contents = readInFile(GTEST_OUTPUT_FILE_1); 79 try { 80 ITestInvocationListener mockRunListener = 81 EasyMock.createMock(ITestInvocationListener.class); 82 mockRunListener.testRunStarted(TEST_MODULE_NAME, 6); 83 // 6 passing test cases in this run 84 for (int i=0; i<6; ++i) { 85 mockRunListener.testStarted((TestDescription) EasyMock.anyObject()); 86 mockRunListener.testEnded( 87 (TestDescription) EasyMock.anyObject(), 88 (HashMap<String, Metric>) EasyMock.anyObject()); 89 } 90 mockRunListener.testRunEnded( 91 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 92 EasyMock.replay(mockRunListener); 93 GTestXmlResultParser resultParser = 94 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 95 resultParser.parseResult(contents, null); 96 EasyMock.verify(mockRunListener); 97 } finally { 98 FileUtil.deleteFile(contents); 99 } 100 } 101 102 /** 103 * Tests the parser for a run with 84 tests. 104 */ 105 @SuppressWarnings("unchecked") 106 public void testParseLargerFile() throws Exception { 107 File contents = readInFile(GTEST_OUTPUT_FILE_2); 108 try { 109 ITestInvocationListener mockRunListener = 110 EasyMock.createMock(ITestInvocationListener.class); 111 mockRunListener.testRunStarted(TEST_MODULE_NAME, 84); 112 // 84 passing test cases in this run 113 for (int i=0; i<84; ++i) { 114 mockRunListener.testStarted((TestDescription) EasyMock.anyObject()); 115 mockRunListener.testEnded( 116 (TestDescription) EasyMock.anyObject(), 117 (HashMap<String, Metric>) EasyMock.anyObject()); 118 } 119 mockRunListener.testRunEnded( 120 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 121 EasyMock.replay(mockRunListener); 122 GTestXmlResultParser resultParser = 123 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 124 resultParser.parseResult(contents, null); 125 EasyMock.verify(mockRunListener); 126 } finally { 127 FileUtil.deleteFile(contents); 128 } 129 } 130 131 /** 132 * Tests the parser for a run with test failures. 133 */ 134 @SuppressWarnings("unchecked") 135 public void testParseWithFailures() throws Exception { 136 String expectedMessage = "Message\nFailed"; 137 138 File contents = readInFile(GTEST_OUTPUT_FILE_3); 139 try { 140 ITestInvocationListener mockRunListener = 141 EasyMock.createMock(ITestInvocationListener.class); 142 mockRunListener.testRunStarted(TEST_MODULE_NAME, 7); 143 // 6 passing test cases in this run 144 for (int i=0; i<6; ++i) { 145 mockRunListener.testStarted((TestDescription) EasyMock.anyObject()); 146 mockRunListener.testEnded( 147 (TestDescription) EasyMock.anyObject(), 148 (HashMap<String, Metric>) EasyMock.anyObject()); 149 } 150 // 1 failed test 151 mockRunListener.testStarted((TestDescription) EasyMock.anyObject()); 152 mockRunListener.testFailed( 153 (TestDescription) EasyMock.anyObject(), EasyMock.eq(expectedMessage)); 154 mockRunListener.testEnded( 155 (TestDescription) EasyMock.anyObject(), 156 (HashMap<String, Metric>) EasyMock.anyObject()); 157 mockRunListener.testRunEnded( 158 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 159 EasyMock.replay(mockRunListener); 160 GTestXmlResultParser resultParser = 161 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 162 resultParser.parseResult(contents, null); 163 EasyMock.verify(mockRunListener); 164 } finally { 165 FileUtil.deleteFile(contents); 166 } 167 } 168 169 /** 170 * Tests the parser for a run with a bad file, as if the test hadn't outputed. 171 */ 172 @SuppressWarnings("unchecked") 173 public void testParseWithEmptyFile() throws Exception { 174 String expected = "Failed to get an xml output from tests, it probably crashed"; 175 176 File contents = FileUtil.createTempFile("test", ".xml"); 177 try { 178 ITestInvocationListener mockRunListener = 179 EasyMock.createMock(ITestInvocationListener.class); 180 mockRunListener.testRunStarted(TEST_MODULE_NAME, 0); 181 mockRunListener.testRunFailed(expected); 182 mockRunListener.testRunEnded( 183 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 184 EasyMock.replay(mockRunListener); 185 GTestXmlResultParser resultParser = 186 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 187 resultParser.parseResult(contents, null); 188 EasyMock.verify(mockRunListener); 189 } finally { 190 FileUtil.deleteFile(contents); 191 } 192 } 193 194 /** 195 * Tests the parser for a simple test run output with 6 tests but report expected 7. 196 */ 197 @SuppressWarnings("unchecked") 198 public void testParseUnexpectedNumberTest() throws Exception { 199 String expected = "Test run incomplete. Expected 7 tests, received 6"; 200 File contents = readInFile(GTEST_OUTPUT_FILE_4); 201 try { 202 ITestInvocationListener mockRunListener = 203 EasyMock.createMock(ITestInvocationListener.class); 204 mockRunListener.testRunStarted(TEST_MODULE_NAME, 7); 205 // 6 passing test cases in this run 206 for (int i=0; i<6; ++i) { 207 mockRunListener.testStarted((TestDescription) EasyMock.anyObject()); 208 mockRunListener.testEnded( 209 (TestDescription) EasyMock.anyObject(), 210 (HashMap<String, Metric>) EasyMock.anyObject()); 211 } 212 mockRunListener.testRunFailed(expected); 213 mockRunListener.testRunEnded( 214 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 215 EasyMock.replay(mockRunListener); 216 GTestXmlResultParser resultParser = 217 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 218 resultParser.parseResult(contents, null); 219 EasyMock.verify(mockRunListener); 220 } finally { 221 FileUtil.deleteFile(contents); 222 } 223 } 224 225 /** 226 * Tests the parser for a simple test run output with 6 tests but a bad xml tag so some tests 227 * won't be parsed. 228 */ 229 @SuppressWarnings("unchecked") 230 public void testParseSimpleFile_badXmltag() throws Exception { 231 String expected = "Test run incomplete. Expected 6 tests, received 3"; 232 File contents = readInFile(GTEST_OUTPUT_FILE_5); 233 try { 234 ITestInvocationListener mockRunListener = 235 EasyMock.createMock(ITestInvocationListener.class); 236 mockRunListener.testRunStarted(TEST_MODULE_NAME, 6); 237 // 6 passing test cases in this run 238 for (int i=0; i<3; ++i) { 239 mockRunListener.testStarted((TestDescription) EasyMock.anyObject()); 240 mockRunListener.testEnded( 241 (TestDescription) EasyMock.anyObject(), 242 (HashMap<String, Metric>) EasyMock.anyObject()); 243 } 244 mockRunListener.testRunFailed(expected); 245 mockRunListener.testRunEnded( 246 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 247 EasyMock.replay(mockRunListener); 248 GTestXmlResultParser resultParser = 249 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 250 resultParser.parseResult(contents, null); 251 EasyMock.verify(mockRunListener); 252 } finally { 253 FileUtil.deleteFile(contents); 254 } 255 } 256 257 /** 258 * Tests the parser for a run with a bad file, with Collector output to get some logs. 259 */ 260 @SuppressWarnings("unchecked") 261 public void testParseWithEmptyFile_AdditionalOutput() throws Exception { 262 final String exec_log = "EXECUTION LOG"; 263 CollectingOutputReceiver fake = new CollectingOutputReceiver() { 264 @Override 265 public String getOutput() { 266 return exec_log; 267 } 268 }; 269 String expected = "Failed to get an xml output from tests, it probably crashed\nlogs:\n" 270 + exec_log; 271 272 File contents = FileUtil.createTempFile("test", ".xml"); 273 try { 274 ITestInvocationListener mockRunListener = 275 EasyMock.createMock(ITestInvocationListener.class); 276 mockRunListener.testRunStarted(TEST_MODULE_NAME, 0); 277 mockRunListener.testRunFailed(expected); 278 mockRunListener.testRunEnded( 279 EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 280 EasyMock.replay(mockRunListener); 281 GTestXmlResultParser resultParser = 282 new GTestXmlResultParser(TEST_MODULE_NAME, mockRunListener); 283 resultParser.parseResult(contents, fake); 284 EasyMock.verify(mockRunListener); 285 } finally { 286 FileUtil.deleteFile(contents); 287 } 288 } 289 } 290
