1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result.suite; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertNotNull; 20 import static org.junit.Assert.assertTrue; 21 22 import com.android.tradefed.invoker.IInvocationContext; 23 import com.android.tradefed.invoker.InvocationContext; 24 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 25 import com.android.tradefed.result.LogDataType; 26 import com.android.tradefed.result.LogFile; 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.result.TestRunResult; 29 import com.android.tradefed.testtype.Abi; 30 import com.android.tradefed.testtype.IAbi; 31 import com.android.tradefed.util.FileUtil; 32 33 import org.junit.After; 34 import org.junit.Before; 35 import org.junit.Test; 36 import org.junit.runner.RunWith; 37 import org.junit.runners.JUnit4; 38 import org.w3c.dom.Element; 39 import org.w3c.dom.NodeList; 40 import org.xml.sax.InputSource; 41 42 import java.io.File; 43 import java.io.StringReader; 44 import java.util.ArrayList; 45 import java.util.Collection; 46 import java.util.HashMap; 47 import java.util.Map; 48 49 import javax.xml.xpath.XPath; 50 import javax.xml.xpath.XPathConstants; 51 import javax.xml.xpath.XPathExpressionException; 52 import javax.xml.xpath.XPathFactory; 53 54 /** Unit tests for {@link XmlSuiteResultFormatter}. */ 55 @RunWith(JUnit4.class) 56 public class XmlSuiteResultFormatterTest { 57 private XmlSuiteResultFormatter mFormatter; 58 private SuiteResultHolder mResultHolder; 59 private IInvocationContext mContext; 60 private File mResultDir; 61 62 @Before 63 public void setUp() throws Exception { 64 mFormatter = new XmlSuiteResultFormatter(); 65 mResultHolder = new SuiteResultHolder(); 66 mContext = new InvocationContext(); 67 mResultDir = FileUtil.createTempDir("result-dir"); 68 } 69 70 @After 71 public void tearDown() throws Exception { 72 FileUtil.recursiveDelete(mResultDir); 73 } 74 75 /** Check that the basic overall structure is good an contains all the information. */ 76 @Test 77 public void testBasicFormat() throws Exception { 78 mResultHolder.context = mContext; 79 80 Collection<TestRunResult> runResults = new ArrayList<>(); 81 runResults.add(createFakeResult("module1", 2, 0)); 82 runResults.add(createFakeResult("module2", 1, 0)); 83 mResultHolder.runResults = runResults; 84 85 Map<String, IAbi> modulesAbi = new HashMap<>(); 86 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 87 modulesAbi.put("module2", new Abi("armeabi-v7a", "32")); 88 mResultHolder.modulesAbi = modulesAbi; 89 90 mResultHolder.completeModules = 2; 91 mResultHolder.totalModules = 2; 92 mResultHolder.passedTests = 2; 93 mResultHolder.failedTests = 0; 94 mResultHolder.startTime = 0L; 95 mResultHolder.endTime = 10L; 96 File res = mFormatter.writeResults(mResultHolder, mResultDir); 97 String content = FileUtil.readStringFromFile(res); 98 assertXmlContainsNode(content, "Result"); 99 // Verify that the summary has been populated 100 assertXmlContainsNode(content, "Result/Summary"); 101 assertXmlContainsAttribute(content, "Result/Summary", "pass", "2"); 102 assertXmlContainsAttribute(content, "Result/Summary", "failed", "0"); 103 assertXmlContainsAttribute(content, "Result/Summary", "modules_done", "2"); 104 assertXmlContainsAttribute(content, "Result/Summary", "modules_total", "2"); 105 // Verify that each module results are available 106 assertXmlContainsNode(content, "Result/Module"); 107 assertXmlContainsAttribute(content, "Result/Module", "name", "module1"); 108 assertXmlContainsAttribute(content, "Result/Module", "abi", "armeabi-v7a"); 109 assertXmlContainsAttribute(content, "Result/Module", "runtime", "10"); 110 assertXmlContainsAttribute(content, "Result/Module", "done", "true"); 111 assertXmlContainsAttribute(content, "Result/Module", "pass", "2"); 112 // Verify the test cases that passed are present 113 assertXmlContainsNode(content, "Result/Module/TestCase"); 114 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 115 assertXmlContainsAttribute( 116 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 117 assertXmlContainsAttribute( 118 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 119 120 assertXmlContainsAttribute(content, "Result/Module", "name", "module2"); 121 assertXmlContainsAttribute(content, "Result/Module", "pass", "1"); 122 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module2"); 123 assertXmlContainsAttribute( 124 content, "Result/Module/TestCase/Test", "name", "module2.method0"); 125 } 126 127 /** Check that the test failures are properly reported. */ 128 @Test 129 public void testFailuresReporting() throws Exception { 130 mResultHolder.context = mContext; 131 132 Collection<TestRunResult> runResults = new ArrayList<>(); 133 runResults.add(createFakeResult("module1", 2, 1)); 134 mResultHolder.runResults = runResults; 135 136 Map<String, IAbi> modulesAbi = new HashMap<>(); 137 modulesAbi.put("module1", new Abi("armeabi-v7a", "32")); 138 mResultHolder.modulesAbi = modulesAbi; 139 140 mResultHolder.completeModules = 2; 141 mResultHolder.totalModules = 1; 142 mResultHolder.passedTests = 2; 143 mResultHolder.failedTests = 1; 144 mResultHolder.startTime = 0L; 145 mResultHolder.endTime = 10L; 146 File res = mFormatter.writeResults(mResultHolder, mResultDir); 147 String content = FileUtil.readStringFromFile(res); 148 149 assertXmlContainsNode(content, "Result/Module"); 150 assertXmlContainsAttribute(content, "Result/Module/TestCase", "name", "com.class.module1"); 151 assertXmlContainsAttribute( 152 content, "Result/Module/TestCase/Test", "name", "module1.method0"); 153 assertXmlContainsAttribute( 154 content, "Result/Module/TestCase/Test", "name", "module1.method1"); 155 // Check that failures are showing in the xml for the test cases 156 assertXmlContainsAttribute( 157 content, "Result/Module/TestCase/Test", "name", "module1.failed0"); 158 assertXmlContainsAttribute( 159 content, "Result/Module/TestCase/Test/Failure", "message", "module1 failed."); 160 assertXmlContainsValue( 161 content, 162 "Result/Module/TestCase/Test/Failure/StackTrace", 163 "module1 failed.\nstack\nstack"); 164 // Test that we can read back the informations 165 SuiteResultHolder holder = mFormatter.parseResults(mResultDir); 166 assertEquals(holder.completeModules, mResultHolder.completeModules); 167 assertEquals(holder.totalModules, mResultHolder.totalModules); 168 assertEquals(holder.passedTests, mResultHolder.passedTests); 169 assertEquals(holder.failedTests, mResultHolder.failedTests); 170 assertEquals(holder.startTime, mResultHolder.startTime); 171 assertEquals(holder.endTime, mResultHolder.endTime); 172 assertEquals( 173 holder.modulesAbi.get("armeabi-v7a module1"), 174 mResultHolder.modulesAbi.get("module1")); 175 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 176 } 177 178 /** Check that the logs for each test case are reported. */ 179 @Test 180 public void testLogReporting() throws Exception { 181 mResultHolder.context = mContext; 182 183 Collection<TestRunResult> runResults = new ArrayList<>(); 184 runResults.add(createResultWithLog("armeabi-v7a module1", 1, LogDataType.LOGCAT)); 185 runResults.add(createResultWithLog("module2", 1, LogDataType.BUGREPORT)); 186 runResults.add(createResultWithLog("module3", 1, LogDataType.PNG)); 187 mResultHolder.runResults = runResults; 188 189 Map<String, IAbi> modulesAbi = new HashMap<>(); 190 modulesAbi.put("armeabi-v7a module1", new Abi("armeabi-v7a", "32")); 191 mResultHolder.modulesAbi = modulesAbi; 192 193 mResultHolder.completeModules = 2; 194 mResultHolder.totalModules = 2; 195 mResultHolder.passedTests = 2; 196 mResultHolder.failedTests = 0; 197 mResultHolder.startTime = 0L; 198 mResultHolder.endTime = 10L; 199 File res = mFormatter.writeResults(mResultHolder, mResultDir); 200 String content = FileUtil.readStringFromFile(res); 201 // One logcat and one bugreport are found in the report 202 assertXmlContainsValue( 203 content, "Result/Module/TestCase/Test/Logcat", "http:url/armeabi-v7a module1"); 204 assertXmlContainsValue( 205 content, "Result/Module/TestCase/Test/BugReport", "http:url/module2"); 206 assertXmlContainsValue( 207 content, "Result/Module/TestCase/Test/Screenshot", "http:url/module3"); 208 209 // Test that we can read back the informations for log files 210 SuiteResultHolder holder = mFormatter.parseResults(mResultDir); 211 assertEquals( 212 holder.modulesAbi.get("armeabi-v7a module1"), 213 mResultHolder.modulesAbi.get("armeabi-v7a module1")); 214 assertEquals(holder.runResults.size(), mResultHolder.runResults.size()); 215 for (TestRunResult result : holder.runResults) { 216 TestDescription description = 217 new TestDescription( 218 "com.class." + result.getName(), result.getName() + ".method0"); 219 // Check that we reloaded the logged files. 220 assertTrue( 221 result.getTestResults() 222 .get(description) 223 .getLoggedFiles() 224 .get(result.getName() + "log0") 225 != null); 226 } 227 } 228 229 private TestRunResult createResultWithLog(String runName, int count, LogDataType type) { 230 TestRunResult fakeRes = new TestRunResult(); 231 fakeRes.testRunStarted(runName, count); 232 for (int i = 0; i < count; i++) { 233 TestDescription description = 234 new TestDescription("com.class." + runName, runName + ".method" + i); 235 fakeRes.testStarted(description); 236 fakeRes.testLogSaved( 237 runName + "log" + i, new LogFile("path", "http:url/" + runName, type)); 238 fakeRes.testEnded(description, new HashMap<String, Metric>()); 239 } 240 fakeRes.testRunEnded(10L, new HashMap<String, String>()); 241 return fakeRes; 242 } 243 244 private TestRunResult createFakeResult(String runName, int passed, int failed) { 245 TestRunResult fakeRes = new TestRunResult(); 246 fakeRes.testRunStarted(runName, passed + failed); 247 for (int i = 0; i < passed; i++) { 248 TestDescription description = 249 new TestDescription("com.class." + runName, runName + ".method" + i); 250 fakeRes.testStarted(description); 251 fakeRes.testEnded(description, new HashMap<String, Metric>()); 252 } 253 for (int i = 0; i < failed; i++) { 254 TestDescription description = 255 new TestDescription("com.class." + runName, runName + ".failed" + i); 256 fakeRes.testStarted(description); 257 fakeRes.testFailed(description, runName + " failed.\nstack\nstack"); 258 fakeRes.testEnded(description, new HashMap<String, Metric>()); 259 } 260 fakeRes.testRunEnded(10L, new HashMap<String, Metric>()); 261 return fakeRes; 262 } 263 264 /** Return all XML nodes that match the given xPathExpression. */ 265 private NodeList getXmlNodes(String xml, String xPathExpression) 266 throws XPathExpressionException { 267 268 InputSource inputSource = new InputSource(new StringReader(xml)); 269 XPath xpath = XPathFactory.newInstance().newXPath(); 270 return (NodeList) xpath.evaluate(xPathExpression, inputSource, XPathConstants.NODESET); 271 } 272 273 /** Assert that the XML contains a node matching the given xPathExpression. */ 274 private NodeList assertXmlContainsNode(String xml, String xPathExpression) 275 throws XPathExpressionException { 276 NodeList nodes = getXmlNodes(xml, xPathExpression); 277 assertNotNull( 278 String.format("XML '%s' returned null for xpath '%s'.", xml, xPathExpression), 279 nodes); 280 assertTrue( 281 String.format( 282 "XML '%s' should have returned at least 1 node for xpath '%s', " 283 + "but returned %s nodes instead.", 284 xml, xPathExpression, nodes.getLength()), 285 nodes.getLength() >= 1); 286 return nodes; 287 } 288 289 /** 290 * Assert that the XML contains a node matching the given xPathExpression and that the node has 291 * a given value. 292 */ 293 private void assertXmlContainsValue(String xml, String xPathExpression, String value) 294 throws XPathExpressionException { 295 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 296 boolean found = false; 297 298 for (int i = 0; i < nodes.getLength(); i++) { 299 Element element = (Element) nodes.item(i); 300 if (element.getTextContent().equals(value)) { 301 found = true; 302 break; 303 } 304 } 305 306 assertTrue( 307 String.format( 308 "xPath '%s' should contain value '%s' but does not. XML: '%s'", 309 xPathExpression, value, xml), 310 found); 311 } 312 313 /** 314 * Assert that the XML contains a node matching the given xPathExpression and that the node has 315 * a given value. 316 */ 317 private void assertXmlContainsAttribute( 318 String xml, String xPathExpression, String attributeName, String attributeValue) 319 throws XPathExpressionException { 320 NodeList nodes = assertXmlContainsNode(xml, xPathExpression); 321 boolean found = false; 322 323 for (int i = 0; i < nodes.getLength(); i++) { 324 Element element = (Element) nodes.item(i); 325 String value = element.getAttribute(attributeName); 326 if (attributeValue.equals(value)) { 327 found = true; 328 break; 329 } 330 } 331 332 assertTrue( 333 String.format( 334 "xPath '%s' should contain attribute '%s' but does not. XML: '%s'", 335 xPathExpression, attributeName, xml), 336 found); 337 } 338 } 339
