1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.cts.tradefed.result; 17 18 import android.tests.getinfo.DeviceInfoConstants; 19 20 import com.android.cts.tradefed.device.DeviceInfoCollector; 21 import com.android.cts.tradefed.testtype.CtsTest; 22 import com.android.ddmlib.testrunner.TestIdentifier; 23 import com.android.tradefed.log.LogUtil.CLog; 24 25 import org.kxml2.io.KXmlSerializer; 26 import org.xmlpull.v1.XmlPullParser; 27 import org.xmlpull.v1.XmlPullParserException; 28 29 import java.io.IOException; 30 import java.util.Collection; 31 import java.util.Collections; 32 import java.util.Deque; 33 import java.util.HashMap; 34 import java.util.HashSet; 35 import java.util.LinkedList; 36 import java.util.List; 37 import java.util.Map; 38 import java.util.Set; 39 40 /** 41 * Data structure for a CTS test package result. 42 * <p/> 43 * Provides methods to serialize to XML. 44 */ 45 class TestPackageResult extends AbstractXmlPullParser { 46 47 static final String TAG = "TestPackage"; 48 private static final String DIGEST_ATTR = "digest"; 49 private static final String APP_PACKAGE_NAME_ATTR = "appPackageName"; 50 private static final String NAME_ATTR = "name"; 51 private static final String ns = CtsXmlResultReporter.ns; 52 private static final String SIGNATURE_TEST_PKG = "android.tests.sigtest"; 53 54 private String mAppPackageName; 55 private String mName; 56 private String mDigest; 57 58 private Map<String, String> mMetrics = new HashMap<String, String>(); 59 60 private TestSuite mSuiteRoot = new TestSuite(null); 61 62 public void setAppPackageName(String appPackageName) { 63 mAppPackageName = appPackageName; 64 } 65 66 public String getAppPackageName() { 67 return mAppPackageName; 68 } 69 70 public void setName(String name) { 71 mName = name; 72 } 73 74 public String getName() { 75 return mName; 76 } 77 78 public void setDigest(String digest) { 79 mDigest = digest; 80 } 81 82 public String getDigest() { 83 return mDigest; 84 } 85 86 /** 87 * Return the {@link TestSuite}s 88 */ 89 public Collection<TestSuite> getTestSuites() { 90 return mSuiteRoot.getTestSuites(); 91 } 92 93 /** 94 * Adds a test result to this test package 95 * 96 * @param testId 97 * @param testResult 98 */ 99 public Test insertTest(TestIdentifier testId) { 100 return findTest(testId, true); 101 } 102 103 private Test findTest(TestIdentifier testId, boolean insertIfMissing) { 104 List<String> classNameSegments = new LinkedList<String>(); 105 Collections.addAll(classNameSegments, testId.getClassName().split("\\.")); 106 if (classNameSegments.size() <= 0) { 107 CLog.e("Unrecognized package name format for test class '%s'", 108 testId.getClassName()); 109 // should never happen 110 classNameSegments.add("UnknownTestClass"); 111 } 112 String testCaseName = classNameSegments.remove(classNameSegments.size()-1); 113 return mSuiteRoot.findTest(classNameSegments, testCaseName, testId.getTestName(), insertIfMissing); 114 } 115 116 117 /** 118 * Find the test result for given {@link TestIdentifier}. 119 * @param testId 120 * @return the {@link Test} or <code>null</code> 121 */ 122 public Test findTest(TestIdentifier testId) { 123 return findTest(testId, false); 124 } 125 126 /** 127 * Serialize this object and all its contents to XML. 128 * 129 * @param serializer 130 * @throws IOException 131 */ 132 public void serialize(KXmlSerializer serializer) throws IOException { 133 serializer.startTag(ns, TAG); 134 serializeAttribute(serializer, NAME_ATTR, mName); 135 serializeAttribute(serializer, APP_PACKAGE_NAME_ATTR, mAppPackageName); 136 serializeAttribute(serializer, DIGEST_ATTR, getDigest()); 137 if (SIGNATURE_TEST_PKG.equals(mName)) { 138 serializer.attribute(ns, "signatureCheck", "true"); 139 } 140 mSuiteRoot.serialize(serializer); 141 serializer.endTag(ns, TAG); 142 } 143 144 /** 145 * Helper method to serialize attributes. 146 * Can handle null values. Useful for cases where test package has not been fully populated 147 * such as when unit testing. 148 * 149 * @param attrName 150 * @param attrValue 151 * @throws IOException 152 */ 153 private void serializeAttribute(KXmlSerializer serializer, String attrName, String attrValue) 154 throws IOException { 155 attrValue = attrValue == null ? "" : attrValue; 156 serializer.attribute(ns, attrName, attrValue); 157 } 158 159 /** 160 * Populates this class with package result data parsed from XML. 161 * 162 * @param parser the {@link XmlPullParser}. Expected to be pointing at start 163 * of TestPackage tag 164 */ 165 @Override 166 void parse(XmlPullParser parser) throws XmlPullParserException, IOException { 167 if (!parser.getName().equals(TAG)) { 168 throw new XmlPullParserException(String.format( 169 "invalid XML: Expected %s tag but received %s", TAG, parser.getName())); 170 } 171 setAppPackageName(getAttribute(parser, APP_PACKAGE_NAME_ATTR)); 172 setName(getAttribute(parser, NAME_ATTR)); 173 setDigest(getAttribute(parser, DIGEST_ATTR)); 174 int eventType = parser.getEventType(); 175 while (eventType != XmlPullParser.END_DOCUMENT) { 176 if (eventType == XmlPullParser.START_TAG && parser.getName().equals(TestSuite.TAG)) { 177 TestSuite suite = new TestSuite(); 178 suite.parse(parser); 179 mSuiteRoot.insertSuite(suite); 180 } 181 if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) { 182 return; 183 } 184 eventType = parser.next(); 185 } 186 } 187 188 /** 189 * Return a list of {@link TestIdentifer}s contained in this result with the given status 190 * 191 * @param resultFilter the {@link CtsTestStatus} to filter by 192 * @return a collection of {@link TestIdentifer}s 193 */ 194 public Collection<TestIdentifier> getTestsWithStatus(CtsTestStatus resultFilter) { 195 Collection<TestIdentifier> tests = new LinkedList<TestIdentifier>(); 196 Deque<String> suiteNames = new LinkedList<String>(); 197 mSuiteRoot.addTestsWithStatus(tests, suiteNames, resultFilter); 198 return tests; 199 } 200 201 /** 202 * Populate values in this package result from run metrics 203 * @param runResult 204 */ 205 public void populateMetrics(Map<String, String> metrics) { 206 String name = metrics.get(CtsTest.PACKAGE_NAME_METRIC); 207 if (name != null) { 208 setName(name); 209 } 210 String digest = metrics.get(CtsTest.PACKAGE_DIGEST_METRIC); 211 if (digest != null) { 212 setDigest(digest); 213 } 214 if (DeviceInfoCollector.APP_PACKAGE_NAME.equals(getAppPackageName())) { 215 storeDeviceMetrics(metrics); 216 } else { 217 mMetrics.putAll(metrics); 218 } 219 } 220 221 /** 222 * Check that the provided device info metrics are consistent with the currently stored metrics. 223 * <p/> 224 * If any inconsistencies occur, logs errors and stores error messages in the metrics map 225 * 226 * @param metrics 227 */ 228 private void storeDeviceMetrics(Map<String, String> metrics) { 229 // TODO centralize all the device metrics handling into a single class 230 if (mMetrics.isEmpty()) { 231 // nothing to check! 232 mMetrics.putAll(metrics); 233 return; 234 } 235 // ensure all the metrics we expect to be identical actually are 236 checkMetrics(metrics, DeviceInfoConstants.BUILD_FINGERPRINT, 237 DeviceInfoConstants.BUILD_MODEL, DeviceInfoConstants.BUILD_BRAND, 238 DeviceInfoConstants.BUILD_MANUFACTURER, DeviceInfoConstants.BUILD_BOARD, 239 DeviceInfoConstants.BUILD_DEVICE, DeviceInfoConstants.PRODUCT_NAME, 240 DeviceInfoConstants.BUILD_ABI, DeviceInfoConstants.BUILD_ABI2, 241 DeviceInfoConstants.SCREEN_SIZE); 242 } 243 244 private void checkMetrics(Map<String, String> metrics, String... keysToCheck) { 245 Set<String> keyCheckSet = new HashSet<String>(); 246 Collections.addAll(keyCheckSet, keysToCheck); 247 for (Map.Entry<String, String> metricEntry : metrics.entrySet()) { 248 String currentValue = mMetrics.get(metricEntry.getKey()); 249 if (keyCheckSet.contains(metricEntry.getKey()) && currentValue != null 250 && !metricEntry.getValue().equals(currentValue)) { 251 CLog.e("Inconsistent info collected from devices. " 252 + "Current result has %s='%s', Received '%s'. Are you sharding or " + 253 "resuming a test run across different devices and/or builds?", 254 metricEntry.getKey(), currentValue, metricEntry.getValue()); 255 mMetrics.put(metricEntry.getKey(), 256 String.format("ERROR: Inconsistent results: %s, %s", 257 metricEntry.getValue(), currentValue)); 258 } else { 259 mMetrics.put(metricEntry.getKey(), metricEntry.getValue()); 260 } 261 } 262 } 263 264 /** 265 * Report the given test as a failure. 266 * 267 * @param test 268 * @param status 269 * @param trace 270 */ 271 public void reportTestFailure(TestIdentifier test, CtsTestStatus status, String trace) { 272 Test result = findTest(test); 273 result.setResultStatus(status); 274 result.setStackTrace(trace); 275 } 276 277 /** 278 * report performance result 279 * @param test 280 * @param status 281 * @param perf 282 */ 283 public void reportPerformanceResult(TestIdentifier test, CtsTestStatus status, String summary, String details) { 284 Test result = findTest(test); 285 result.setResultStatus(status); 286 result.setSummary(summary); 287 result.setDetails(details); 288 } 289 290 /** 291 * Report that the given test has completed. 292 * 293 * @param test 294 */ 295 public void reportTestEnded(TestIdentifier test) { 296 Test result = findTest(test); 297 if (!result.getResult().equals(CtsTestStatus.FAIL)) { 298 result.setResultStatus(CtsTestStatus.PASS); 299 } 300 result.updateEndTime(); 301 } 302 303 /** 304 * Return the number of tests with given status 305 * 306 * @param status 307 * @return the total number of tests with given status 308 */ 309 public int countTests(CtsTestStatus status) { 310 return mSuiteRoot.countTests(status); 311 } 312 313 /** 314 * @return 315 */ 316 public Map<String, String> getMetrics() { 317 return mMetrics; 318 } 319 } 320
