1 /* 2 * Copyright (C) 2008 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.cts; 17 18 import org.w3c.dom.Document; 19 import org.w3c.dom.Node; 20 import org.w3c.dom.NodeList; 21 import org.xml.sax.SAXException; 22 23 import java.io.File; 24 import java.io.FileNotFoundException; 25 import java.io.IOException; 26 import java.security.NoSuchAlgorithmException; 27 import java.util.ArrayList; 28 import java.util.Collection; 29 import java.util.HashMap; 30 31 import javax.xml.parsers.DocumentBuilder; 32 import javax.xml.parsers.DocumentBuilderFactory; 33 import javax.xml.parsers.ParserConfigurationException; 34 import javax.xml.transform.TransformerException; 35 import javax.xml.transform.TransformerFactoryConfigurationError; 36 37 /** 38 * Builder of test plan and also provides serialization for a test plan. 39 */ 40 public class TestSessionBuilder extends XMLResourceHandler { 41 // defined for external document, which is from the configuration files 42 // this should keep synchronized with the format of the configuration files 43 44 private static final String TAG_TEST_SUITE = "TestSuite"; 45 private static final String TAG_TEST_CASE = "TestCase"; 46 public static final String TAG_TEST = "Test"; 47 48 // attribute name define 49 public static final String ATTRIBUTE_SIGNATURE_CHECK = "signatureCheck"; 50 public static final String ATTRIBUTE_REFERENCE_APP_TEST = "referenceAppTest"; 51 public static final String ATTRIBUTE_PRIORITY = "priority"; 52 53 private static final String ATTRIBUTE_NAME = "name"; 54 private static final String ATTRIBUTE_RUNNER = "runner"; 55 private static final String ATTRIBUTE_JAR_PATH = "jarPath"; 56 private static final String ATTRIBUTE_APP_NAME_SPACE = "appNameSpace"; 57 public static final String ATTRIBUTE_APP_PACKAGE_NAME = "appPackageName"; 58 private static final String ATTRIBUTE_TARGET_NAME_SPACE = "targetNameSpace"; 59 private static final String ATTRIBUTE_TARGET_BINARY_NAME = "targetBinaryName"; 60 private static final String ATTRIBUTE_TYPE = "type"; 61 private static final String ATTRIBUTE_CONTROLLER = "HostController"; 62 private static final String ATTRIBUTE_KNOWN_FAILURE = "KnownFailure"; 63 private static final String ATTRIBUTE_HOST_SIDE_ONLY = "hostSideOnly"; 64 private static final String ATTRIBUTE_VERSION = "version"; 65 private static final String ATTRIBUTE_FRAMEWORK_VERSION = "AndroidFramework"; 66 private static final String ATTRIBUTE_APK_TO_TEST_NAME = "apkToTestName"; 67 private static final String ATTRIBUTE_PACKAGE_TO_TEST = "packageToTest"; 68 private static final String ATTRIBUTE_JAVA_PACKAGE_FILTER = "javaPackageFilter"; 69 private static TestSessionBuilder sInstance; 70 71 private DocumentBuilder mDocBuilder; 72 73 public static TestSessionBuilder getInstance() 74 throws ParserConfigurationException { 75 if (sInstance == null) { 76 sInstance = new TestSessionBuilder(); 77 } 78 79 return sInstance; 80 } 81 82 private TestSessionBuilder() throws ParserConfigurationException { 83 mDocBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); 84 } 85 86 /** 87 * Create TestSession via TestSessionLog. 88 * 89 * @param log The test session log. 90 * @return The test session. 91 */ 92 public TestSession build(TestSessionLog log) { 93 if (log == null) { 94 return null; 95 } 96 return new TestSession(log, 1); 97 } 98 99 /** 100 * Create TestSession via TestPlan XML configuration file. 101 * 102 * @param config TestPlan XML configuration file. 103 * @return TestSession. 104 */ 105 public TestSession build(final String config) throws SAXException, IOException, 106 TestPlanNotFoundException, TestNotFoundException, NoSuchAlgorithmException { 107 File file = new File(config); 108 if (!file.exists()) { 109 throw new TestPlanNotFoundException(); 110 } 111 Document doc = mDocBuilder.parse(file); 112 113 // parse device configuration 114 int numOfRequiredDevices = 1; // default is 1 115 try { 116 Node deviceConfigNode = doc.getElementsByTagName( 117 TestPlan.Tag.REQUIRED_DEVICE).item(0); 118 numOfRequiredDevices = getAttributeValue(deviceConfigNode, 119 TestPlan.Attribute.AMOUNT); 120 } catch (Exception e) { 121 } 122 123 Collection<TestPackage> packages = loadPackages(doc); 124 if (packages.size() == 0) { 125 throw new TestNotFoundException("No valid package in test plan."); 126 } 127 128 String planFileName = file.getName(); 129 int index = planFileName.indexOf("."); 130 String planName; 131 if (index != -1) { 132 planName = planFileName.substring(0, planFileName.indexOf(".")); 133 } else{ 134 planName = planFileName; 135 } 136 137 TestSessionLog sessionLog = new TestSessionLog(packages, planName); 138 TestSession ts = new TestSession(sessionLog, numOfRequiredDevices); 139 return ts; 140 } 141 142 /** 143 * Load TestPackages from a TestPlan DOM doc. 144 * 145 * @param doc TestPlan DOM Document 146 * @return loaded test package from TestPlan DOM Document 147 */ 148 private Collection<TestPackage> loadPackages(Document doc) 149 throws SAXException, IOException, NoSuchAlgorithmException { 150 151 ArrayList<TestPackage> packages = new ArrayList<TestPackage>(); 152 NodeList packageList = doc.getElementsByTagName(TestPlan.Tag.ENTRY); 153 ArrayList<String> removedPkgList = new ArrayList<String>(); 154 for (int i = 0; i < packageList.getLength(); i++) { 155 Node pNode = packageList.item(i); 156 String uri = getStringAttributeValue(pNode, TestPlan.Attribute.URI); 157 String list = getStringAttributeValue(pNode, TestPlan.Attribute.EXCLUDE); 158 ArrayList<String> excludedList = null; 159 if ((list != null) && (list.length() != 0)) { 160 excludedList = getStrArrayList(list); 161 } 162 163 String packageBinaryName = HostConfig.getInstance().getPackageBinaryName(uri); 164 if (packageBinaryName != null) { 165 String xmlConfigFilePath = 166 HostConfig.getInstance().getCaseRepository().getXmlPath(packageBinaryName); 167 File xmlFile = new File(xmlConfigFilePath); 168 TestPackage pkg = loadPackage(xmlFile, excludedList); 169 if (pkg instanceof SignatureCheckPackage) { 170 // insert the signature check package 171 // to the head of the list 172 packages.add(0, pkg); 173 } else { 174 packages.add(pkg); 175 } 176 } else{ 177 removedPkgList.add(uri); 178 } 179 } 180 if (removedPkgList.size() != 0) { 181 CUIOutputStream.println("The following package(s) doesn't exist:"); 182 for (String pkgName : removedPkgList) { 183 CUIOutputStream.println(" " + pkgName); 184 } 185 } 186 return packages; 187 } 188 189 /** 190 * Load TestPackage via Package XML configuration file. 191 * 192 * @param packageConfigFile test package XML file 193 * @param excludedList The list containing the excluded suites and sub types. 194 * @return loaded TestPackage from test package XML configuration file 195 */ 196 public TestPackage loadPackage(final File packageConfigFile, ArrayList<String> excludedList) 197 throws SAXException, IOException, NoSuchAlgorithmException { 198 Node pNode = mDocBuilder.parse(packageConfigFile).getDocumentElement(); 199 return loadPackage(pNode, excludedList); 200 } 201 202 /** 203 * Load TestPackage via Package XML configuration file. 204 * 205 * @param pkgNode the test package node in the XML file 206 * @param excludedList The list containing the excluded suites and sub types. 207 * @return loaded TestPackage from test package XML configuration file 208 */ 209 public TestPackage loadPackage(final Node pkgNode, ArrayList<String> excludedList) 210 throws NoSuchAlgorithmException { 211 212 NodeList suiteList = pkgNode.getChildNodes(); 213 214 String appBinaryName = getStringAttributeValue(pkgNode, ATTRIBUTE_NAME); 215 String targetNameSpace = getStringAttributeValue(pkgNode, ATTRIBUTE_TARGET_NAME_SPACE); 216 String targetBinaryName = getStringAttributeValue(pkgNode, ATTRIBUTE_TARGET_BINARY_NAME); 217 String version = getStringAttributeValue(pkgNode, ATTRIBUTE_VERSION); 218 String frameworkVersion = getStringAttributeValue(pkgNode, ATTRIBUTE_FRAMEWORK_VERSION); 219 String runner = getStringAttributeValue(pkgNode, ATTRIBUTE_RUNNER); 220 String jarPath = getStringAttributeValue(pkgNode, ATTRIBUTE_JAR_PATH); 221 String appNameSpace = getStringAttributeValue(pkgNode, ATTRIBUTE_APP_NAME_SPACE); 222 String appPackageName = getStringAttributeValue(pkgNode, ATTRIBUTE_APP_PACKAGE_NAME); 223 String hostSideOnly = getStringAttributeValue(pkgNode, ATTRIBUTE_HOST_SIDE_ONLY); 224 String signature = getStringAttributeValue(pkgNode, ATTRIBUTE_SIGNATURE_CHECK); 225 String referenceAppTest = getStringAttributeValue(pkgNode, ATTRIBUTE_REFERENCE_APP_TEST); 226 String javaPackageFilter = getStringAttributeValue(pkgNode, ATTRIBUTE_JAVA_PACKAGE_FILTER); 227 TestPackage pkg = null; 228 229 if ("true".equals(referenceAppTest)) { 230 String apkToTestName = getStringAttributeValue(pkgNode, ATTRIBUTE_APK_TO_TEST_NAME); 231 String packageUnderTest = getStringAttributeValue(pkgNode, ATTRIBUTE_PACKAGE_TO_TEST); 232 pkg = new ReferenceAppTestPackage(runner, appBinaryName, targetNameSpace, 233 targetBinaryName, version, frameworkVersion, jarPath, 234 appNameSpace, appPackageName, 235 apkToTestName, packageUnderTest); 236 } else if ("true".equals(signature)) { 237 pkg = new SignatureCheckPackage(runner, appBinaryName, targetNameSpace, 238 targetBinaryName, version, frameworkVersion, jarPath, 239 appNameSpace, appPackageName); 240 } else if ("true".equals(hostSideOnly)) { 241 pkg = new HostSideOnlyPackage(appBinaryName, version, frameworkVersion, 242 jarPath, appPackageName); 243 } else { 244 pkg = new TestPackage(runner, appBinaryName, targetNameSpace, targetBinaryName, version, 245 frameworkVersion, jarPath, appNameSpace, appPackageName, javaPackageFilter); 246 } 247 248 for (int i = 0; i < suiteList.getLength(); i++) { 249 Node sNode = suiteList.item(i); 250 if (sNode.getNodeType() == Document.ELEMENT_NODE 251 && TAG_TEST_SUITE.equals(sNode.getNodeName())) { 252 String fullSuiteName = getFullSuiteName(sNode); 253 if (checkFullMatch(excludedList, fullSuiteName) == false) { 254 ArrayList<String> excludedCaseList = 255 getExcludedList(excludedList, fullSuiteName); 256 TestSuite suite = loadSuite(pkg, sNode, excludedCaseList); 257 if ((suite.getTestCases().size() != 0) || (suite.getSubSuites().size() != 0)) { 258 pkg.addTestSuite(suite); 259 } 260 } else { 261 Log.d("suite=" + fullSuiteName + " is fully excluded"); 262 } 263 } 264 } 265 266 return pkg; 267 } 268 269 /** 270 * Get string ArrayList from string. 271 * 272 * @param str The given string. 273 * @return The list. 274 */ 275 private ArrayList<String> getStrArrayList(String str) { 276 if ((str == null) || (str.length() == 0)) { 277 return null; 278 } 279 280 String[] list = str.split(TestPlan.EXCLUDE_SEPARATOR); 281 if ((list == null) || (list.length == 0)) { 282 return null; 283 } 284 285 ArrayList<String> result = new ArrayList<String>(); 286 for (String s : list) { 287 result.add(s); 288 } 289 290 return result; 291 } 292 293 /** 294 * Get excluded list from a list by offered expectation. 295 * 296 * @param excludedList The list containing excluded items. 297 * @param expectation The expectations. 298 * @return The excluded list. 299 */ 300 private ArrayList<String> getExcludedList(ArrayList<String> excludedList, String expectation) { 301 if ((excludedList == null) || (excludedList.size() == 0)) { 302 return null; 303 } 304 305 ArrayList<String> list = new ArrayList<String>(); 306 for (String str : excludedList) { 307 if (str.startsWith(expectation)) { 308 list.add(str); 309 } 310 } 311 312 if (list.size() == 0) { 313 return null; 314 } else { 315 return list; 316 } 317 } 318 319 /** 320 * Check if the expectation is fully matched among a list. 321 * 322 * @param list The array list. 323 * @param expectation The expectation. 324 * @return If there is full match of expectation among the list, return true; 325 * else, return false. 326 */ 327 private boolean checkFullMatch(ArrayList<String> list, String expectation) { 328 if ((list == null) || (list.size() == 0)) { 329 return false; 330 } 331 332 for (String str : list) { 333 if (str.equals(expectation)) { 334 return true; 335 } 336 } 337 338 return false; 339 } 340 341 /** 342 * Load TestSuite via suite node. Package Name is used to output test result. 343 * 344 * @param pkg TestPackage 345 * @param sNode suite node 346 * @param excludedCaseList The list containing the excluded cases and sub types. 347 * @return TestSuite 348 */ 349 private TestSuite loadSuite(final TestPackage pkg, Node sNode, 350 ArrayList<String> excludedCaseList) { 351 NodeList cNodes = sNode.getChildNodes(); 352 String fullSuiteName = getFullSuiteName(sNode); 353 String suiteName = getStringAttributeValue(sNode, TestPlan.Attribute.NAME); 354 TestSuite suite = new TestSuite(pkg, suiteName, fullSuiteName); 355 356 for (int i = 0; i < cNodes.getLength(); i++) { 357 Node cNode = cNodes.item(i); 358 if (cNode.getNodeType() == Document.ELEMENT_NODE) { 359 if (cNode.getNodeName().equals(TAG_TEST_SUITE)) { 360 String subSuiteName = getFullSuiteName(cNode); 361 if (checkFullMatch(excludedCaseList, subSuiteName) == false) { 362 ArrayList<String> excludedList = getExcludedList(excludedCaseList, 363 subSuiteName); 364 TestSuite subSuite = loadSuite(pkg, cNode, excludedList); 365 if ((subSuite.getTestCases().size() != 0) 366 || (subSuite.getSubSuites().size() != 0)) { 367 suite.addSubSuite(subSuite); 368 } 369 } else { 370 Log.d("suite=" + subSuiteName + " is fully excluded"); 371 } 372 } else if (cNode.getNodeName().equals(TAG_TEST_CASE)) { 373 String cName = getStringAttributeValue(cNode, ATTRIBUTE_NAME); 374 String priority = getStringAttributeValue(cNode, ATTRIBUTE_PRIORITY); 375 376 TestCase testCase = new TestCase(suite, cName, priority); 377 String fullCaseName = fullSuiteName + "." + testCase.getName(); 378 if (checkFullMatch(excludedCaseList, fullCaseName) == false) { 379 NodeList mNodes = cNode.getChildNodes(); 380 for (int t = 0; t < mNodes.getLength(); t ++) { 381 Node testNode = mNodes.item(t); 382 if ((testNode.getNodeType() == Document.ELEMENT_NODE) 383 && (testNode.getNodeName().equals(TAG_TEST))) { 384 Test test = loadTest(pkg, testCase, testNode); 385 if (!checkFullMatch(excludedCaseList, test.getFullName())) { 386 testCase.addTest(test); 387 } else { 388 Log.d("Test=" + test.getFullName() + " is excluded"); 389 } 390 } 391 } 392 if (testCase.getTests().size() != 0) { 393 suite.addTestCase(testCase); 394 } 395 } else { 396 Log.d("case=" + fullCaseName + " is fully excluded"); 397 } 398 } 399 } 400 } 401 402 return suite; 403 } 404 405 /** 406 * Load test via test node. 407 * 408 * @param pkg The test package. 409 * @param testCase The test case. 410 * @param testNode The test node. 411 * @return The test loaded. 412 */ 413 private Test loadTest(final TestPackage pkg, TestCase testCase, 414 Node testNode) { 415 String cType = getStringAttributeValue(testNode, ATTRIBUTE_TYPE); 416 String name = getStringAttributeValue(testNode, ATTRIBUTE_NAME); 417 String description = getStringAttributeValue(testNode, 418 ATTRIBUTE_CONTROLLER); 419 String knownFailure = getStringAttributeValue(testNode, 420 ATTRIBUTE_KNOWN_FAILURE); 421 String fullJarPath = 422 HostConfig.getInstance().getCaseRepository().getRoot() 423 + File.separator + pkg.getJarPath(); 424 CtsTestResult testResult = loadTestResult(testNode); 425 Test test = null; 426 if (pkg.isHostSideOnly()) { 427 test = new HostSideOnlyTest(testCase, name, cType, 428 knownFailure, 429 CtsTestResult.CODE_NOT_EXECUTED); 430 description = test.getFullName(); 431 } else { 432 test = new Test(testCase, name, cType, 433 knownFailure, 434 CtsTestResult.CODE_NOT_EXECUTED); 435 } 436 437 TestController controller = 438 genTestControler(fullJarPath, description); 439 test.setTestController(controller); 440 if (testResult != null) { 441 test.addResult(testResult); 442 } 443 return test; 444 } 445 446 /** 447 * Load the CTS test result from the test node. 448 * 449 * @param testNode The test node. 450 * @return The CTS test result. 451 */ 452 private CtsTestResult loadTestResult(Node testNode) { 453 String result = getStringAttributeValue(testNode, 454 TestSessionLog.ATTRIBUTE_RESULT); 455 456 String failedMessage = null; 457 String stackTrace = null; 458 NodeList nodes = testNode.getChildNodes(); 459 for (int i = 0; i < nodes.getLength(); i ++) { 460 Node rNode = nodes.item(i); 461 if ((rNode.getNodeType() == Document.ELEMENT_NODE) 462 && (rNode.getNodeName().equals(TestSessionLog.TAG_FAILED_SCENE))) { 463 failedMessage = getStringAttributeValue(rNode, TestSessionLog.TAG_FAILED_MESSAGE); 464 stackTrace = getStringAttributeValue(rNode, TestSessionLog.TAG_STACK_TRACE); 465 if (stackTrace == null) { 466 NodeList sNodeList = rNode.getChildNodes(); 467 for (int j = 0; j < sNodeList.getLength(); j ++) { 468 Node sNode = sNodeList.item(i); 469 if ((sNode.getNodeType() == Document.ELEMENT_NODE) 470 && (sNode.getNodeName().equals(TestSessionLog.TAG_STACK_TRACE))) { 471 stackTrace = sNode.getTextContent(); 472 } 473 } 474 } 475 break; 476 } 477 } 478 479 CtsTestResult testResult = null; 480 if (result != null) { 481 try { 482 testResult = new CtsTestResult(result, failedMessage, stackTrace); 483 } catch (InvalidTestResultStringException e) { 484 } 485 } 486 487 return testResult; 488 } 489 490 /** 491 * Generate controller according to the description string. 492 * 493 * @return The test controller. 494 */ 495 public TestController genTestControler(String jarPath, String description) { 496 if ((jarPath == null) || (jarPath.length() == 0) 497 || (description == null) || (description.length() == 0)) { 498 return null; 499 } 500 501 String packageName = description.substring(0, description.lastIndexOf(".")); 502 String className = description.substring(packageName.length() + 1, 503 description.lastIndexOf(Test.METHOD_SEPARATOR)); 504 String methodName = description.substring( 505 description.lastIndexOf(Test.METHOD_SEPARATOR) + 1, 506 description.length()); 507 508 return new TestController(jarPath, packageName, className, methodName); 509 } 510 511 /** 512 * Get the full suite name of the specified suite node. Since the test 513 * suite can be nested, so the full name of a tests suite is combined 514 * with his name and his ancestor suite's names. 515 * 516 * @param node The specified suite node. 517 * @return The full name of the given suite node. 518 */ 519 private String getFullSuiteName(Node node) { 520 StringBuilder buf = new StringBuilder(); 521 buf.append(getStringAttributeValue(node, TestPlan.Attribute.NAME)); 522 523 Node parent = node.getParentNode(); 524 while (parent != null) { 525 if (parent.getNodeType() == Document.ELEMENT_NODE 526 && parent.getNodeName() == TAG_TEST_SUITE) { 527 buf.insert(0, "."); 528 buf.insert(0, getStringAttributeValue(parent, TestPlan.Attribute.NAME)); 529 } 530 531 parent = parent.getParentNode(); 532 } 533 534 return buf.toString(); 535 } 536 537 /** 538 * Create TestPlan which contain a series TestPackages. 539 * 540 * @param planName test plan name 541 * @param packageNames Package names to be added 542 * @param selectedResult The selected result mapping selected 543 * package with selected removal result. 544 */ 545 public void serialize(String planName, 546 ArrayList<String> packageNames, HashMap<String, ArrayList<String>> selectedResult) 547 throws ParserConfigurationException, FileNotFoundException, IOException, 548 TransformerFactoryConfigurationError, TransformerException { 549 File plan = new File(HostConfig.getInstance().getPlanRepository() 550 .getPlanPath(planName)); 551 if (plan.exists()) { 552 Log.e("Plan " + planName + " already exist, please use another name!", 553 null); 554 return; 555 } 556 557 Document doc = DocumentBuilderFactory.newInstance() 558 .newDocumentBuilder().newDocument(); 559 Node root = doc.createElement(TestPlan.Tag.TEST_PLAN); 560 setAttribute(doc, root, ATTRIBUTE_VERSION, "1.0"); 561 doc.appendChild(root); 562 563 // append device configure node 564 Node deviceConfigNode = doc.createElement(TestPlan.Tag.PLAN_SETTING); 565 566 root.appendChild(deviceConfigNode); 567 568 // append test packages 569 for (String pName : packageNames) { 570 if (selectedResult.containsKey(pName)) { 571 Node entryNode = doc.createElement(TestPlan.Tag.ENTRY); 572 573 setAttribute(doc, entryNode, TestPlan.Attribute.URI, pName); 574 ArrayList<String> excluded = selectedResult.get(pName); 575 if ((excluded != null) && (excluded.size() != 0)) { 576 String excludedList = ""; 577 for (String str : excluded) { 578 excludedList += str + TestPlan.EXCLUDE_SEPARATOR; 579 } 580 setAttribute(doc, entryNode, TestPlan.Attribute.EXCLUDE, excludedList); 581 } 582 root.appendChild(entryNode); 583 } 584 } 585 586 writeToFile(plan, doc); 587 } 588 589 } 590
