Home | History | Annotate | Download | only in utils 
      1 /*
      2  * Copyright (C) 2008 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 import org.w3c.dom.Document;
     18 import org.w3c.dom.Element;
     19 import org.w3c.dom.Node;
     20 import org.w3c.dom.NodeList;
     21 
     22 import vogar.Expectation;
     23 import vogar.ExpectationStore;
     24 import vogar.ModeId;
     25 
     26 import java.io.BufferedReader;
     27 import java.io.File;
     28 import java.io.FileInputStream;
     29 import java.io.FileReader;
     30 import java.io.FilenameFilter;
     31 import java.io.IOException;
     32 import java.lang.annotation.Annotation;
     33 import java.lang.reflect.InvocationTargetException;
     34 import java.lang.reflect.Method;
     35 import java.lang.reflect.Modifier;
     36 import java.util.ArrayList;
     37 import java.util.Arrays;
     38 import java.util.Enumeration;
     39 import java.util.HashSet;
     40 import java.util.Iterator;
     41 import java.util.LinkedHashMap;
     42 import java.util.Map;
     43 import java.util.Set;
     44 import java.util.jar.JarEntry;
     45 import java.util.jar.JarFile;
     46 
     47 import javax.xml.parsers.DocumentBuilderFactory;
     48 import javax.xml.parsers.ParserConfigurationException;
     49 
     50 import junit.framework.Test;
     51 import junit.framework.TestCase;
     52 import junit.framework.TestResult;
     53 import junit.textui.ResultPrinter;
     54 import junit.textui.TestRunner;
     55 
     56 public class CollectAllTests extends DescriptionGenerator {
     57 
     58     private static final String ATTRIBUTE_RUNNER = "runner";
     59     private static final String ATTRIBUTE_PACKAGE = "appPackageName";
     60     private static final String ATTRIBUTE_NS = "appNameSpace";
     61     private static final String ATTRIBUTE_TARGET = "targetNameSpace";
     62     private static final String ATTRIBUTE_TARGET_BINARY = "targetBinaryName";
     63     private static final String ATTRIBUTE_HOST_SIDE_ONLY = "hostSideOnly";
     64     private static final String ATTRIBUTE_VM_HOST_TEST = "vmHostTest";
     65     private static final String ATTRIBUTE_JAR_PATH = "jarPath";
     66     private static final String ATTRIBUTE_JAVA_PACKAGE_FILTER = "javaPackageFilter";
     67 
     68     private static final String JAR_PATH = "LOCAL_JAR_PATH :=";
     69     private static final String TEST_TYPE = "LOCAL_TEST_TYPE :";
     70 
     71     public static void main(String[] args) {
     72         if (args.length < 4 || args.length > 6) {
     73             System.err.println("usage: CollectAllTests <output-file> <manifest-file> <jar-file> "
     74                                + "<java-package> [expectation-dir [makefile-file]]");
     75             if (args.length != 0) {
     76                 System.err.println("received:");
     77                 for (String arg : args) {
     78                     System.err.println("  " + arg);
     79                 }
     80             }
     81             System.exit(1);
     82         }
     83 
     84         final String outputPathPrefix = args[0];
     85         File manifestFile = new File(args[1]);
     86         String jarFileName = args[2];
     87         final String javaPackageFilter = args[3];
     88         // Validate the javaPackageFilter value if non null.
     89         if (javaPackageFilter.length() != 0) {
     90             if (!isValidJavaPackage(javaPackageFilter)) {
     91                 System.err.println("Invalid " + ATTRIBUTE_JAVA_PACKAGE_FILTER + ": " +
     92                         javaPackageFilter);
     93                 System.exit(1);
     94                 return;
     95             }
     96         }
     97         String libcoreExpectationDir = (args.length > 4) ? args[4] : null;
     98         String androidMakeFile = (args.length > 5) ? args[5] : null;
     99 
    100         final TestType testType = TestType.getTestType(androidMakeFile);
    101 
    102         Document manifest;
    103         try {
    104             manifest = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(
    105                   new FileInputStream(manifestFile));
    106         } catch (Exception e) {
    107             System.err.println("cannot open manifest " + manifestFile);
    108             e.printStackTrace();
    109             System.exit(1);
    110             return;
    111         }
    112 
    113         Element documentElement = manifest.getDocumentElement();
    114         documentElement.getAttribute("package");
    115         final String runner = getElementAttribute(documentElement,
    116                                                   "instrumentation",
    117                                                   "android:name");
    118         final String packageName = documentElement.getAttribute("package");
    119         final String target = getElementAttribute(documentElement,
    120                                                   "instrumentation",
    121                                                   "android:targetPackage");
    122 
    123         String outputXmlFile = outputPathPrefix + ".xml";
    124         final String xmlName = new File(outputPathPrefix).getName();
    125         XMLGenerator xmlGenerator;
    126         try {
    127             xmlGenerator = new XMLGenerator(outputXmlFile) {
    128                 {
    129                     Node testPackageElem = mDoc.getDocumentElement();
    130 
    131                     setAttribute(testPackageElem, ATTRIBUTE_NAME, xmlName);
    132                     setAttribute(testPackageElem, ATTRIBUTE_RUNNER, runner);
    133                     setAttribute(testPackageElem, ATTRIBUTE_PACKAGE, packageName);
    134                     setAttribute(testPackageElem, ATTRIBUTE_NS, packageName);
    135                     setAttribute(testPackageElem, ATTRIBUTE_JAVA_PACKAGE_FILTER, javaPackageFilter);
    136 
    137                     if (testType.type == TestType.HOST_SIDE_ONLY) {
    138                         setAttribute(testPackageElem, ATTRIBUTE_HOST_SIDE_ONLY, "true");
    139                         setAttribute(testPackageElem, ATTRIBUTE_JAR_PATH, testType.jarPath);
    140                     }
    141 
    142                     if (testType.type == TestType.VM_HOST_TEST) {
    143                         setAttribute(testPackageElem, ATTRIBUTE_VM_HOST_TEST, "true");
    144                         setAttribute(testPackageElem, ATTRIBUTE_JAR_PATH, testType.jarPath);
    145                     }
    146 
    147                     if (!packageName.equals(target)) {
    148                         setAttribute(testPackageElem, ATTRIBUTE_TARGET, target);
    149                         setAttribute(testPackageElem, ATTRIBUTE_TARGET_BINARY, target);
    150                     }
    151                 }
    152             };
    153         } catch (ParserConfigurationException e) {
    154             System.err.println("Can't initialize XML Generator " + outputXmlFile);
    155             System.exit(1);
    156             return;
    157         }
    158 
    159         ExpectationStore libcoreVogarExpectationStore;
    160         ExpectationStore ctsVogarExpectationStore;
    161 
    162         try {
    163             libcoreVogarExpectationStore
    164                     = VogarUtils.provideExpectationStore(libcoreExpectationDir);
    165             ctsVogarExpectationStore = VogarUtils.provideExpectationStore(CTS_EXPECTATION_DIR);
    166         } catch (IOException e) {
    167             System.err.println("Can't initialize vogar expectation store from "
    168                                + libcoreExpectationDir);
    169             e.printStackTrace(System.err);
    170             System.exit(1);
    171             return;
    172         }
    173         ExpectationStore[] expectations = new ExpectationStore[] {
    174             libcoreVogarExpectationStore, ctsVogarExpectationStore
    175         };
    176 
    177         JarFile jarFile = null;
    178         try {
    179             jarFile = new JarFile(jarFileName);
    180         } catch (Exception e) {
    181             System.err.println("cannot open jarfile " + jarFileName);
    182             e.printStackTrace();
    183             System.exit(1);
    184         }
    185 
    186         Map<String,TestClass> testCases = new LinkedHashMap<String, TestClass>();
    187 
    188         String javaPackagePrefix = javaPackageFilter.isEmpty() ? "" : (javaPackageFilter + ".");
    189 
    190         Enumeration<JarEntry> jarEntries = jarFile.entries();
    191         while (jarEntries.hasMoreElements()) {
    192             JarEntry jarEntry = jarEntries.nextElement();
    193             String name = jarEntry.getName();
    194             if (!name.endsWith(".class")) {
    195                 continue;
    196             }
    197             String className
    198                     = name.substring(0, name.length() - ".class".length()).replace('/', '.');
    199             if (!className.startsWith(javaPackagePrefix)) {
    200                 continue;
    201             }
    202             try {
    203                 Class<?> klass = Class.forName(className,
    204                                                false,
    205                                                CollectAllTests.class.getClassLoader());
    206                 if (!TestCase.class.isAssignableFrom(klass)) {
    207                     continue;
    208                 }
    209                 if (Modifier.isAbstract(klass.getModifiers())) {
    210                     continue;
    211                 }
    212                 if (!Modifier.isPublic(klass.getModifiers())) {
    213                     continue;
    214                 }
    215                 try {
    216                     klass.getConstructor(new Class<?>[] { String.class } );
    217                     addToTests(expectations, testCases, klass.asSubclass(TestCase.class));
    218                     continue;
    219                 } catch (NoSuchMethodException e) {
    220                 } catch (SecurityException e) {
    221                     System.out.println("Known bug (Working as intended): problem with class "
    222                             + className);
    223                     e.printStackTrace();
    224                 }
    225                 try {
    226                     klass.getConstructor(new Class<?>[0]);
    227                     addToTests(expectations, testCases, klass.asSubclass(TestCase.class));
    228                     continue;
    229                 } catch (NoSuchMethodException e) {
    230                 } catch (SecurityException e) {
    231                     System.out.println("Known bug (Working as intended): problem with class "
    232                             + className);
    233                     e.printStackTrace();
    234                 }
    235             } catch (ClassNotFoundException e) {
    236                 System.out.println("class not found " + className);
    237                 e.printStackTrace();
    238                 System.exit(1);
    239             }
    240         }
    241 
    242         for (Iterator<TestClass> iterator = testCases.values().iterator(); iterator.hasNext();) {
    243             TestClass type = iterator.next();
    244             xmlGenerator.addTestClass(type);
    245         }
    246 
    247         try {
    248             xmlGenerator.dump();
    249         } catch (Exception e) {
    250             System.err.println("cannot dump xml to " + outputXmlFile);
    251             e.printStackTrace();
    252             System.exit(1);
    253         }
    254     }
    255 
    256     private static class TestType {
    257         private static final int HOST_SIDE_ONLY = 1;
    258         private static final int DEVICE_SIDE_ONLY = 2;
    259         private static final int VM_HOST_TEST = 3;
    260 
    261         private final int type;
    262         private final String jarPath;
    263 
    264         private TestType (int type, String jarPath) {
    265             this.type = type;
    266             this.jarPath = jarPath;
    267         }
    268 
    269         private static TestType getTestType(String makeFileName) {
    270             if (makeFileName == null || makeFileName.isEmpty()) {
    271                 return new TestType(DEVICE_SIDE_ONLY, null);
    272             }
    273             int type = TestType.DEVICE_SIDE_ONLY;
    274             String jarPath = null;
    275             try {
    276                 BufferedReader reader = new BufferedReader(new FileReader(makeFileName));
    277                 String line;
    278 
    279                 while ((line =reader.readLine())!=null) {
    280                     if (line.startsWith(TEST_TYPE)) {
    281                         if (line.indexOf(ATTRIBUTE_VM_HOST_TEST) >= 0) {
    282                             type = VM_HOST_TEST;
    283                         } else {
    284                             type = HOST_SIDE_ONLY;
    285                         }
    286                     } else if (line.startsWith(JAR_PATH)) {
    287                         jarPath = line.substring(JAR_PATH.length(), line.length()).trim();
    288                     }
    289                 }
    290                 reader.close();
    291             } catch (IOException e) {
    292             }
    293             return new TestType(type, jarPath);
    294         }
    295     }
    296 
    297     private static Element getElement(Element element, String tagName) {
    298         NodeList elements = element.getElementsByTagName(tagName);
    299         if (elements.getLength() > 0) {
    300             return (Element) elements.item(0);
    301         } else {
    302             return null;
    303         }
    304     }
    305 
    306     private static String getElementAttribute(Element element,
    307                                               String elementName,
    308                                               String attributeName) {
    309         Element e = getElement(element, elementName);
    310         if (e != null) {
    311             return e.getAttribute(attributeName);
    312         } else {
    313             return "";
    314         }
    315     }
    316 
    317     private static String getKnownFailure(final Class<? extends TestCase> testClass,
    318             final String testName) {
    319         return getAnnotation(testClass, testName, KNOWN_FAILURE);
    320     }
    321 
    322     private static boolean isKnownFailure(final Class<? extends TestCase> testClass,
    323             final String testName) {
    324         return getAnnotation(testClass, testName, KNOWN_FAILURE) != null;
    325     }
    326 
    327     private static boolean isBrokenTest(final Class<? extends TestCase> testClass,
    328             final String testName)  {
    329         return getAnnotation(testClass, testName, BROKEN_TEST) != null;
    330     }
    331 
    332     private static boolean isSuppressed(final Class<? extends TestCase> testClass,
    333             final String testName)  {
    334         return getAnnotation(testClass, testName, SUPPRESSED_TEST) != null;
    335     }
    336 
    337     private static boolean hasSideEffects(final Class<? extends TestCase> testClass,
    338             final String testName) {
    339         return getAnnotation(testClass, testName, SIDE_EFFECT) != null;
    340     }
    341 
    342     private static String getAnnotation(final Class<? extends TestCase> testClass,
    343             final String testName, final String annotationName) {
    344         try {
    345             Method testMethod = testClass.getMethod(testName, (Class[])null);
    346             Annotation[] annotations = testMethod.getAnnotations();
    347             for (Annotation annot : annotations) {
    348 
    349                 if (annot.annotationType().getName().equals(annotationName)) {
    350                     String annotStr = annot.toString();
    351                     String knownFailure = null;
    352                     if (annotStr.contains("(value=")) {
    353                         knownFailure =
    354                             annotStr.substring(annotStr.indexOf("=") + 1,
    355                                     annotStr.length() - 1);
    356 
    357                     }
    358 
    359                     if (knownFailure == null) {
    360                         knownFailure = "true";
    361                     }
    362 
    363                     return knownFailure;
    364                 }
    365 
    366             }
    367 
    368         } catch (NoSuchMethodException e) {
    369         }
    370 
    371         return null;
    372     }
    373 
    374     private static void addToTests(ExpectationStore[] expectations,
    375                                    Map<String,TestClass> testCases,
    376                                    Class<? extends TestCase> test) {
    377         Class testClass = test;
    378         Set<String> testNames = new HashSet<String>();
    379         while (TestCase.class.isAssignableFrom(testClass)) {
    380             Method[] testMethods = testClass.getDeclaredMethods();
    381             for (Method testMethod : testMethods) {
    382                 String testName = testMethod.getName();
    383                 if (testNames.contains(testName)) {
    384                     continue;
    385                 }
    386                 if (!testName.startsWith("test")) {
    387                     continue;
    388                 }
    389                 if (testMethod.getParameterTypes().length != 0) {
    390                     continue;
    391                 }
    392                 if (!testMethod.getReturnType().equals(Void.TYPE)) {
    393                     continue;
    394                 }
    395                 if (!Modifier.isPublic(testMethod.getModifiers())) {
    396                     continue;
    397                 }
    398                 testNames.add(testName);
    399                 addToTests(expectations, testCases, test, testName);
    400             }
    401             testClass = testClass.getSuperclass();
    402         }
    403     }
    404 
    405     private static void addToTests(ExpectationStore[] expectations,
    406                                    Map<String,TestClass> testCases,
    407                                    Class<? extends TestCase> test,
    408                                    String testName) {
    409 
    410         String testClassName = test.getName();
    411         String knownFailure = getKnownFailure(test, testName);
    412 
    413         if (isKnownFailure(test, testName)) {
    414             System.out.println("ignoring known failure: " + test + "#" + testName);
    415             return;
    416         } else if (isBrokenTest(test, testName)) {
    417             System.out.println("ignoring broken test: " + test + "#" + testName);
    418             return;
    419         } else if (isSuppressed(test, testName)) {
    420             System.out.println("ignoring suppressed test: " + test + "#" + testName);
    421             return;
    422         } else if (hasSideEffects(test, testName)) {
    423             System.out.println("ignoring test with side effects: " + test + "#" + testName);
    424             return;
    425         } else if (VogarUtils.isVogarKnownFailure(expectations,
    426                                                   testClassName,
    427                                                   testName)) {
    428             System.out.println("ignoring expectation known failure: " + test
    429                                + "#" + testName);
    430             return;
    431         }
    432 
    433         TestClass testClass = null;
    434         if (testCases.containsKey(testClassName)) {
    435             testClass = testCases.get(testClassName);
    436         } else {
    437             testClass = new TestClass(testClassName, new ArrayList<TestMethod>());
    438             testCases.put(testClassName, testClass);
    439         }
    440 
    441         testClass.mCases.add(new TestMethod(testName, "", "", knownFailure, false, false));
    442     }
    443 
    444     /**
    445      * Determines if a given string is a valid java package name
    446      * @param javaPackageName
    447      * @return true if it is valid, false otherwise
    448      */
    449     private static boolean isValidJavaPackage(String javaPackageName) {
    450         String[] strSections = javaPackageName.split(".");
    451         for (String strSection : strSections) {
    452           if (!isValidJavaIdentifier(strSection)) {
    453               return false;
    454           }
    455         }
    456         return true;
    457     }
    458 
    459     /**
    460      * Determines if a given string is a valid java identifier.
    461      * @param javaIdentifier
    462      * @return true if it is a valid identifier, false otherwise
    463      */
    464     private static boolean isValidJavaIdentifier(String javaIdentifier) {
    465         if (javaIdentifier.length() == 0 ||
    466                 !Character.isJavaIdentifierStart(javaIdentifier.charAt(0))) {
    467             return false;
    468         }
    469         for (int i = 1; i < javaIdentifier.length(); i++) {
    470             if (!Character.isJavaIdentifierPart(javaIdentifier.charAt(i))) {
    471                 return false;
    472             }
    473         }
    474         return true;
    475     }
    476 }
    477