Home | History | Annotate | Download | only in utils 
      1 #!/usr/bin/python
      2 
      3 # Copyright (C) 2009 The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #       http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 """Module for generating CTS test descriptions and test plans."""
     18 
     19 import glob
     20 import os
     21 import re
     22 import shutil
     23 import subprocess
     24 import sys
     25 import xml.dom.minidom as dom
     26 from cts import tools
     27 from multiprocessing import Pool
     28 
     29 def GetSubDirectories(root):
     30   """Return all directories under the given root directory."""
     31   return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))]
     32 
     33 
     34 def GetMakeFileVars(makefile_path):
     35   """Extracts variable definitions from the given make file.
     36 
     37   Args:
     38     makefile_path: Path to the make file.
     39 
     40   Returns:
     41     A dictionary mapping variable names to their assigned value.
     42   """
     43   result = {}
     44   pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL)
     45   stream = open(makefile_path, 'r')
     46   content = stream.read()
     47   for match in pattern.finditer(content):
     48     result[match.group(1)] = match.group(2)
     49   stream.close()
     50   return result
     51 
     52 
     53 class CtsBuilder(object):
     54   """Main class for generating test descriptions and test plans."""
     55 
     56   def __init__(self, argv):
     57     """Initialize the CtsBuilder from command line arguments."""
     58     if len(argv) != 6:
     59       print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0]
     60       print ''
     61       print 'testRoot:       Directory under which to search for CTS tests.'
     62       print 'ctsOutputDir:   Directory in which the CTS repository should be created.'
     63       print 'tempDir:        Directory to use for storing temporary files.'
     64       print 'androidRootDir: Root directory of the Android source tree.'
     65       print 'docletPath:     Class path where the DescriptionGenerator doclet can be found.'
     66       sys.exit(1)
     67     self.test_root = sys.argv[1]
     68     self.out_dir = sys.argv[2]
     69     self.temp_dir = sys.argv[3]
     70     self.android_root = sys.argv[4]
     71     self.doclet_path = sys.argv[5]
     72 
     73     self.test_repository = os.path.join(self.out_dir, 'repository/testcases')
     74     self.plan_repository = os.path.join(self.out_dir, 'repository/plans')
     75     self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans')
     76 
     77   def GenerateTestDescriptions(self):
     78     """Generate test descriptions for all packages."""
     79     pool = Pool(processes=2)
     80 
     81     # generate test descriptions for android tests
     82     results = []
     83     pool.close()
     84     pool.join()
     85     return sum(map(lambda result: result.get(), results))
     86 
     87   def __WritePlan(self, plan, plan_name):
     88     print 'Generating test plan %s' % plan_name
     89     plan.Write(os.path.join(self.plan_repository, plan_name + '.xml'))
     90 
     91   def GenerateTestPlans(self):
     92     """Generate default test plans."""
     93     # TODO: Instead of hard-coding the plans here, use a configuration file,
     94     # such as test_defs.xml
     95     packages = []
     96     descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml')))
     97     for description in descriptions:
     98       doc = tools.XmlFile(description)
     99       packages.append(doc.GetAttr('TestPackage', 'appPackageName'))
    100     # sort the list to give the same sequence based on name
    101     packages.sort()
    102 
    103     plan = tools.TestPlan(packages)
    104     plan.Exclude('android\.performance.*')
    105     self.__WritePlan(plan, 'CTS')
    106     self.__WritePlan(plan, 'CTS-TF')
    107 
    108     plan = tools.TestPlan(packages)
    109     plan.Exclude('android\.performance.*')
    110     plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*')
    111     # Test plan to not include media streaming tests
    112     self.__WritePlan(plan, 'CTS-No-Media-Stream')
    113 
    114     plan = tools.TestPlan(packages)
    115     plan.Exclude('android\.performance.*')
    116     self.__WritePlan(plan, 'SDK')
    117 
    118     plan.Exclude(r'android\.signature')
    119     plan.Exclude(r'android\.core.*')
    120     self.__WritePlan(plan, 'Android')
    121 
    122     plan = tools.TestPlan(packages)
    123     plan.Include(r'android\.core\.tests.*')
    124     plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*')
    125     self.__WritePlan(plan, 'Java')
    126 
    127     # TODO: remove this once the tests are fixed and merged into Java plan above.
    128     plan = tools.TestPlan(packages)
    129     plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*')
    130     self.__WritePlan(plan, 'Harmony')
    131 
    132     plan = tools.TestPlan(packages)
    133     plan.Include(r'android\.core\.vm-tests-tf')
    134     self.__WritePlan(plan, 'VM-TF')
    135 
    136     plan = tools.TestPlan(packages)
    137     plan.Include(r'android\.tests\.appsecurity')
    138     self.__WritePlan(plan, 'AppSecurity')
    139 
    140     # hard-coded white list for PDK plan
    141     plan.Exclude('.*')
    142     plan.Include('android\.aadb')
    143     plan.Include('android\.bluetooth')
    144     plan.Include('android\.graphics.*')
    145     plan.Include('android\.hardware')
    146     plan.Include('android\.media')
    147     plan.Exclude('android\.mediastress')
    148     plan.Include('android\.net')
    149     plan.Include('android\.opengl.*')
    150     plan.Include('android\.renderscript')
    151     plan.Include('android\.telephony')
    152     plan.Include('android\.nativemedia.*')
    153     plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these?
    154     self.__WritePlan(plan, 'PDK')
    155 
    156     flaky_tests = BuildCtsFlakyTestList()
    157 
    158     # CTS Stable plan
    159     plan = tools.TestPlan(packages)
    160     plan.Exclude(r'com\.android\.cts\.browserbench')
    161     for package, test_list in flaky_tests.iteritems():
    162       plan.ExcludeTests(package, test_list)
    163     self.__WritePlan(plan, 'CTS-stable')
    164 
    165     # CTS Flaky plan - list of tests known to be flaky in lab environment
    166     plan = tools.TestPlan(packages)
    167     plan.Exclude('.*')
    168     plan.Include(r'com\.android\.cts\.browserbench')
    169     for package, test_list in flaky_tests.iteritems():
    170       plan.Include(package+'$')
    171       plan.IncludeTests(package, test_list)
    172     self.__WritePlan(plan, 'CTS-flaky')
    173 
    174     small_tests = BuildAospSmallSizeTestList()
    175     medium_tests = BuildAospMediumSizeTestList()
    176     new_test_packages = BuildCtsVettedNewPackagesList()
    177 
    178     # CTS - sub plan for public, small size tests
    179     plan = tools.TestPlan(packages)
    180     plan.Exclude('.*')
    181     for package, test_list in small_tests.iteritems():
    182       plan.Include(package+'$')
    183     plan.Exclude(r'com\.android\.cts\.browserbench')
    184     for package, test_list in flaky_tests.iteritems():
    185       plan.ExcludeTests(package, test_list)
    186     self.__WritePlan(plan, 'CTS-kitkat-small')
    187 
    188     # CTS - sub plan for public, medium size tests
    189     plan = tools.TestPlan(packages)
    190     plan.Exclude('.*')
    191     for package, test_list in medium_tests.iteritems():
    192       plan.Include(package+'$')
    193     plan.Exclude(r'com\.android\.cts\.browserbench')
    194     for package, test_list in flaky_tests.iteritems():
    195       plan.ExcludeTests(package, test_list)
    196     self.__WritePlan(plan, 'CTS-kitkat-medium')
    197 
    198     # CTS - sub plan for hardware tests which is public, large
    199     plan = tools.TestPlan(packages)
    200     plan.Exclude('.*')
    201     plan.Include(r'android\.hardware$')
    202     plan.Exclude(r'com\.android\.cts\.browserbench')
    203     for package, test_list in flaky_tests.iteritems():
    204       plan.ExcludeTests(package, test_list)
    205     self.__WritePlan(plan, 'CTS-hardware')
    206 
    207     # CTS - sub plan for media tests which is public, large
    208     plan = tools.TestPlan(packages)
    209     plan.Exclude('.*')
    210     plan.Include(r'android\.media$')
    211     plan.Include(r'android\.view$')
    212     plan.Exclude(r'com\.android\.cts\.browserbench')
    213     for package, test_list in flaky_tests.iteritems():
    214       plan.ExcludeTests(package, test_list)
    215     self.__WritePlan(plan, 'CTS-media')
    216 
    217     # CTS - sub plan for mediastress tests which is public, large
    218     plan = tools.TestPlan(packages)
    219     plan.Exclude('.*')
    220     plan.Include(r'android\.mediastress$')
    221     plan.Exclude(r'com\.android\.cts\.browserbench')
    222     for package, test_list in flaky_tests.iteritems():
    223       plan.ExcludeTests(package, test_list)
    224     self.__WritePlan(plan, 'CTS-mediastress')
    225 
    226     # CTS - sub plan for new tests that is vetted for L launch
    227     plan = tools.TestPlan(packages)
    228     plan.Exclude('.*')
    229     for package, test_list in new_test_packages.iteritems():
    230       plan.Include(package+'$')
    231     plan.Exclude(r'com\.android\.cts\.browserbench')
    232     for package, test_list in flaky_tests.iteritems():
    233       plan.ExcludeTests(package, test_list)
    234     self.__WritePlan(plan, 'CTS-l-tests')
    235 
    236     #CTS - sub plan for new test packages added for staging
    237     plan = tools.TestPlan(packages)
    238     for package, test_list in small_tests.iteritems():
    239       plan.Exclude(package+'$')
    240     for package, test_list in medium_tests.iteritems():
    241       plan.Exclude(package+'$')
    242     for package, tests_list in new_test_packages.iteritems():
    243       plan.Exclude(package+'$')
    244     plan.Exclude(r'android\.hardware$')
    245     plan.Exclude(r'android\.media$')
    246     plan.Exclude(r'android\.view$')
    247     plan.Exclude(r'android\.mediastress$')
    248     plan.Exclude(r'com\.android\.cts\.browserbench')
    249     for package, test_list in flaky_tests.iteritems():
    250       plan.ExcludeTests(package, test_list)
    251     self.__WritePlan(plan, 'CTS-staging')
    252 
    253     plan = tools.TestPlan(packages)
    254     plan.Exclude('.*')
    255     plan.Include(r'android\.core\.tests\.libcore\.')
    256     plan.Include(r'android\.jdwp')
    257     for package, test_list in small_tests.iteritems():
    258       plan.Exclude(package+'$')
    259     for package, test_list in medium_tests.iteritems():
    260       plan.Exclude(package+'$')
    261     for package, tests_list in new_test_packages.iteritems():
    262       plan.Exclude(package+'$')
    263     self.__WritePlan(plan, 'CTS-ART')
    264 
    265     plan = tools.TestPlan(packages)
    266     plan.Exclude('.*')
    267     plan.Include(r'com\.drawelements\.')
    268     self.__WritePlan(plan, 'CTS-DEQP')
    269 
    270     plan = tools.TestPlan(packages)
    271     plan.Exclude('.*')
    272     plan.Include(r'android\.webgl')
    273     self.__WritePlan(plan, 'CTS-webview')
    274 
    275 
    276 def BuildAospMediumSizeTestList():
    277   """ Construct a defaultdic that lists package names of medium tests
    278       already published to aosp. """
    279   return {
    280       'android.app' : [],
    281       'android.core.tests.libcore.package.libcore' : [],
    282       'android.core.tests.libcore.package.org' : [],
    283       'android.core.vm-tests-tf' : [],
    284       'android.dpi' : [],
    285       'android.host.security' : [],
    286       'android.net' : [],
    287       'android.os' : [],
    288       'android.permission2' : [],
    289       'android.security' : [],
    290       'android.telephony' : [],
    291       'android.webkit' : [],
    292       'android.widget' : [],
    293       'com.android.cts.browserbench' : []}
    294 
    295 def BuildAospSmallSizeTestList():
    296   """ Construct a defaultdict that lists packages names of small tests
    297       already published to aosp. """
    298   return {
    299       'android.aadb' : [],
    300       'android.acceleration' : [],
    301       'android.accessibility' : [],
    302       'android.accessibilityservice' : [],
    303       'android.accounts' : [],
    304       'android.admin' : [],
    305       'android.animation' : [],
    306       'android.bionic' : [],
    307       'android.bluetooth' : [],
    308       'android.calendarcommon' : [],
    309       'android.content' : [],
    310       'android.core.tests.libcore.package.com' : [],
    311       'android.core.tests.libcore.package.conscrypt' : [],
    312       'android.core.tests.libcore.package.dalvik' : [],
    313       'android.core.tests.libcore.package.sun' : [],
    314       'android.core.tests.libcore.package.tests' : [],
    315       'android.database' : [],
    316       'android.dreams' : [],
    317       'android.drm' : [],
    318       'android.effect' : [],
    319       'android.gesture' : [],
    320       'android.graphics' : [],
    321       'android.graphics2' : [],
    322       'android.jni' : [],
    323       'android.keystore' : [],
    324       'android.location' : [],
    325       'android.nativemedia.sl' : [],
    326       'android.nativemedia.xa' : [],
    327       'android.nativeopengl' : [],
    328       'android.ndef' : [],
    329       'android.opengl' : [],
    330       'android.openglperf' : [],
    331       'android.permission' : [],
    332       'android.preference' : [],
    333       'android.preference2' : [],
    334       'android.provider' : [],
    335       'android.renderscript' : [],
    336       'android.rscpp' : [],
    337       'android.rsg' : [],
    338       'android.sax' : [],
    339       'android.signature' : [],
    340       'android.speech' : [],
    341       'android.tests.appsecurity' : [],
    342       'android.text' : [],
    343       'android.textureview' : [],
    344       'android.theme' : [],
    345       'android.usb' : [],
    346       'android.util' : [],
    347       'com.android.cts.dram' : [],
    348       'com.android.cts.filesystemperf' : [],
    349       'com.android.cts.jank' : [],
    350       'com.android.cts.opengl' : [],
    351       'com.android.cts.simplecpu' : [],
    352       'com.android.cts.ui' : [],
    353       'com.android.cts.uihost' : [],
    354       'com.android.cts.videoperf' : [],
    355       'zzz.android.monkey' : []}
    356 
    357 def BuildCtsVettedNewPackagesList():
    358   """ Construct a defaultdict that maps package names that is vetted for L. """
    359   return {
    360       'android.JobScheduler' : [],
    361       'android.core.tests.libcore.package.harmony_annotation' : [],
    362       'android.core.tests.libcore.package.harmony_beans' : [],
    363       'android.core.tests.libcore.package.harmony_java_io' : [],
    364       'android.core.tests.libcore.package.harmony_java_lang' : [],
    365       'android.core.tests.libcore.package.harmony_java_math' : [],
    366       'android.core.tests.libcore.package.harmony_java_net' : [],
    367       'android.core.tests.libcore.package.harmony_java_nio' : [],
    368       'android.core.tests.libcore.package.harmony_java_util' : [],
    369       'android.core.tests.libcore.package.harmony_java_text' : [],
    370       'android.core.tests.libcore.package.harmony_javax_security' : [],
    371       'android.core.tests.libcore.package.harmony_logging' : [],
    372       'android.core.tests.libcore.package.harmony_prefs' : [],
    373       'android.core.tests.libcore.package.harmony_sql' : [],
    374       'android.core.tests.libcore.package.jsr166' : [],
    375       'android.core.tests.libcore.package.okhttp' : [],
    376       'android.display' : [],
    377       'android.host.theme' : [],
    378       'android.jdwp' : [],
    379       'android.location2' : [],
    380       'android.print' : [],
    381       'android.renderscriptlegacy' : [],
    382       'android.signature' : [],
    383       'android.tv' : [],
    384       'android.uiautomation' : [],
    385       'android.uirendering' : [],
    386       'android.webgl' : [],
    387       'com.drawelements.deqp.gles3' : [],
    388       'com.drawelements.deqp.gles31' : []}
    389 
    390 def BuildCtsFlakyTestList():
    391   """ Construct a defaultdict that maps package name to a list of tests
    392       that are known to be flaky in the lab or not passing on userdebug builds. """
    393   return {
    394       'android.app' : [
    395           'cts.ActivityManagerTest#testIsRunningInTestHarness',],
    396       'android.dpi' : [
    397           'cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',],
    398       'android.hardware' : [
    399           'cts.CameraTest#testVideoSnapshot',
    400           'cts.CameraGLTest#testCameraToSurfaceTextureMetadata',
    401           'cts.CameraGLTest#testSetPreviewTextureBothCallbacks',
    402           'cts.CameraGLTest#testSetPreviewTexturePreviewCallback',],
    403       'android.media' : [
    404           'cts.DecoderTest#testCodecResetsH264WithSurface',
    405           'cts.StreamingMediaPlayerTest#testHLS',],
    406       'android.net' : [
    407           'cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri',
    408           'cts.DnsTest#testDnsWorks',
    409           'cts.SSLCertificateSocketFactoryTest#testCreateSocket',
    410           'cts.SSLCertificateSocketFactoryTest#test_createSocket_bind',
    411           'cts.SSLCertificateSocketFactoryTest#test_createSocket_simple',
    412           'cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping',
    413           'cts.TrafficStatsTest#testTrafficStatsForLocalhost',
    414           'wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',],
    415       'android.os' : [
    416           'cts.BuildVersionTest#testReleaseVersion',
    417           'cts.BuildTest#testIsSecureUserBuild',],
    418       'android.security' : [
    419           'cts.BannedFilesTest#testNoSu',
    420           'cts.BannedFilesTest#testNoSuInPath',
    421           'cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports',
    422           'cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts',
    423           'cts.PackageSignatureTest#testPackageSignatures',
    424           'cts.SELinuxDomainTest#testSuDomain',
    425           'cts.SELinuxHostTest#testAllEnforcing',],
    426       'android.webkit' : [
    427           'cts.WebViewClientTest#testOnUnhandledKeyEvent',],
    428       'com.android.cts.filesystemperf' : [
    429           'RandomRWTest#testRandomRead',
    430           'RandomRWTest#testRandomUpdate',],
    431       '' : []}
    432 
    433 def LogGenerateDescription(name):
    434   print 'Generating test description for package %s' % name
    435 
    436 if __name__ == '__main__':
    437   builder = CtsBuilder(sys.argv)
    438   result = builder.GenerateTestDescriptions()
    439   if result != 0:
    440     sys.exit(result)
    441   builder.GenerateTestPlans()
    442 
    443