Home | History | Annotate | Download | only in utils 
      1 #!/usr/bin/python
      2 
      3 # Copyright (C) 2009 The Android Open Source Project
      4 #
      5 # Licensed under the Apache License, Version 2.0 (the "License");
      6 # you may not use this file except in compliance with the License.
      7 # You may obtain a copy of the License at
      8 #
      9 #       http://www.apache.org/licenses/LICENSE-2.0
     10 #
     11 # Unless required by applicable law or agreed to in writing, software
     12 # distributed under the License is distributed on an "AS IS" BASIS,
     13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     14 # See the License for the specific language governing permissions and
     15 # limitations under the License.
     16 
     17 """Module for generating CTS test descriptions and test plans."""
     18 
     19 import glob
     20 import os
     21 import re
     22 import shutil
     23 import subprocess
     24 import sys
     25 import xml.dom.minidom as dom
     26 from cts import tools
     27 from multiprocessing import Pool
     28 
     29 def GetSubDirectories(root):
     30   """Return all directories under the given root directory."""
     31   return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))]
     32 
     33 def ReadFileLines(filePath):
     34   """Reads a file and returns its contents as a line list."""
     35   f = open(filePath, 'r');
     36   lines = [line.strip() for line in f.readlines()]
     37   f.close()
     38   return lines
     39 
     40 def GetMakeFileVars(makefile_path):
     41   """Extracts variable definitions from the given make file.
     42 
     43   Args:
     44     makefile_path: Path to the make file.
     45 
     46   Returns:
     47     A dictionary mapping variable names to their assigned value.
     48   """
     49   result = {}
     50   pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL)
     51   stream = open(makefile_path, 'r')
     52   content = stream.read()
     53   for match in pattern.finditer(content):
     54     result[match.group(1)] = match.group(2)
     55   stream.close()
     56   return result
     57 
     58 
     59 class CtsBuilder(object):
     60   """Main class for generating test descriptions and test plans."""
     61 
     62   def __init__(self, argv):
     63     """Initialize the CtsBuilder from command line arguments."""
     64     if len(argv) != 6:
     65       print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0]
     66       print ''
     67       print 'testRoot:       Directory under which to search for CTS tests.'
     68       print 'ctsOutputDir:   Directory in which the CTS repository should be created.'
     69       print 'tempDir:        Directory to use for storing temporary files.'
     70       print 'androidRootDir: Root directory of the Android source tree.'
     71       print 'docletPath:     Class path where the DescriptionGenerator doclet can be found.'
     72       sys.exit(1)
     73     self.test_root = sys.argv[1]
     74     self.out_dir = sys.argv[2]
     75     self.temp_dir = sys.argv[3]
     76     self.android_root = sys.argv[4]
     77     self.doclet_path = sys.argv[5]
     78 
     79     self.test_repository = os.path.join(self.out_dir, 'repository/testcases')
     80     self.plan_repository = os.path.join(self.out_dir, 'repository/plans')
     81     self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans')
     82 
     83   def GenerateTestDescriptions(self):
     84     """Generate test descriptions for all packages."""
     85     pool = Pool(processes=2)
     86 
     87     # generate test descriptions for android tests
     88     results = []
     89     pool.close()
     90     pool.join()
     91     return sum(map(lambda result: result.get(), results))
     92 
     93   def __WritePlan(self, plan, plan_name):
     94     print 'Generating test plan %s' % plan_name
     95     plan.Write(os.path.join(self.plan_repository, plan_name + '.xml'))
     96 
     97   def GenerateTestPlans(self):
     98     """Generate default test plans."""
     99     # TODO: Instead of hard-coding the plans here, use a configuration file,
    100     # such as test_defs.xml
    101     packages = []
    102     descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml')))
    103     for description in descriptions:
    104       doc = tools.XmlFile(description)
    105       packages.append(doc.GetAttr('TestPackage', 'appPackageName'))
    106     # sort the list to give the same sequence based on name
    107     packages.sort()
    108 
    109     plan = tools.TestPlan(packages)
    110     plan.Exclude('android\.performance.*')
    111     self.__WritePlan(plan, 'CTS')
    112     self.__WritePlan(plan, 'CTS-TF')
    113 
    114     plan = tools.TestPlan(packages)
    115     plan.Exclude('android\.performance.*')
    116     plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*')
    117     # Test plan to not include media streaming tests
    118     self.__WritePlan(plan, 'CTS-No-Media-Stream')
    119 
    120     plan = tools.TestPlan(packages)
    121     plan.Exclude('android\.performance.*')
    122     self.__WritePlan(plan, 'SDK')
    123 
    124     plan.Exclude(r'android\.signature')
    125     plan.Exclude(r'android\.core.*')
    126     self.__WritePlan(plan, 'Android')
    127 
    128     plan = tools.TestPlan(packages)
    129     plan.Include(r'android\.core\.tests.*')
    130     plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*')
    131     self.__WritePlan(plan, 'Java')
    132 
    133     # TODO: remove this once the tests are fixed and merged into Java plan above.
    134     plan = tools.TestPlan(packages)
    135     plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*')
    136     self.__WritePlan(plan, 'Harmony')
    137 
    138     plan = tools.TestPlan(packages)
    139     plan.Include(r'android\.core\.vm-tests-tf')
    140     self.__WritePlan(plan, 'VM-TF')
    141 
    142     plan = tools.TestPlan(packages)
    143     plan.Include(r'android\.tests\.appsecurity')
    144     self.__WritePlan(plan, 'AppSecurity')
    145 
    146     # hard-coded white list for PDK plan
    147     plan.Exclude('.*')
    148     plan.Include('android\.aadb')
    149     plan.Include('android\.bluetooth')
    150     plan.Include('android\.graphics.*')
    151     plan.Include('android\.hardware')
    152     plan.Include('android\.media')
    153     plan.Exclude('android\.mediastress')
    154     plan.Include('android\.net')
    155     plan.Include('android\.opengl.*')
    156     plan.Include('android\.renderscript')
    157     plan.Include('android\.telephony')
    158     plan.Include('android\.nativemedia.*')
    159     plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these?
    160     self.__WritePlan(plan, 'PDK')
    161 
    162     temporarily_known_failure_tests = BuildCtsTemporarilyKnownFailureList();
    163     flaky_tests = BuildCtsFlakyTestList()
    164     releasekey_tests = BuildListForReleaseBuildTest()
    165 
    166     # CTS Stable plan
    167     plan = tools.TestPlan(packages)
    168     plan.Exclude(r'com\.android\.cts\.browserbench')
    169     for package, test_list in flaky_tests.iteritems():
    170       plan.ExcludeTests(package, test_list)
    171     for package, test_list in releasekey_tests.iteritems():
    172       plan.ExcludeTests(package, test_list)
    173     self.__WritePlan(plan, 'CTS-stable')
    174 
    175     # CTS Flaky plan - list of tests known to be flaky in lab environment
    176     plan = tools.TestPlan(packages)
    177     plan.Exclude('.*')
    178     plan.Include(r'com\.android\.cts\.browserbench')
    179     for package, test_list in flaky_tests.iteritems():
    180       plan.Include(package+'$')
    181       plan.IncludeTests(package, test_list)
    182     self.__WritePlan(plan, 'CTS-flaky')
    183 
    184     small_tests = BuildAospSmallSizeTestList()
    185     medium_tests = BuildAospMediumSizeTestList()
    186     new_test_packages = BuildCtsVettedNewPackagesList()
    187 
    188     # CTS - sub plan for public, small size tests
    189     plan = tools.TestPlan(packages)
    190     plan.Exclude('.*')
    191     for package, test_list in small_tests.iteritems():
    192       plan.Include(package+'$')
    193     plan.Exclude(r'com\.android\.cts\.browserbench')
    194     for package, test_list in flaky_tests.iteritems():
    195       plan.ExcludeTests(package, test_list)
    196     for package, test_list in releasekey_tests.iteritems():
    197       plan.ExcludeTests(package, test_list)
    198     self.__WritePlan(plan, 'CTS-kitkat-small')
    199     self.__WritePlan(plan, 'CTS-public-small')
    200 
    201     # CTS - sub plan for public, medium size tests
    202     plan = tools.TestPlan(packages)
    203     plan.Exclude('.*')
    204     for package, test_list in medium_tests.iteritems():
    205       plan.Include(package+'$')
    206     plan.Exclude(r'com\.android\.cts\.browserbench')
    207     for package, test_list in flaky_tests.iteritems():
    208       plan.ExcludeTests(package, test_list)
    209     for package, test_list in releasekey_tests.iteritems():
    210       plan.ExcludeTests(package, test_list)
    211     self.__WritePlan(plan, 'CTS-kitkat-medium')
    212     self.__WritePlan(plan, 'CTS-public-medium')
    213 
    214     # CTS - sub plan for hardware tests which is public, large
    215     plan = tools.TestPlan(packages)
    216     plan.Exclude('.*')
    217     plan.Include(r'android\.hardware$')
    218     plan.Exclude(r'com\.android\.cts\.browserbench')
    219     for package, test_list in flaky_tests.iteritems():
    220       plan.ExcludeTests(package, test_list)
    221     for package, test_list in releasekey_tests.iteritems():
    222       plan.ExcludeTests(package, test_list)
    223     self.__WritePlan(plan, 'CTS-hardware')
    224 
    225     # CTS - sub plan for media tests which is public, large
    226     plan = tools.TestPlan(packages)
    227     plan.Exclude('.*')
    228     plan.Include(r'android\.media$')
    229     plan.Include(r'android\.view$')
    230     plan.Exclude(r'com\.android\.cts\.browserbench')
    231     for package, test_list in flaky_tests.iteritems():
    232       plan.ExcludeTests(package, test_list)
    233     for package, test_list in releasekey_tests.iteritems():
    234       plan.ExcludeTests(package, test_list)
    235     self.__WritePlan(plan, 'CTS-media')
    236 
    237     # CTS - sub plan for mediastress tests which is public, large
    238     plan = tools.TestPlan(packages)
    239     plan.Exclude('.*')
    240     plan.Include(r'android\.mediastress$')
    241     plan.Exclude(r'com\.android\.cts\.browserbench')
    242     for package, test_list in flaky_tests.iteritems():
    243       plan.ExcludeTests(package, test_list)
    244     for package, test_list in releasekey_tests.iteritems():
    245       plan.ExcludeTests(package, test_list)
    246     self.__WritePlan(plan, 'CTS-mediastress')
    247 
    248     # CTS - sub plan for new tests that is vetted for L launch
    249     plan = tools.TestPlan(packages)
    250     plan.Exclude('.*')
    251     for package, test_list in new_test_packages.iteritems():
    252       plan.Include(package+'$')
    253     plan.Exclude(r'com\.android\.cts\.browserbench')
    254     for package, test_list in flaky_tests.iteritems():
    255       plan.ExcludeTests(package, test_list)
    256     for package, test_list in releasekey_tests.iteritems():
    257       plan.ExcludeTests(package, test_list)
    258     self.__WritePlan(plan, 'CTS-l-tests')
    259 
    260     # CTS - sub plan for tests in drawelement packages
    261     plan = tools.TestPlan(packages)
    262     plan.Exclude('.*')
    263     plan.Include(r'com\.drawelements\.')
    264     self.__WritePlan(plan, 'CTS-DEQP')
    265 
    266     # CTS - sub plan for new test packages added for staging
    267     plan = tools.TestPlan(packages)
    268     for package, test_list in small_tests.iteritems():
    269       plan.Exclude(package+'$')
    270     for package, test_list in medium_tests.iteritems():
    271       plan.Exclude(package+'$')
    272     for package, tests_list in new_test_packages.iteritems():
    273       plan.Exclude(package+'$')
    274     plan.Exclude(r'com\.drawelements\.')
    275     plan.Exclude(r'android\.hardware$')
    276     plan.Exclude(r'android\.media$')
    277     plan.Exclude(r'android\.view$')
    278     plan.Exclude(r'android\.mediastress$')
    279     plan.Exclude(r'com\.android\.cts\.browserbench')
    280     for package, test_list in flaky_tests.iteritems():
    281       plan.ExcludeTests(package, test_list)
    282     for package, test_list in releasekey_tests.iteritems():
    283       plan.ExcludeTests(package, test_list)
    284     self.__WritePlan(plan, 'CTS-m-tests')
    285 
    286 
    287     # CTS - sub plan for new test packages added for staging
    288     plan = tools.TestPlan(packages)
    289     plan.Exclude('.*')
    290     for package, test_list in temporarily_known_failure_tests.iteritems():
    291       plan.Include(package+'$')
    292       plan.IncludeTests(package, test_list)
    293     self.__WritePlan(plan, 'CTS-staging')
    294 
    295     plan = tools.TestPlan(packages)
    296     plan.Exclude('.*')
    297     self.__WritePlan(plan, 'CTS-webview')
    298 
    299 
    300 def BuildAospMediumSizeTestList():
    301   """ Construct a defaultdic that lists package names of medium tests
    302       already published to aosp. """
    303   return {
    304       'android.app' : [],
    305       'android.core.tests.libcore.package.libcore' : [],
    306       'android.core.tests.libcore.package.org' : [],
    307       'android.core.vm-tests-tf' : [],
    308       'android.dpi' : [],
    309       'android.host.security' : [],
    310       'android.net' : [],
    311       'android.os' : [],
    312       'android.permission2' : [],
    313       'android.security' : [],
    314       'android.telephony' : [],
    315       'android.webkit' : [],
    316       'android.widget' : [],
    317       'com.android.cts.browserbench' : []}
    318 
    319 def BuildAospSmallSizeTestList():
    320   """ Construct a default dict that lists packages names of small tests
    321       already published to aosp. """
    322   return {
    323       'android.aadb' : [],
    324       'android.acceleration' : [],
    325       'android.accessibility' : [],
    326       'android.accessibilityservice' : [],
    327       'android.accounts' : [],
    328       'android.admin' : [],
    329       'android.animation' : [],
    330       'android.bionic' : [],
    331       'android.bluetooth' : [],
    332       'android.calendarcommon' : [],
    333       'android.content' : [],
    334       'android.core.tests.libcore.package.com' : [],
    335       'android.core.tests.libcore.package.conscrypt' : [],
    336       'android.core.tests.libcore.package.dalvik' : [],
    337       'android.core.tests.libcore.package.sun' : [],
    338       'android.core.tests.libcore.package.tests' : [],
    339       'android.database' : [],
    340       'android.dreams' : [],
    341       'android.drm' : [],
    342       'android.effect' : [],
    343       'android.gesture' : [],
    344       'android.graphics' : [],
    345       'android.graphics2' : [],
    346       'android.jni' : [],
    347       'android.keystore' : [],
    348       'android.location' : [],
    349       'android.nativemedia.sl' : [],
    350       'android.nativemedia.xa' : [],
    351       'android.nativeopengl' : [],
    352       'android.ndef' : [],
    353       'android.opengl' : [],
    354       'android.openglperf' : [],
    355       'android.permission' : [],
    356       'android.preference' : [],
    357       'android.preference2' : [],
    358       'android.provider' : [],
    359       'android.renderscript' : [],
    360       'android.rscpp' : [],
    361       'android.rsg' : [],
    362       'android.sax' : [],
    363       'android.signature' : [],
    364       'android.speech' : [],
    365       'android.tests.appsecurity' : [],
    366       'android.text' : [],
    367       'android.textureview' : [],
    368       'android.theme' : [],
    369       'android.usb' : [],
    370       'android.util' : [],
    371       'com.android.cts.dram' : [],
    372       'com.android.cts.filesystemperf' : [],
    373       'com.android.cts.jank' : [],
    374       'com.android.cts.jank2' : [],
    375       'com.android.cts.opengl' : [],
    376       'com.android.cts.simplecpu' : [],
    377       'com.android.cts.ui' : [],
    378       'com.android.cts.uihost' : [],
    379       'com.android.cts.videoperf' : [],
    380       'zzz.android.monkey' : []}
    381 
    382 def BuildCtsVettedNewPackagesList():
    383   """ Construct a defaultdict that maps package names that is vetted for L. """
    384   return {
    385       'android.JobScheduler' : [],
    386       'android.core.tests.libcore.package.harmony_annotation' : [],
    387       'android.core.tests.libcore.package.harmony_beans' : [],
    388       'android.core.tests.libcore.package.harmony_java_io' : [],
    389       'android.core.tests.libcore.package.harmony_java_lang' : [],
    390       'android.core.tests.libcore.package.harmony_java_math' : [],
    391       'android.core.tests.libcore.package.harmony_java_net' : [],
    392       'android.core.tests.libcore.package.harmony_java_nio' : [],
    393       'android.core.tests.libcore.package.harmony_java_util' : [],
    394       'android.core.tests.libcore.package.harmony_java_text' : [],
    395       'android.core.tests.libcore.package.harmony_javax_security' : [],
    396       'android.core.tests.libcore.package.harmony_logging' : [],
    397       'android.core.tests.libcore.package.harmony_prefs' : [],
    398       'android.core.tests.libcore.package.harmony_sql' : [],
    399       'android.core.tests.libcore.package.jsr166' : [],
    400       'android.core.tests.libcore.package.okhttp' : [],
    401       'android.display' : [],
    402       'android.host.theme' : [],
    403       'android.jdwp' : [],
    404       'android.location2' : [],
    405       'android.print' : [],
    406       'android.renderscriptlegacy' : [],
    407       'android.signature' : [],
    408       'android.tv' : [],
    409       'android.uiautomation' : [],
    410       'android.uirendering' : []}
    411 
    412 def BuildListForReleaseBuildTest():
    413   """ Construct a defaultdict that maps package name to a list of tests
    414       that are expected to pass only when running against a user/release-key build. """
    415   return {
    416       'android.app' : [
    417           'android.app.cts.ActivityManagerTest#testIsRunningInTestHarness',],
    418       'android.dpi' : [
    419           'android.dpi.cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',],
    420       'android.host.security' : [
    421           'android.cts.security.SELinuxHostTest#testAllEnforcing',
    422           'android.cts.security.SELinuxHostTest#testSuDomain',],
    423       'android.os' : [
    424           'android.os.cts.BuildVersionTest#testReleaseVersion',
    425           'android.os.cts.BuildTest#testIsSecureUserBuild',],
    426       'android.security' : [
    427           'android.security.cts.BannedFilesTest#testNoSu',
    428           'android.security.cts.BannedFilesTest#testNoSuInPath',
    429           'android.security.cts.PackageSignatureTest#testPackageSignatures',
    430           'android.security.cts.SELinuxDomainTest#testSuDomain',],
    431       '' : []}
    432 
    433 def BuildCtsFlakyTestList():
    434   """ Construct a defaultdict that maps package name to a list of tests
    435       that flaky during dev cycle and cause other subsequent tests to fail. """
    436   return {
    437       'android.hardware' : [
    438           'android.hardware.cts.CameraTest#testVideoSnapshot',
    439           'android.hardware.cts.CameraGLTest#testCameraToSurfaceTextureMetadata',
    440           'android.hardware.cts.CameraGLTest#testSetPreviewTextureBothCallbacks',
    441           'android.hardware.cts.CameraGLTest#testSetPreviewTexturePreviewCallback',],
    442       'android.media' : [
    443           'android.media.cts.DecoderTest#testCodecResetsH264WithSurface',
    444           'android.media.cts.StreamingMediaPlayerTest#testHLS',],
    445       'android.net' : [
    446           'android.net.cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri',
    447           'android.net.cts.DnsTest#testDnsWorks',
    448           'android.net.cts.SSLCertificateSocketFactoryTest#testCreateSocket',
    449           'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_bind',
    450           'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_simple',
    451           'android.net.cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping',
    452           'android.net.cts.TrafficStatsTest#testTrafficStatsForLocalhost',
    453           'android.net.wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',],
    454       'android.security' : [
    455           'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports',
    456           'android.security.cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts',],
    457       'android.webkit' : [
    458           'android.webkit.cts.WebViewClientTest#testOnUnhandledKeyEvent',],
    459       'com.android.cts.filesystemperf' : [
    460           'com.android.cts.filesystemperf.RandomRWTest#testRandomRead',
    461           'com.android.cts.filesystemperf.RandomRWTest#testRandomUpdate',],
    462       '' : []}
    463 
    464 def BuildCtsTemporarilyKnownFailureList():
    465   """ Construct a defaultdict that maps package name to a list of tests
    466       that are known failures during dev cycle but expected to be fixed before launch """
    467   return {
    468       'android.alarmclock' : [
    469           'android.alarmclock.cts.DismissAlarmTest#testAll',
    470           'android.alarmclock.cts.SetAlarmTest#testAll',
    471           'android.alarmclock.cts.SnoozeAlarmTest#testAll',
    472       ],
    473       'android.assist' : [
    474           'android.assist.cts.ExtraAssistDataTest',
    475           'android.assist.cts.AssistantContentViewTest',
    476           'android.assist.cts.ScreenshotTest',
    477       ],
    478       'android.calllog' : [
    479           'android.calllog.cts.CallLogBackupTest#testSingleCallBackup',
    480       ],
    481       'android.dumpsys' : [
    482           'android.dumpsys.cts.DumpsysHostTest#testBatterystatsOutput',
    483           'android.dumpsys.cts.DumpsysHostTest#testGfxinfoFramestats',
    484       ],
    485       'android.telecom' : [
    486           'android.telecom.cts.ExtendedInCallServiceTest#testAddNewOutgoingCallAndThenDisconnect',
    487           'android.telecom.cts.RemoteConferenceTest#testRemoteConferenceCallbacks_ConferenceableConnections',
    488       ],
    489       'android.transition' : [
    490           'android.transition.cts.ChangeScrollTest#testChangeScroll',
    491       ],
    492       'android.voicesettings' : [
    493           'android.voicesettings.cts.ZenModeTest#testAll',
    494       ],
    495       'com.android.cts.app.os' : [
    496           'com.android.cts.app.os.OsHostTests#testNonExportedActivities',
    497       ],
    498       'com.android.cts.devicepolicy' : [
    499           'com.android.cts.devicepolicy.MixedDeviceOwnerTest#testPackageInstallUserRestrictions',
    500           'com.android.cts.devicepolicy.MixedProfileOwnerTest#testPackageInstallUserRestrictions',
    501       ],
    502       '' : []}
    503 
    504 def LogGenerateDescription(name):
    505   print 'Generating test description for package %s' % name
    506 
    507 if __name__ == '__main__':
    508   builder = CtsBuilder(sys.argv)
    509   result = builder.GenerateTestDescriptions()
    510   if result != 0:
    511     sys.exit(result)
    512   builder.GenerateTestPlans()
    513 
    514