1 #!/usr/bin/python 2 3 # Copyright (C) 2009 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Module for generating CTS test descriptions and test plans.""" 18 19 import glob 20 import os 21 import re 22 import shutil 23 import subprocess 24 import sys 25 import xml.dom.minidom as dom 26 from cts import tools 27 from multiprocessing import Pool 28 29 def GetSubDirectories(root): 30 """Return all directories under the given root directory.""" 31 return [x for x in os.listdir(root) if os.path.isdir(os.path.join(root, x))] 32 33 34 def GetMakeFileVars(makefile_path): 35 """Extracts variable definitions from the given make file. 36 37 Args: 38 makefile_path: Path to the make file. 39 40 Returns: 41 A dictionary mapping variable names to their assigned value. 42 """ 43 result = {} 44 pattern = re.compile(r'^\s*([^:#=\s]+)\s*:=\s*(.*?[^\\])$', re.MULTILINE + re.DOTALL) 45 stream = open(makefile_path, 'r') 46 content = stream.read() 47 for match in pattern.finditer(content): 48 result[match.group(1)] = match.group(2) 49 stream.close() 50 return result 51 52 53 class CtsBuilder(object): 54 """Main class for generating test descriptions and test plans.""" 55 56 def __init__(self, argv): 57 """Initialize the CtsBuilder from command line arguments.""" 58 if len(argv) != 6: 59 print 'Usage: %s <testRoot> <ctsOutputDir> <tempDir> <androidRootDir> <docletPath>' % argv[0] 60 print '' 61 print 'testRoot: Directory under which to search for CTS tests.' 62 print 'ctsOutputDir: Directory in which the CTS repository should be created.' 63 print 'tempDir: Directory to use for storing temporary files.' 64 print 'androidRootDir: Root directory of the Android source tree.' 65 print 'docletPath: Class path where the DescriptionGenerator doclet can be found.' 66 sys.exit(1) 67 self.test_root = sys.argv[1] 68 self.out_dir = sys.argv[2] 69 self.temp_dir = sys.argv[3] 70 self.android_root = sys.argv[4] 71 self.doclet_path = sys.argv[5] 72 73 self.test_repository = os.path.join(self.out_dir, 'repository/testcases') 74 self.plan_repository = os.path.join(self.out_dir, 'repository/plans') 75 self.definedplans_repository = os.path.join(self.android_root, 'cts/tests/plans') 76 77 def GenerateTestDescriptions(self): 78 """Generate test descriptions for all packages.""" 79 pool = Pool(processes=2) 80 81 # generate test descriptions for android tests 82 results = [] 83 pool.close() 84 pool.join() 85 return sum(map(lambda result: result.get(), results)) 86 87 def __WritePlan(self, plan, plan_name): 88 print 'Generating test plan %s' % plan_name 89 plan.Write(os.path.join(self.plan_repository, plan_name + '.xml')) 90 91 def GenerateTestPlans(self): 92 """Generate default test plans.""" 93 # TODO: Instead of hard-coding the plans here, use a configuration file, 94 # such as test_defs.xml 95 packages = [] 96 descriptions = sorted(glob.glob(os.path.join(self.test_repository, '*.xml'))) 97 for description in descriptions: 98 doc = tools.XmlFile(description) 99 packages.append(doc.GetAttr('TestPackage', 'appPackageName')) 100 # sort the list to give the same sequence based on name 101 packages.sort() 102 103 plan = tools.TestPlan(packages) 104 plan.Exclude('android\.performance.*') 105 self.__WritePlan(plan, 'CTS') 106 self.__WritePlan(plan, 'CTS-TF') 107 108 plan = tools.TestPlan(packages) 109 plan.Exclude('android\.performance.*') 110 plan.Exclude('android\.media\.cts\.StreamingMediaPlayerTest.*') 111 # Test plan to not include media streaming tests 112 self.__WritePlan(plan, 'CTS-No-Media-Stream') 113 114 plan = tools.TestPlan(packages) 115 plan.Exclude('android\.performance.*') 116 self.__WritePlan(plan, 'SDK') 117 118 plan.Exclude(r'android\.signature') 119 plan.Exclude(r'android\.core.*') 120 self.__WritePlan(plan, 'Android') 121 122 plan = tools.TestPlan(packages) 123 plan.Include(r'android\.core\.tests.*') 124 plan.Exclude(r'android\.core\.tests\.libcore.\package.\harmony*') 125 self.__WritePlan(plan, 'Java') 126 127 # TODO: remove this once the tests are fixed and merged into Java plan above. 128 plan = tools.TestPlan(packages) 129 plan.Include(r'android\.core\.tests\.libcore.\package.\harmony*') 130 self.__WritePlan(plan, 'Harmony') 131 132 plan = tools.TestPlan(packages) 133 plan.Include(r'android\.core\.vm-tests-tf') 134 self.__WritePlan(plan, 'VM-TF') 135 136 plan = tools.TestPlan(packages) 137 plan.Include(r'android\.tests\.appsecurity') 138 self.__WritePlan(plan, 'AppSecurity') 139 140 # hard-coded white list for PDK plan 141 plan.Exclude('.*') 142 plan.Include('android\.aadb') 143 plan.Include('android\.bluetooth') 144 plan.Include('android\.graphics.*') 145 plan.Include('android\.hardware') 146 plan.Include('android\.media') 147 plan.Exclude('android\.mediastress') 148 plan.Include('android\.net') 149 plan.Include('android\.opengl.*') 150 plan.Include('android\.renderscript') 151 plan.Include('android\.telephony') 152 plan.Include('android\.nativemedia.*') 153 plan.Include('com\.android\.cts\..*')#TODO(stuartscott): Should PDK have all these? 154 self.__WritePlan(plan, 'PDK') 155 156 flaky_tests = BuildCtsFlakyTestList() 157 158 # CTS Stable plan 159 plan = tools.TestPlan(packages) 160 plan.Exclude(r'com\.android\.cts\.browserbench') 161 for package, test_list in flaky_tests.iteritems(): 162 plan.ExcludeTests(package, test_list) 163 self.__WritePlan(plan, 'CTS-stable') 164 165 # CTS Flaky plan - list of tests known to be flaky in lab environment 166 plan = tools.TestPlan(packages) 167 plan.Exclude('.*') 168 plan.Include(r'com\.android\.cts\.browserbench') 169 for package, test_list in flaky_tests.iteritems(): 170 plan.Include(package+'$') 171 plan.IncludeTests(package, test_list) 172 self.__WritePlan(plan, 'CTS-flaky') 173 174 small_tests = BuildAospSmallSizeTestList() 175 medium_tests = BuildAospMediumSizeTestList() 176 new_test_packages = BuildCtsVettedNewPackagesList() 177 178 # CTS - sub plan for public, small size tests 179 plan = tools.TestPlan(packages) 180 plan.Exclude('.*') 181 for package, test_list in small_tests.iteritems(): 182 plan.Include(package+'$') 183 plan.Exclude(r'com\.android\.cts\.browserbench') 184 for package, test_list in flaky_tests.iteritems(): 185 plan.ExcludeTests(package, test_list) 186 self.__WritePlan(plan, 'CTS-kitkat-small') 187 188 # CTS - sub plan for public, medium size tests 189 plan = tools.TestPlan(packages) 190 plan.Exclude('.*') 191 for package, test_list in medium_tests.iteritems(): 192 plan.Include(package+'$') 193 plan.Exclude(r'com\.android\.cts\.browserbench') 194 for package, test_list in flaky_tests.iteritems(): 195 plan.ExcludeTests(package, test_list) 196 self.__WritePlan(plan, 'CTS-kitkat-medium') 197 198 # CTS - sub plan for hardware tests which is public, large 199 plan = tools.TestPlan(packages) 200 plan.Exclude('.*') 201 plan.Include(r'android\.hardware$') 202 plan.Exclude(r'com\.android\.cts\.browserbench') 203 for package, test_list in flaky_tests.iteritems(): 204 plan.ExcludeTests(package, test_list) 205 self.__WritePlan(plan, 'CTS-hardware') 206 207 # CTS - sub plan for media tests which is public, large 208 plan = tools.TestPlan(packages) 209 plan.Exclude('.*') 210 plan.Include(r'android\.media$') 211 plan.Include(r'android\.view$') 212 plan.Exclude(r'com\.android\.cts\.browserbench') 213 for package, test_list in flaky_tests.iteritems(): 214 plan.ExcludeTests(package, test_list) 215 self.__WritePlan(plan, 'CTS-media') 216 217 # CTS - sub plan for mediastress tests which is public, large 218 plan = tools.TestPlan(packages) 219 plan.Exclude('.*') 220 plan.Include(r'android\.mediastress$') 221 plan.Exclude(r'com\.android\.cts\.browserbench') 222 for package, test_list in flaky_tests.iteritems(): 223 plan.ExcludeTests(package, test_list) 224 self.__WritePlan(plan, 'CTS-mediastress') 225 226 # CTS - sub plan for new tests that is vetted for L launch 227 plan = tools.TestPlan(packages) 228 plan.Exclude('.*') 229 for package, test_list in new_test_packages.iteritems(): 230 plan.Include(package+'$') 231 plan.Exclude(r'com\.android\.cts\.browserbench') 232 for package, test_list in flaky_tests.iteritems(): 233 plan.ExcludeTests(package, test_list) 234 self.__WritePlan(plan, 'CTS-l-tests') 235 236 #CTS - sub plan for new test packages added for staging 237 plan = tools.TestPlan(packages) 238 for package, test_list in small_tests.iteritems(): 239 plan.Exclude(package+'$') 240 for package, test_list in medium_tests.iteritems(): 241 plan.Exclude(package+'$') 242 for package, tests_list in new_test_packages.iteritems(): 243 plan.Exclude(package+'$') 244 plan.Exclude(r'android\.hardware$') 245 plan.Exclude(r'android\.media$') 246 plan.Exclude(r'android\.view$') 247 plan.Exclude(r'android\.mediastress$') 248 plan.Exclude(r'com\.android\.cts\.browserbench') 249 for package, test_list in flaky_tests.iteritems(): 250 plan.ExcludeTests(package, test_list) 251 self.__WritePlan(plan, 'CTS-staging') 252 253 plan = tools.TestPlan(packages) 254 plan.Exclude('.*') 255 plan.Include(r'com\.drawelements\.') 256 self.__WritePlan(plan, 'CTS-DEQP') 257 258 plan = tools.TestPlan(packages) 259 plan.Exclude('.*') 260 plan.Include(r'android\.webgl') 261 self.__WritePlan(plan, 'CTS-webview') 262 263 264 def BuildAospMediumSizeTestList(): 265 """ Construct a defaultdic that lists package names of medium tests 266 already published to aosp. """ 267 return { 268 'android.app' : [], 269 'android.core.tests.libcore.package.libcore' : [], 270 'android.core.tests.libcore.package.org' : [], 271 'android.core.vm-tests-tf' : [], 272 'android.dpi' : [], 273 'android.host.security' : [], 274 'android.net' : [], 275 'android.os' : [], 276 'android.permission2' : [], 277 'android.security' : [], 278 'android.telephony' : [], 279 'android.webkit' : [], 280 'android.widget' : [], 281 'com.android.cts.browserbench' : []} 282 283 def BuildAospSmallSizeTestList(): 284 """ Construct a defaultdict that lists packages names of small tests 285 already published to aosp. """ 286 return { 287 'android.aadb' : [], 288 'android.acceleration' : [], 289 'android.accessibility' : [], 290 'android.accessibilityservice' : [], 291 'android.accounts' : [], 292 'android.admin' : [], 293 'android.animation' : [], 294 'android.bionic' : [], 295 'android.bluetooth' : [], 296 'android.calendarcommon' : [], 297 'android.content' : [], 298 'android.core.tests.libcore.package.com' : [], 299 'android.core.tests.libcore.package.conscrypt' : [], 300 'android.core.tests.libcore.package.dalvik' : [], 301 'android.core.tests.libcore.package.sun' : [], 302 'android.core.tests.libcore.package.tests' : [], 303 'android.database' : [], 304 'android.dreams' : [], 305 'android.drm' : [], 306 'android.effect' : [], 307 'android.gesture' : [], 308 'android.graphics' : [], 309 'android.graphics2' : [], 310 'android.jni' : [], 311 'android.keystore' : [], 312 'android.location' : [], 313 'android.nativemedia.sl' : [], 314 'android.nativemedia.xa' : [], 315 'android.nativeopengl' : [], 316 'android.ndef' : [], 317 'android.opengl' : [], 318 'android.openglperf' : [], 319 'android.permission' : [], 320 'android.preference' : [], 321 'android.preference2' : [], 322 'android.provider' : [], 323 'android.renderscript' : [], 324 'android.rscpp' : [], 325 'android.rsg' : [], 326 'android.sax' : [], 327 'android.signature' : [], 328 'android.speech' : [], 329 'android.tests.appsecurity' : [], 330 'android.text' : [], 331 'android.textureview' : [], 332 'android.theme' : [], 333 'android.usb' : [], 334 'android.util' : [], 335 'com.android.cts.dram' : [], 336 'com.android.cts.filesystemperf' : [], 337 'com.android.cts.jank' : [], 338 'com.android.cts.opengl' : [], 339 'com.android.cts.simplecpu' : [], 340 'com.android.cts.ui' : [], 341 'com.android.cts.uihost' : [], 342 'com.android.cts.videoperf' : [], 343 'zzz.android.monkey' : []} 344 345 def BuildCtsVettedNewPackagesList(): 346 """ Construct a defaultdict that maps package names that is vetted for L. """ 347 return { 348 'android.JobScheduler' : [], 349 'android.core.tests.libcore.package.harmony_annotation' : [], 350 'android.core.tests.libcore.package.harmony_beans' : [], 351 'android.core.tests.libcore.package.harmony_java_io' : [], 352 'android.core.tests.libcore.package.harmony_java_lang' : [], 353 'android.core.tests.libcore.package.harmony_java_math' : [], 354 'android.core.tests.libcore.package.harmony_java_net' : [], 355 'android.core.tests.libcore.package.harmony_java_nio' : [], 356 'android.core.tests.libcore.package.harmony_java_util' : [], 357 'android.core.tests.libcore.package.harmony_java_text' : [], 358 'android.core.tests.libcore.package.harmony_javax_security' : [], 359 'android.core.tests.libcore.package.harmony_logging' : [], 360 'android.core.tests.libcore.package.harmony_prefs' : [], 361 'android.core.tests.libcore.package.harmony_sql' : [], 362 'android.core.tests.libcore.package.jsr166' : [], 363 'android.core.tests.libcore.package.okhttp' : [], 364 'android.display' : [], 365 'android.host.theme' : [], 366 'android.jdwp' : [], 367 'android.location2' : [], 368 'android.print' : [], 369 'android.renderscriptlegacy' : [], 370 'android.signature' : [], 371 'android.tv' : [], 372 'android.uiautomation' : [], 373 'android.uirendering' : [], 374 'android.webgl' : [], 375 'com.drawelements.deqp.gles3' : [], 376 'com.drawelements.deqp.gles31' : []} 377 378 def BuildCtsFlakyTestList(): 379 """ Construct a defaultdict that maps package name to a list of tests 380 that are known to be flaky in the lab or not passing on userdebug builds. """ 381 return { 382 'android.app' : [ 383 'cts.ActivityManagerTest#testIsRunningInTestHarness',], 384 'android.dpi' : [ 385 'cts.DefaultManifestAttributesSdkTest#testPackageHasExpectedSdkVersion',], 386 'android.hardware' : [ 387 'cts.CameraTest#testVideoSnapshot', 388 'cts.CameraGLTest#testCameraToSurfaceTextureMetadata', 389 'cts.CameraGLTest#testSetPreviewTextureBothCallbacks', 390 'cts.CameraGLTest#testSetPreviewTexturePreviewCallback',], 391 'android.media' : [ 392 'cts.DecoderTest#testCodecResetsH264WithSurface', 393 'cts.StreamingMediaPlayerTest#testHLS',], 394 'android.net' : [ 395 'cts.ConnectivityManagerTest#testStartUsingNetworkFeature_enableHipri', 396 'cts.DnsTest#testDnsWorks', 397 'cts.SSLCertificateSocketFactoryTest#testCreateSocket', 398 'cts.SSLCertificateSocketFactoryTest#test_createSocket_bind', 399 'cts.SSLCertificateSocketFactoryTest#test_createSocket_simple', 400 'cts.SSLCertificateSocketFactoryTest#test_createSocket_wrapping', 401 'cts.TrafficStatsTest#testTrafficStatsForLocalhost', 402 'wifi.cts.NsdManagerTest#testAndroidTestCaseSetupProperly',], 403 'android.os' : [ 404 'cts.BuildVersionTest#testReleaseVersion', 405 'cts.BuildTest#testIsSecureUserBuild',], 406 'android.security' : [ 407 'cts.BannedFilesTest#testNoSu', 408 'cts.BannedFilesTest#testNoSuInPath', 409 'cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdp6Ports', 410 'cts.ListeningPortsTest#testNoRemotelyAccessibleListeningUdpPorts', 411 'cts.PackageSignatureTest#testPackageSignatures', 412 'cts.SELinuxDomainTest#testSuDomain', 413 'cts.SELinuxHostTest#testAllEnforcing',], 414 'android.webkit' : [ 415 'cts.WebViewClientTest#testOnUnhandledKeyEvent',], 416 'com.android.cts.filesystemperf' : [ 417 'RandomRWTest#testRandomRead', 418 'RandomRWTest#testRandomUpdate',], 419 '' : []} 420 421 def LogGenerateDescription(name): 422 print 'Generating test description for package %s' % name 423 424 if __name__ == '__main__': 425 builder = CtsBuilder(sys.argv) 426 result = builder.GenerateTestDescriptions() 427 if result != 0: 428 sys.exit(result) 429 builder.GenerateTestPlans() 430 431