1 #!/usr/bin/env python 2 3 """ 4 This script generates autotest control files for dEQP. It supports 5 1) Generate control files for tests with Passing expectations. 6 2) Generate control files to run tests that are not passing. 7 3) Decomposing a test into shards. Ideally shard_count is chosen such that 8 each shard will run less than 1 minute. It mostly makes sense in 9 combination with "hasty". 10 """ 11 12 from collections import namedtuple 13 # Use 'sudo pip install enum34' to install. 14 from enum import Enum 15 16 Test = namedtuple('Test', 'filter, suite, shards, time, hasty, notpass') 17 18 ATTRIBUTES_BVT_CQ = ( 19 'suite:deqp, suite:graphics_per-day, suite:graphics_system, suite:bvt-cq') 20 ATTRIBUTES_BVT_PB = ( 21 'suite:deqp, suite:graphics_per-day, suite:graphics_system, ' 22 'suite:bvt-perbuild' 23 ) 24 ATTRIBUTES_DAILY = 'suite:deqp, suite:graphics_per-day, suite:graphics_system' 25 SUITE_BVT_CQ = 'deqp, graphics_per-day, graphics_system, bvt-cq' 26 SUITE_BVT_PB = 'deqp, graphics_per-day, graphics_system, bvt-perbuild' 27 SUITE_DAILY = 'deqp, graphics_per-day, graphics_system' 28 29 class Suite(Enum): 30 none = 1 31 daily = 2 32 bvtcq = 3 33 bvtpb = 4 34 35 tests = [ 36 Test('dEQP-EGL.functional', Suite.none, shards=1, hasty=False, notpass=True, time='LENGTHY'), 37 Test('dEQP-EGL.info', Suite.none, shards=1, hasty=False, notpass=True, time='SHORT'), 38 Test('dEQP-EGL.performance', Suite.none, shards=1, hasty=False, notpass=True, time='SHORT'), 39 Test('dEQP-EGL.stress', Suite.none, shards=1, hasty=False, notpass=True, time='LONG'), 40 Test('dEQP-GLES2.accuracy', Suite.bvtpb, shards=1, hasty=False, notpass=True, time='FAST'), 41 Test('dEQP-GLES2.capability', Suite.bvtpb, shards=1, hasty=False, notpass=True, time='FAST'), 42 Test('dEQP-GLES2.functional', Suite.daily, shards=1, hasty=False, notpass=True, time='LENGTHY'), 43 Test('dEQP-GLES2.functional', Suite.daily, shards=1, hasty=True, notpass=False, time='LONG'), 44 Test('dEQP-GLES2.functional', Suite.bvtpb, shards=10, hasty=True, notpass=False, time='FAST'), 45 Test('dEQP-GLES2.info', Suite.bvtpb, shards=1, hasty=False, notpass=True, time='FAST'), 46 Test('dEQP-GLES2.performance', Suite.daily, shards=1, hasty=False, notpass=True, time='LONG'), 47 Test('dEQP-GLES2.stress', Suite.daily, shards=1, hasty=False, notpass=True, time='LONG'), 48 Test('dEQP-GLES3.accuracy', Suite.bvtpb, shards=1, hasty=False, notpass=True, time='FAST'), 49 Test('dEQP-GLES3.functional', Suite.daily, shards=1, hasty=False, notpass=True, time='LENGTHY'), 50 Test('dEQP-GLES3.functional', Suite.daily, shards=1, hasty=True, notpass=False, time='LONG'), 51 Test('dEQP-GLES3.functional', Suite.daily, shards=10, hasty=True, notpass=False, time='FAST'), 52 Test('dEQP-GLES3.info', Suite.bvtpb, shards=1, hasty=False, notpass=True, time='FAST'), 53 Test('dEQP-GLES3.performance', Suite.daily, shards=1, hasty=False, notpass=True, time='LONG'), 54 Test('dEQP-GLES3.stress', Suite.daily, shards=1, hasty=False, notpass=True, time='LONG'), 55 Test('dEQP-GLES31.functional', Suite.none, shards=1, hasty=False, notpass=True, time='LENGTHY'), 56 Test('dEQP-GLES31.info', Suite.none, shards=1, hasty=False, notpass=True, time='FAST'), 57 Test('dEQP-GLES31.stress', Suite.none, shards=1, hasty=False, notpass=True, time='LONG'), 58 ] 59 60 CONTROLFILE_TEMPLATE = ( 61 """\ 62 # Copyright 2015 The Chromium OS Authors. All rights reserved. 63 # Use of this source code is governed by a BSD-style license that can be 64 # found in the LICENSE file. 65 66 # Please do not edit this file! It has been created by generate_controlfiles.py. 67 68 NAME = '{0}' 69 AUTHOR = 'chromeos-gfx' 70 PURPOSE = 'Run the drawElements Quality Program test suite.' 71 CRITERIA = 'All of the individual tests must pass.' 72 ATTRIBUTES = '{1}' 73 SUITE = '{2}' 74 TIME = '{3}' 75 {4}TEST_CATEGORY = 'Functional' 76 TEST_CLASS = 'graphics' 77 TEST_TYPE = 'client' 78 DOC = \"\"\" 79 This test runs the drawElements Quality Program test suite. 80 \"\"\" 81 82 job.run_test('graphics_dEQP', opts = args + ['filter={5}', 83 'subset_to_run={6}', 84 'hasty={7}', 85 'shard_number={8}', 86 'shard_count={9}'])""") 87 88 #Unlike the normal version it batches many tests in a single run 89 #to reduce testing time. Unfortunately this is less robust and 90 #can lead to spurious failures. 91 92 93 def get_controlfilename(test, shard=0): 94 return 'control.%s' % get_name(test, shard) 95 96 def get_dependencies(test): 97 if test.notpass: 98 return "DEPENDENCIES = 'cleanup-reboot'\n" 99 return '' 100 101 def get_suite(test): 102 if test.suite == Suite.bvtcq: 103 return SUITE_BVT_CQ 104 if test.suite == Suite.bvtpb: 105 return SUITE_BVT_PB 106 if test.suite == Suite.daily: 107 return SUITE_DAILY 108 return '' 109 110 def get_attributes(test): 111 if test.suite == Suite.bvtcq: 112 return ATTRIBUTES_BVT_CQ 113 if test.suite == Suite.bvtpb: 114 return ATTRIBUTES_BVT_PB 115 if test.suite == Suite.daily: 116 return ATTRIBUTES_DAILY 117 return '' 118 119 def get_time(test): 120 return test.time 121 122 def get_name(test, shard): 123 name = test.filter.replace('dEQP-', '', 1).lower() 124 if test.hasty: 125 name = '%s.hasty' % name 126 if test.shards > 1: 127 name = '%s.%d' % (name, shard) 128 if test.notpass: 129 name = name + '.NotPass' 130 return name 131 132 def get_testname(test, shard=0): 133 return 'graphics_dEQP.%s' % get_name(test, shard) 134 135 def write_controlfile(filename, content): 136 print 'Writing %s.' % filename 137 with open(filename, 'w+') as f: 138 f.write(content) 139 140 def write_controlfiles(test): 141 attributes = get_attributes(test) 142 suite = get_suite(test) 143 time = get_time(test) 144 dependencies = get_dependencies(test) 145 if test.shards > 1: 146 for shard in xrange(0, test.shards): 147 subset = 'Pass' 148 testname = get_testname(test, shard) 149 filename = get_controlfilename(test, shard) 150 content = CONTROLFILE_TEMPLATE.format( 151 testname, attributes, suite, time, dependencies, test.filter, 152 subset, test.hasty, shard, test.shards) 153 write_controlfile(filename, content) 154 else: 155 if test.notpass: 156 subset = 'NotPass' 157 testname = get_testname(test) 158 filename = get_controlfilename(test) 159 content = CONTROLFILE_TEMPLATE.format( 160 testname, attributes, suite, time, dependencies, test.filter, 161 subset, test.hasty, 0, test.shards) 162 write_controlfile(filename, content) 163 test = Test(test.filter, 164 test.suite, 165 test.shards, 166 test.time, 167 test.hasty, 168 notpass=False) 169 dependencies = get_dependencies(test) 170 subset = 'Pass' 171 testname = get_testname(test) 172 filename = get_controlfilename(test) 173 content = CONTROLFILE_TEMPLATE.format(testname, attributes, suite, time, 174 dependencies, test.filter, subset, 175 test.hasty, 0, test.shards) 176 write_controlfile(filename, content) 177 178 179 for test in tests: 180 write_controlfiles(test) 181
