1 #!/usr/bin/python 2 3 # Script to compare testsuite failures against a list of known-to-fail 4 # tests. 5 6 # Contributed by Diego Novillo <dnovillo (at] google.com> 7 # Overhaul by Krystian Baclawski <kbaclawski (at] google.com> 8 # 9 # Copyright (C) 2011 Free Software Foundation, Inc. 10 # 11 # This file is part of GCC. 12 # 13 # GCC is free software; you can redistribute it and/or modify 14 # it under the terms of the GNU General Public License as published by 15 # the Free Software Foundation; either version 3, or (at your option) 16 # any later version. 17 # 18 # GCC is distributed in the hope that it will be useful, 19 # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 # GNU General Public License for more details. 22 # 23 # You should have received a copy of the GNU General Public License 24 # along with GCC; see the file COPYING. If not, write to 25 # the Free Software Foundation, 51 Franklin Street, Fifth Floor, 26 # Boston, MA 02110-1301, USA. 27 """This script provides a coarser XFAILing mechanism that requires no 28 detailed DejaGNU markings. This is useful in a variety of scenarios: 29 30 - Development branches with many known failures waiting to be fixed. 31 - Release branches with known failures that are not considered 32 important for the particular release criteria used in that branch. 33 34 The script must be executed from the toplevel build directory. When 35 executed it will: 36 37 1) Determine the target built: TARGET 38 2) Determine the source directory: SRCDIR 39 3) Look for a failure manifest file in 40 <SRCDIR>/contrib/testsuite-management/<TARGET>.xfail 41 4) Collect all the <tool>.sum files from the build tree. 42 5) Produce a report stating: 43 a) Failures expected in the manifest but not present in the build. 44 b) Failures in the build not expected in the manifest. 45 6) If all the build failures are expected in the manifest, it exits 46 with exit code 0. Otherwise, it exits with error code 1. 47 """ 48 49 import optparse 50 import logging 51 import os 52 import sys 53 54 sys.path.append(os.path.dirname(os.path.abspath(__file__))) 55 56 from dejagnu.manifest import Manifest 57 from dejagnu.summary import DejaGnuTestResult 58 from dejagnu.summary import DejaGnuTestRun 59 60 # Pattern for naming manifest files. The first argument should be 61 # the toplevel GCC source directory. The second argument is the 62 # target triple used during the build. 63 _MANIFEST_PATH_PATTERN = '%s/contrib/testsuite-management/%s.xfail' 64 65 66 def GetMakefileVars(makefile_path): 67 assert os.path.exists(makefile_path) 68 69 with open(makefile_path) as lines: 70 kvs = [line.split('=', 1) for line in lines if '=' in line] 71 72 return dict((k.strip(), v.strip()) for k, v in kvs) 73 74 75 def GetSumFiles(build_dir): 76 summaries = [] 77 78 for root, _, filenames in os.walk(build_dir): 79 summaries.extend([os.path.join(root, filename) 80 for filename in filenames if filename.endswith('.sum')]) 81 82 return map(os.path.normpath, summaries) 83 84 85 def ValidBuildDirectory(build_dir, target): 86 mandatory_paths = [build_dir, os.path.join(build_dir, 'Makefile')] 87 88 extra_paths = [os.path.join(build_dir, target), 89 os.path.join(build_dir, 'build-%s' % target)] 90 91 return (all(map(os.path.exists, mandatory_paths)) and 92 any(map(os.path.exists, extra_paths))) 93 94 95 def GetManifestPath(build_dir): 96 makefile = GetMakefileVars(os.path.join(build_dir, 'Makefile')) 97 srcdir = makefile['srcdir'] 98 target = makefile['target'] 99 100 if not ValidBuildDirectory(build_dir, target): 101 target = makefile['target_alias'] 102 103 if not ValidBuildDirectory(build_dir, target): 104 logging.error('%s is not a valid GCC top level build directory.', build_dir) 105 sys.exit(1) 106 107 logging.info('Discovered source directory: "%s"', srcdir) 108 logging.info('Discovered build target: "%s"', target) 109 110 return _MANIFEST_PATH_PATTERN % (srcdir, target) 111 112 113 def CompareResults(manifest, actual): 114 """Compare sets of results and return two lists: 115 - List of results present in MANIFEST but missing from ACTUAL. 116 - List of results present in ACTUAL but missing from MANIFEST. 117 """ 118 # Report all the actual results not present in the manifest. 119 actual_vs_manifest = actual - manifest 120 121 # Filter out tests marked flaky. 122 manifest_without_flaky_tests = set(filter(lambda result: not result.flaky, 123 manifest)) 124 125 # Simlarly for all the tests in the manifest. 126 manifest_vs_actual = manifest_without_flaky_tests - actual 127 128 return actual_vs_manifest, manifest_vs_actual 129 130 131 def LogResults(level, results): 132 log_fun = getattr(logging, level) 133 134 for num, result in enumerate(sorted(results), start=1): 135 log_fun(' %d) %s', num, result) 136 137 138 def CheckExpectedResults(manifest_path, build_dir): 139 logging.info('Reading manifest file: "%s"', manifest_path) 140 141 manifest = set(Manifest.FromFile(manifest_path)) 142 143 logging.info('Getting actual results from build directory: "%s"', 144 os.path.realpath(build_dir)) 145 146 summaries = GetSumFiles(build_dir) 147 148 actual = set() 149 150 for summary in summaries: 151 test_run = DejaGnuTestRun.FromFile(summary) 152 failures = set(Manifest.FromDejaGnuTestRun(test_run)) 153 actual.update(failures) 154 155 if manifest: 156 logging.debug('Tests expected to fail:') 157 LogResults('debug', manifest) 158 159 if actual: 160 logging.debug('Actual test failures:') 161 LogResults('debug', actual) 162 163 actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual) 164 165 if actual_vs_manifest: 166 logging.info('Build results not in the manifest:') 167 LogResults('info', actual_vs_manifest) 168 169 if manifest_vs_actual: 170 logging.info('Manifest results not present in the build:') 171 LogResults('info', manifest_vs_actual) 172 logging.info('NOTE: This is not a failure! ', 173 'It just means that the manifest expected these tests to ' 174 'fail, but they worked in this configuration.') 175 176 if actual_vs_manifest or manifest_vs_actual: 177 sys.exit(1) 178 179 logging.info('No unexpected failures.') 180 181 182 def ProduceManifest(manifest_path, build_dir, overwrite): 183 if os.path.exists(manifest_path) and not overwrite: 184 logging.error('Manifest file "%s" already exists.', manifest_path) 185 logging.error('Use --force to overwrite.') 186 sys.exit(1) 187 188 testruns = map(DejaGnuTestRun.FromFile, GetSumFiles(build_dir)) 189 manifests = map(Manifest.FromDejaGnuTestRun, testruns) 190 191 with open(manifest_path, 'w') as manifest_file: 192 manifest_strings = [manifest.Generate() for manifest in manifests] 193 logging.info('Writing manifest to "%s".', manifest_path) 194 manifest_file.write('\n'.join(manifest_strings)) 195 196 197 def Main(argv): 198 parser = optparse.OptionParser(usage=__doc__) 199 parser.add_option( 200 '-b', 201 '--build_dir', 202 dest='build_dir', 203 action='store', 204 metavar='PATH', 205 default=os.getcwd(), 206 help='Build directory to check. (default: current directory)') 207 parser.add_option('-m', 208 '--manifest', 209 dest='manifest', 210 action='store_true', 211 help='Produce the manifest for the current build.') 212 parser.add_option( 213 '-f', 214 '--force', 215 dest='force', 216 action='store_true', 217 help=('Overwrite an existing manifest file, if user requested creating ' 218 'new one. (default: False)')) 219 parser.add_option('-v', 220 '--verbose', 221 dest='verbose', 222 action='store_true', 223 help='Increase verbosity.') 224 options, _ = parser.parse_args(argv[1:]) 225 226 if options.verbose: 227 logging.root.setLevel(logging.DEBUG) 228 229 manifest_path = GetManifestPath(options.build_dir) 230 231 if options.manifest: 232 ProduceManifest(manifest_path, options.build_dir, options.force) 233 else: 234 CheckExpectedResults(manifest_path, options.build_dir) 235 236 237 if __name__ == '__main__': 238 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) 239 Main(sys.argv) 240
