1 #!/usr/bin/env python 2 # Copyright 2017 The PDFium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Generates a coverage report for given binaries using llvm-gcov & lcov. 7 8 Requires llvm-cov 3.5 or later. 9 Requires lcov 1.11 or later. 10 Requires that 'use_coverage = true' is set in args.gn. 11 """ 12 13 import argparse 14 from collections import namedtuple 15 import os 16 import pprint 17 import re 18 import subprocess 19 import sys 20 21 22 # Add src dir to path to avoid having to set PYTHONPATH. 23 sys.path.append( 24 os.path.abspath( 25 os.path.join( 26 os.path.dirname(__file__), 27 os.path.pardir, 28 os.path.pardir, 29 os.path.pardir))) 30 31 from testing.tools.common import GetBooleanGnArg 32 33 34 # 'binary' is the file that is to be run for the test. 35 # 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus 36 # requires special handling. 37 TestSpec = namedtuple('TestSpec', 'binary, use_test_runner') 38 39 # All of the coverage tests that the script knows how to run. 40 COVERAGE_TESTS = { 41 'pdfium_unittests': TestSpec('pdfium_unittests', False), 42 'pdfium_embeddertests': TestSpec('pdfium_embeddertests', False), 43 'corpus_tests': TestSpec('run_corpus_tests.py', True), 44 'javascript_tests': TestSpec('run_javascript_tests.py', True), 45 'pixel_tests': TestSpec('run_pixel_tests.py', True), 46 } 47 48 # Coverage tests that are known to take a long time to run, so are not in the 49 # default set. The user must either explicitly invoke these tests or pass in 50 # --slow. 51 SLOW_TESTS = ['corpus_tests', 'javascript_tests', 'pixel_tests'] 52 53 class CoverageExecutor(object): 54 55 def __init__(self, parser, args): 56 """Initialize executor based on the current script environment 57 58 Args: 59 parser: argparse.ArgumentParser for handling improper inputs. 60 args: Dictionary of arguments passed into the calling script. 61 """ 62 self.dry_run = args['dry_run'] 63 self.verbose = args['verbose'] 64 65 llvm_cov = self.determine_proper_llvm_cov() 66 if not llvm_cov: 67 print 'Unable to find appropriate llvm-cov to use' 68 sys.exit(1) 69 self.lcov_env = os.environ 70 self.lcov_env['LLVM_COV_BIN'] = llvm_cov 71 72 self.lcov = self.determine_proper_lcov() 73 if not self.lcov: 74 print 'Unable to find appropriate lcov to use' 75 sys.exit(1) 76 77 self.coverage_files = set() 78 self.source_directory = args['source_directory'] 79 if not os.path.isdir(self.source_directory): 80 parser.error("'%s' needs to be a directory" % self.source_directory) 81 82 self.build_directory = args['build_directory'] 83 if not os.path.isdir(self.build_directory): 84 parser.error("'%s' needs to be a directory" % self.build_directory) 85 86 self.coverage_tests = self.calculate_coverage_tests(args) 87 if not self.coverage_tests: 88 parser.error( 89 'No valid tests in set to be run. This is likely due to bad command ' 90 'line arguments') 91 92 if not GetBooleanGnArg('use_coverage', self.build_directory, self.verbose): 93 parser.error( 94 'use_coverage does not appear to be set to true for build, but is ' 95 'needed') 96 97 self.use_goma = GetBooleanGnArg('use_goma', self.build_directory, 98 self.verbose) 99 100 self.output_directory = args['output_directory'] 101 if not os.path.exists(self.output_directory): 102 if not self.dry_run: 103 os.makedirs(self.output_directory) 104 elif not os.path.isdir(self.output_directory): 105 parser.error('%s exists, but is not a directory' % self.output_directory) 106 self.coverage_totals_path = os.path.join(self.output_directory, 107 'pdfium_totals.info') 108 109 def check_output(self, args, dry_run=False, env=None): 110 """Dry run aware wrapper of subprocess.check_output()""" 111 if dry_run: 112 print "Would have run '%s'" % ' '.join(args) 113 return '' 114 115 output = subprocess.check_output(args, env=env) 116 117 if self.verbose: 118 print "check_output(%s) returned '%s'" % (args, output) 119 return output 120 121 def call(self, args, dry_run=False, env=None): 122 """Dry run aware wrapper of subprocess.call()""" 123 if dry_run: 124 print "Would have run '%s'" % ' '.join(args) 125 return 0 126 127 output = subprocess.call(args, env=env) 128 129 if self.verbose: 130 print 'call(%s) returned %s' % (args, output) 131 return output 132 133 def call_lcov(self, args, dry_run=False, needs_directory=True): 134 """Wrapper to call lcov that adds appropriate arguments as needed.""" 135 lcov_args = [ 136 self.lcov, '--config-file', 137 os.path.join(self.source_directory, 'testing', 'tools', 'coverage', 138 'lcovrc'), 139 '--gcov-tool', 140 os.path.join(self.source_directory, 'testing', 'tools', 'coverage', 141 'llvm-gcov') 142 ] 143 if needs_directory: 144 lcov_args.extend(['--directory', self.source_directory]) 145 if not self.verbose: 146 lcov_args.append('--quiet') 147 lcov_args.extend(args) 148 return self.call(lcov_args, dry_run=dry_run, env=self.lcov_env) 149 150 def calculate_coverage_tests(self, args): 151 """Determine which tests should be run.""" 152 testing_tools_directory = os.path.join(self.source_directory, 'testing', 153 'tools') 154 coverage_tests = {} 155 for name in COVERAGE_TESTS.keys(): 156 test_spec = COVERAGE_TESTS[name] 157 if test_spec.use_test_runner: 158 binary_path = os.path.join(testing_tools_directory, test_spec.binary) 159 else: 160 binary_path = os.path.join(self.build_directory, test_spec.binary) 161 coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner) 162 163 if args['tests']: 164 return {name: spec 165 for name, spec in coverage_tests.iteritems() if name in args['tests']} 166 elif not args['slow']: 167 return {name: spec 168 for name, spec in coverage_tests.iteritems() if name not in SLOW_TESTS} 169 else: 170 return coverage_tests 171 172 def find_acceptable_binary(self, binary_name, version_regex, 173 min_major_version, min_minor_version): 174 """Find the newest version of binary that meets the min version.""" 175 min_version = (min_major_version, min_minor_version) 176 parsed_versions = {} 177 # When calling Bash builtins like this the command and arguments must be 178 # passed in as a single string instead of as separate list members. 179 potential_binaries = self.check_output( 180 ['bash', '-c', 'compgen -abck %s' % binary_name]).splitlines() 181 for binary in potential_binaries: 182 if self.verbose: 183 print 'Testing llvm-cov binary, %s' % binary 184 # Assuming that scripts that don't respond to --version correctly are not 185 # valid binaries and just happened to get globbed in. This is true for 186 # lcov and llvm-cov 187 try: 188 version_output = self.check_output([binary, '--version']).splitlines() 189 except subprocess.CalledProcessError: 190 if self.verbose: 191 print '--version returned failure status 1, so ignoring' 192 continue 193 194 for line in version_output: 195 matcher = re.match(version_regex, line) 196 if matcher: 197 parsed_version = (int(matcher.group(1)), int(matcher.group(2))) 198 if parsed_version >= min_version: 199 parsed_versions[parsed_version] = binary 200 break 201 202 if not parsed_versions: 203 return None 204 return parsed_versions[max(parsed_versions)] 205 206 def determine_proper_llvm_cov(self): 207 """Find a version of llvm_cov that will work with the script.""" 208 version_regex = re.compile('.*LLVM version ([\d]+)\.([\d]+).*') 209 return self.find_acceptable_binary('llvm-cov', version_regex, 3, 5) 210 211 def determine_proper_lcov(self): 212 """Find a version of lcov that will work with the script.""" 213 version_regex = re.compile('.*LCOV version ([\d]+)\.([\d]+).*') 214 return self.find_acceptable_binary('lcov', version_regex, 1, 11) 215 216 def build_binaries(self): 217 """Build all the binaries that are going to be needed for coverage 218 generation.""" 219 call_args = ['ninja'] 220 if self.use_goma: 221 call_args.extend(['-j', '250']) 222 call_args.extend(['-C', self.build_directory]) 223 return self.call(call_args, dry_run=self.dry_run) == 0 224 225 def generate_coverage(self, name, spec): 226 """Generate the coverage data for a test 227 228 Args: 229 name: Name associated with the test to be run. This is used as a label 230 in the coverage data, so should be unique across all of the tests 231 being run. 232 spec: Tuple containing the path to the binary to run, and if this test 233 uses test_runner.py. 234 """ 235 if self.verbose: 236 print "Generating coverage for test '%s', using data '%s'" % (name, spec) 237 if not os.path.exists(spec.binary): 238 print('Unable to generate coverage for %s, since it appears to not exist' 239 ' @ %s') % (name, spec.binary) 240 return False 241 242 if self.call_lcov(['--zerocounters'], dry_run=self.dry_run): 243 print 'Unable to clear counters for %s' % name 244 return False 245 246 binary_args = [spec.binary] 247 if spec.use_test_runner: 248 # Test runner performs multi-threading in the wrapper script, not the test 249 # binary, so need -j 1, otherwise multiple processes will be writing to 250 # the code coverage files, invalidating results. 251 # TODO(pdfium:811): Rewrite how test runner tests work, so that they can 252 # be run in multi-threaded mode. 253 binary_args.extend(['-j', '1', '--build-dir', self.build_directory]) 254 if self.call(binary_args, dry_run=self.dry_run) and self.verbose: 255 print('Running %s appears to have failed, which might affect ' 256 'results') % spec.binary 257 258 output_raw_path = os.path.join(self.output_directory, '%s_raw.info' % name) 259 if self.call_lcov( 260 ['--capture', '--test-name', name, '--output-file', output_raw_path], 261 dry_run=self.dry_run): 262 print 'Unable to capture coverage data for %s' % name 263 return False 264 265 output_filtered_path = os.path.join(self.output_directory, 266 '%s_filtered.info' % name) 267 output_filters = [ 268 '/usr/include/*', '*third_party*', '*testing*', '*_unittest.cpp', 269 '*_embeddertest.cpp' 270 ] 271 if self.call_lcov( 272 ['--remove', output_raw_path] + output_filters + 273 ['--output-file', output_filtered_path], 274 dry_run=self.dry_run, 275 needs_directory=False): 276 print 'Unable to filter coverage data for %s' % name 277 return False 278 279 self.coverage_files.add(output_filtered_path) 280 return True 281 282 def merge_coverage(self): 283 """Merge all of the coverage data sets into one for report generation.""" 284 merge_args = [] 285 for coverage_file in self.coverage_files: 286 merge_args.extend(['--add-tracefile', coverage_file]) 287 288 merge_args.extend(['--output-file', self.coverage_totals_path]) 289 return self.call_lcov( 290 merge_args, dry_run=self.dry_run, needs_directory=False) == 0 291 292 def generate_report(self): 293 """Produce HTML coverage report based on combined coverage data set.""" 294 config_file = os.path.join( 295 self.source_directory, 'testing', 'tools', 'coverage', 'lcovrc') 296 297 lcov_args = ['genhtml', 298 '--config-file', config_file, 299 '--legend', 300 '--demangle-cpp', 301 '--show-details', 302 '--prefix', self.source_directory, 303 '--ignore-errors', 304 'source', self.coverage_totals_path, 305 '--output-directory', self.output_directory] 306 return self.call(lcov_args, dry_run=self.dry_run) == 0 307 308 def run(self): 309 """Setup environment, execute the tests and generate coverage report""" 310 if not self.build_binaries(): 311 print 'Failed to successfully build binaries' 312 return False 313 314 for name in self.coverage_tests.keys(): 315 if not self.generate_coverage(name, self.coverage_tests[name]): 316 print 'Failed to successfully generate coverage data' 317 return False 318 319 if not self.merge_coverage(): 320 print 'Failed to successfully merge generated coverage data' 321 return False 322 323 if not self.generate_report(): 324 print 'Failed to successfully generated coverage report' 325 return False 326 327 return True 328 329 330 def main(): 331 parser = argparse.ArgumentParser() 332 parser.formatter_class = argparse.RawDescriptionHelpFormatter 333 parser.description = ('Generates a coverage report for given binaries using ' 334 'llvm-cov & lcov.\n\n' 335 'Requires llvm-cov 3.5 or later.\n' 336 'Requires lcov 1.11 or later.\n\n' 337 'By default runs pdfium_unittests and ' 338 'pdfium_embeddertests. If --slow is passed in then all ' 339 'tests will be run. If any of the tests are specified ' 340 'on the command line, then only those will be run.') 341 parser.add_argument( 342 '-s', 343 '--source_directory', 344 help='Location of PDFium source directory, defaults to CWD', 345 default=os.getcwd()) 346 build_default = os.path.join('out', 'Coverage') 347 parser.add_argument( 348 '-b', 349 '--build_directory', 350 help= 351 'Location of PDFium build directory with coverage enabled, defaults to ' 352 '%s under CWD' % build_default, 353 default=os.path.join(os.getcwd(), build_default)) 354 output_default = 'coverage_report' 355 parser.add_argument( 356 '-o', 357 '--output_directory', 358 help='Location to write out coverage report to, defaults to %s under CWD ' 359 % output_default, 360 default=os.path.join(os.getcwd(), output_default)) 361 parser.add_argument( 362 '-n', 363 '--dry-run', 364 help='Output commands instead of executing them', 365 action='store_true') 366 parser.add_argument( 367 '-v', 368 '--verbose', 369 help='Output additional diagnostic information', 370 action='store_true') 371 parser.add_argument( 372 '--slow', 373 help='Run all tests, even those known to take a long time. Ignored if ' 374 'specific tests are passed in.', 375 action='store_true') 376 parser.add_argument( 377 'tests', 378 help='Tests to be run, defaults to all. Valid entries are %s' % 379 COVERAGE_TESTS.keys(), 380 nargs='*') 381 382 args = vars(parser.parse_args()) 383 if args['verbose']: 384 pprint.pprint(args) 385 386 executor = CoverageExecutor(parser, args) 387 if executor.run(): 388 return 0 389 return 1 390 391 392 if __name__ == '__main__': 393 sys.exit(main()) 394
