Home | History | Annotate | Download | only in scripts 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Test harness for chromium clang tools."""
      7 
      8 import difflib
      9 import glob
     10 import json
     11 import os
     12 import os.path
     13 import subprocess
     14 import shutil
     15 import sys
     16 
     17 
     18 def _GenerateCompileCommands(files, include_paths):
     19   """Returns a JSON string containing a compilation database for the input."""
     20   include_path_flags = ' '.join('-I %s' % include_path
     21                                for include_path in include_paths)
     22   return json.dumps([{'directory': '.',
     23                       'command': 'clang++ -fsyntax-only %s -c %s' % (
     24                           include_path_flags, f),
     25                       'file': f} for f in files], indent=2)
     26 
     27 
     28 def _NumberOfTestsToString(tests):
     29   """Returns an English describing the number of tests."""
     30   return "%d test%s" % (tests, 's' if tests != 1 else '')
     31 
     32 
     33 def main(argv):
     34   if len(argv) < 1:
     35     print 'Usage: test_tool.py <clang tool>'
     36     print '  <clang tool> is the clang tool to be tested.'
     37     sys.exit(1)
     38 
     39   tool_to_test = argv[0]
     40   tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__))
     41   tools_clang_directory = os.path.dirname(tools_clang_scripts_directory)
     42   test_directory_for_tool = os.path.join(
     43       tools_clang_directory, tool_to_test, 'tests')
     44   compile_database = os.path.join(test_directory_for_tool,
     45                                   'compile_commands.json')
     46   source_files = glob.glob(os.path.join(test_directory_for_tool,
     47                                         '*-original.cc'))
     48   actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc'])
     49                   for source_file in source_files]
     50   expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.cc'])
     51                     for source_file in source_files]
     52   include_paths = []
     53   include_paths.append(
     54       os.path.realpath(os.path.join(tools_clang_directory, '../..')))
     55   # Many gtest headers expect to have testing/gtest/include in the include
     56   # search path.
     57   include_paths.append(
     58       os.path.realpath(os.path.join(tools_clang_directory,
     59                                     '../..',
     60                                     'testing/gtest/include')))
     61 
     62   try:
     63     # Set up the test environment.
     64     for source, actual in zip(source_files, actual_files):
     65       shutil.copyfile(source, actual)
     66     # Stage the test files in the git index. If they aren't staged, then
     67     # run_tools.py will skip them when applying replacements.
     68     args = ['git', 'add']
     69     args.extend(actual_files)
     70     subprocess.check_call(args)
     71     # Generate a temporary compilation database to run the tool over.
     72     with open(compile_database, 'w') as f:
     73       f.write(_GenerateCompileCommands(actual_files, include_paths))
     74 
     75     args = ['python',
     76             os.path.join(tools_clang_scripts_directory, 'run_tool.py'),
     77             tool_to_test,
     78             test_directory_for_tool]
     79     args.extend(actual_files)
     80     run_tool = subprocess.Popen(args, stdout=subprocess.PIPE)
     81     stdout, _ = run_tool.communicate()
     82     if run_tool.returncode != 0:
     83       print 'run_tool failed:\n%s' % stdout
     84       sys.exit(1)
     85 
     86     passed = 0
     87     failed = 0
     88     for expected, actual in zip(expected_files, actual_files):
     89       print '[ RUN      ] %s' % os.path.relpath(actual)
     90       expected_output = actual_output = None
     91       with open(expected, 'r') as f:
     92         expected_output = f.readlines()
     93       with open(actual, 'r') as f:
     94         actual_output = f.readlines()
     95       if actual_output != expected_output:
     96         failed += 1
     97         for line in difflib.unified_diff(expected_output, actual_output,
     98                                          fromfile=os.path.relpath(expected),
     99                                          tofile=os.path.relpath(actual)):
    100           sys.stdout.write(line)
    101         print '[  FAILED  ] %s' % os.path.relpath(actual)
    102         # Don't clean up the file on failure, so the results can be referenced
    103         # more easily.
    104         continue
    105       print '[       OK ] %s' % os.path.relpath(actual)
    106       passed += 1
    107       os.remove(actual)
    108 
    109     if failed == 0:
    110       os.remove(compile_database)
    111 
    112     print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files))
    113     if passed > 0:
    114       print '[  PASSED  ] %s.' % _NumberOfTestsToString(passed)
    115     if failed > 0:
    116       print '[  FAILED  ] %s.' % _NumberOfTestsToString(failed)
    117   finally:
    118     # No matter what, unstage the git changes we made earlier to avoid polluting
    119     # the index.
    120     args = ['git', 'reset', '--quiet', 'HEAD']
    121     args.extend(actual_files)
    122     subprocess.call(args)
    123 
    124 
    125 if __name__ == '__main__':
    126   sys.exit(main(sys.argv[1:]))
    127