1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Test harness for chromium clang tools.""" 7 8 import difflib 9 import glob 10 import json 11 import os 12 import os.path 13 import shutil 14 import subprocess 15 import sys 16 17 18 def _RunGit(args): 19 if sys.platform == 'win32': 20 args = ['git.bat'] + args 21 else: 22 args = ['git'] + args 23 subprocess.check_call(args) 24 25 26 def _GenerateCompileCommands(files, include_paths): 27 """Returns a JSON string containing a compilation database for the input.""" 28 # Note: in theory, backslashes in the compile DB should work but the tools 29 # that write compile DBs and the tools that read them don't agree on the 30 # escaping convention: https://llvm.org/bugs/show_bug.cgi?id=19687 31 files = [f.replace('\\', '/') for f in files] 32 include_path_flags = ' '.join('-I %s' % include_path.replace('\\', '/') 33 for include_path in include_paths) 34 return json.dumps([{'directory': '.', 35 'command': 'clang++ -std=c++11 -fsyntax-only %s -c %s' % ( 36 include_path_flags, f), 37 'file': f} for f in files], indent=2) 38 39 40 def _NumberOfTestsToString(tests): 41 """Returns an English describing the number of tests.""" 42 return '%d test%s' % (tests, 's' if tests != 1 else '') 43 44 45 def _RunToolAndApplyEdits(tools_clang_scripts_directory, 46 tool_to_test, 47 test_directory_for_tool, 48 actual_files): 49 try: 50 # Stage the test files in the git index. If they aren't staged, then 51 # run_tool.py will skip them when applying replacements. 52 args = ['add'] 53 args.extend(actual_files) 54 _RunGit(args) 55 56 # Launch the following pipeline: 57 # run_tool.py ... | extract_edits.py | apply_edits.py ... 58 args = ['python', 59 os.path.join(tools_clang_scripts_directory, 'run_tool.py'), 60 tool_to_test, 61 test_directory_for_tool] 62 args.extend(actual_files) 63 run_tool = subprocess.Popen(args, stdout=subprocess.PIPE) 64 65 args = ['python', 66 os.path.join(tools_clang_scripts_directory, 'extract_edits.py')] 67 extract_edits = subprocess.Popen(args, stdin=run_tool.stdout, 68 stdout=subprocess.PIPE) 69 70 args = ['python', 71 os.path.join(tools_clang_scripts_directory, 'apply_edits.py'), 72 test_directory_for_tool] 73 apply_edits = subprocess.Popen(args, stdin=extract_edits.stdout, 74 stdout=subprocess.PIPE) 75 76 # Wait for the pipeline to finish running + check exit codes. 77 stdout, _ = apply_edits.communicate() 78 for process in [run_tool, extract_edits, apply_edits]: 79 process.wait() 80 if process.returncode != 0: 81 print "Failure while running the tool." 82 return process.returncode 83 84 # Reformat the resulting edits via: git cl format. 85 args = ['cl', 'format'] 86 args.extend(actual_files) 87 _RunGit(args) 88 89 return 0 90 91 finally: 92 # No matter what, unstage the git changes we made earlier to avoid polluting 93 # the index. 94 args = ['reset', '--quiet', 'HEAD'] 95 args.extend(actual_files) 96 _RunGit(args) 97 98 99 def main(argv): 100 if len(argv) < 1: 101 print 'Usage: test_tool.py <clang tool>' 102 print ' <clang tool> is the clang tool to be tested.' 103 sys.exit(1) 104 105 tool_to_test = argv[0] 106 print '\nTesting %s\n' % tool_to_test 107 tools_clang_scripts_directory = os.path.dirname(os.path.realpath(__file__)) 108 tools_clang_directory = os.path.dirname(tools_clang_scripts_directory) 109 test_directory_for_tool = os.path.join( 110 tools_clang_directory, tool_to_test, 'tests') 111 compile_database = os.path.join(test_directory_for_tool, 112 'compile_commands.json') 113 source_files = glob.glob(os.path.join(test_directory_for_tool, 114 '*-original.cc')) 115 actual_files = ['-'.join([source_file.rsplit('-', 1)[0], 'actual.cc']) 116 for source_file in source_files] 117 expected_files = ['-'.join([source_file.rsplit('-', 1)[0], 'expected.cc']) 118 for source_file in source_files] 119 include_paths = [] 120 include_paths.append( 121 os.path.realpath(os.path.join(tools_clang_directory, '../..'))) 122 # Many gtest and gmock headers expect to have testing/gtest/include and/or 123 # testing/gmock/include in the include search path. 124 include_paths.append( 125 os.path.realpath(os.path.join(tools_clang_directory, 126 '../..', 127 'testing/gtest/include'))) 128 include_paths.append( 129 os.path.realpath(os.path.join(tools_clang_directory, 130 '../..', 131 'testing/gmock/include'))) 132 133 if len(actual_files) == 0: 134 print 'Tool "%s" does not have compatible test files.' % tool_to_test 135 return 1 136 137 # Set up the test environment. 138 for source, actual in zip(source_files, actual_files): 139 shutil.copyfile(source, actual) 140 # Generate a temporary compilation database to run the tool over. 141 with open(compile_database, 'w') as f: 142 f.write(_GenerateCompileCommands(actual_files, include_paths)) 143 144 # Run the tool. 145 exitcode = _RunToolAndApplyEdits(tools_clang_scripts_directory, tool_to_test, 146 test_directory_for_tool, actual_files) 147 if (exitcode != 0): 148 return exitcode 149 150 # Compare actual-vs-expected results. 151 passed = 0 152 failed = 0 153 for expected, actual in zip(expected_files, actual_files): 154 print '[ RUN ] %s' % os.path.relpath(actual) 155 expected_output = actual_output = None 156 with open(expected, 'r') as f: 157 expected_output = f.readlines() 158 with open(actual, 'r') as f: 159 actual_output = f.readlines() 160 if actual_output != expected_output: 161 failed += 1 162 for line in difflib.unified_diff(expected_output, actual_output, 163 fromfile=os.path.relpath(expected), 164 tofile=os.path.relpath(actual)): 165 sys.stdout.write(line) 166 print '[ FAILED ] %s' % os.path.relpath(actual) 167 # Don't clean up the file on failure, so the results can be referenced 168 # more easily. 169 continue 170 print '[ OK ] %s' % os.path.relpath(actual) 171 passed += 1 172 os.remove(actual) 173 174 if failed == 0: 175 os.remove(compile_database) 176 177 print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files)) 178 if passed > 0: 179 print '[ PASSED ] %s.' % _NumberOfTestsToString(passed) 180 if failed > 0: 181 print '[ FAILED ] %s.' % _NumberOfTestsToString(failed) 182 return 1 183 184 185 if __name__ == '__main__': 186 sys.exit(main(sys.argv[1:])) 187
