1 #!/bin/env python 2 # Copyright (c) 2011 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 7 """Module to setup and generate code coverage data 8 9 This module first sets up the environment for code coverage, instruments the 10 binaries, runs the tests and collects the code coverage data. 11 12 13 Usage: 14 coverage.py --upload=<upload_location> 15 --revision=<revision_number> 16 --src_root=<root_of_source_tree> 17 [--tools_path=<tools_path>] 18 """ 19 20 import logging 21 import optparse 22 import os 23 import shutil 24 import subprocess 25 import sys 26 import tempfile 27 28 import google.logging_utils 29 import google.process_utils as proc 30 31 32 # The list of binaries that will be instrumented for code coverage 33 # TODO(niranjan): Re-enable instrumentation of chrome.exe and chrome.dll once we 34 # resolve the issue where vsinstr.exe is confused while reading symbols. 35 windows_binaries = [#'chrome.exe', 36 #'chrome.dll', 37 'unit_tests.exe', 38 'automated_ui_tests.exe', 39 'installer_util_unittests.exe', 40 'ipc_tests.exe', 41 'memory_test.exe', 42 'page_cycler_tests.exe', 43 'perf_tests.exe', 44 'reliability_tests.exe', 45 'security_tests.dll', 46 'startup_tests.exe', 47 'tab_switching_test.exe', 48 'test_shell.exe'] 49 50 # The list of [tests, args] that will be run. 51 # Failing tests have been commented out. 52 # TODO(niranjan): Need to add layout tests that excercise the test shell. 53 windows_tests = [ 54 ['unit_tests.exe', ''], 55 # ['automated_ui_tests.exe', ''], 56 ['installer_util_unittests.exe', ''], 57 ['ipc_tests.exe', ''], 58 ['page_cycler_tests.exe', '--gtest_filter=*File --no-sandbox'], 59 ['reliability_tests.exe', '--no-sandbox'], 60 ['startup_tests.exe', '--no-sandbox'], 61 ['tab_switching_test.exe', '--no-sandbox'], 62 ] 63 64 65 def IsWindows(): 66 """Checks if the current platform is Windows. 67 """ 68 return sys.platform[:3] == 'win' 69 70 71 class Coverage(object): 72 """Class to set up and generate code coverage. 73 74 This class contains methods that are useful to set up the environment for 75 code coverage. 76 77 Attributes: 78 instrumented: A boolean indicating if all the binaries have been 79 instrumented. 80 """ 81 82 def __init__(self, 83 revision, 84 src_path = None, 85 tools_path = None, 86 archive=None): 87 """Init method for the Coverage class. 88 89 Args: 90 revision: Revision number of the Chromium source tree. 91 src_path: Location of the Chromium source base. 92 tools_path: Location of the Visual Studio Team Tools. (Win32 only) 93 archive: Archive location for the intermediate .coverage results. 94 """ 95 google.logging_utils.config_root() 96 self.revision = revision 97 self.instrumented = False 98 self.tools_path = tools_path 99 self.src_path = src_path 100 self._dir = tempfile.mkdtemp() 101 self._archive = archive 102 103 def SetUp(self, binaries): 104 """Set up the platform specific environment and instrument the binaries for 105 coverage. 106 107 This method sets up the environment, instruments all the compiled binaries 108 and sets up the code coverage counters. 109 110 Args: 111 binaries: List of binaries that need to be instrumented. 112 113 Returns: 114 True on success. 115 False on error. 116 """ 117 if self.instrumented: 118 logging.error('Binaries already instrumented') 119 return False 120 if IsWindows(): 121 # Stop all previous instance of VSPerfMon counters 122 counters_command = ('%s -shutdown' % 123 (os.path.join(self.tools_path, 'vsperfcmd.exe'))) 124 (retcode, output) = proc.RunCommandFull(counters_command, 125 collect_output=True) 126 # TODO(niranjan): Add a check that to verify that the binaries were built 127 # using the /PROFILE linker flag. 128 if self.tools_path == None: 129 logging.error('Could not locate Visual Studio Team Server tools') 130 return False 131 # Remove trailing slashes 132 self.tools_path = self.tools_path.rstrip('\\') 133 # Add this to the env PATH. 134 os.environ['PATH'] = os.environ['PATH'] + ';' + self.tools_path 135 instrument_command = '%s /COVERAGE ' % (os.path.join(self.tools_path, 136 'vsinstr.exe')) 137 for binary in binaries: 138 logging.info('binary = %s' % (binary)) 139 logging.info('instrument_command = %s' % (instrument_command)) 140 # Instrument each binary in the list 141 binary = os.path.join(self.src_path, 'chrome', 'Release', binary) 142 (retcode, output) = proc.RunCommandFull(instrument_command + binary, 143 collect_output=True) 144 # Check if the file has been instrumented correctly. 145 if output.pop().rfind('Successfully instrumented') == -1: 146 logging.error('Error instrumenting %s' % (binary)) 147 return False 148 # We are now ready to run tests and measure code coverage. 149 self.instrumented = True 150 return True 151 152 def TearDown(self): 153 """Tear down method. 154 155 This method shuts down the counters, and cleans up all the intermediate 156 artifacts. 157 """ 158 if self.instrumented == False: 159 return 160 161 if IsWindows(): 162 # Stop counters 163 counters_command = ('%s -shutdown' % 164 (os.path.join(self.tools_path, 'vsperfcmd.exe'))) 165 (retcode, output) = proc.RunCommandFull(counters_command, 166 collect_output=True) 167 logging.info('Counters shut down: %s' % (output)) 168 # TODO(niranjan): Revert the instrumented binaries to their original 169 # versions. 170 else: 171 return 172 if self._archive: 173 shutil.copytree(self._dir, os.path.join(self._archive, self.revision)) 174 logging.info('Archived the .coverage files') 175 # Delete all the temp files and folders 176 if self._dir != None: 177 shutil.rmtree(self._dir, ignore_errors=True) 178 logging.info('Cleaned up temporary files and folders') 179 # Reset the instrumented flag. 180 self.instrumented = False 181 182 def RunTest(self, src_root, test): 183 """Run tests and collect the .coverage file 184 185 Args: 186 src_root: Path to the root of the source. 187 test: Path to the test to be run. 188 189 Returns: 190 Path of the intermediate .coverage file on success. 191 None on error. 192 """ 193 # Generate the intermediate file name for the coverage results 194 test_name = os.path.split(test[0])[1].strip('.exe') 195 # test_command = binary + args 196 test_command = '%s %s' % (os.path.join(src_root, 197 'chrome', 198 'Release', 199 test[0]), 200 test[1]) 201 202 coverage_file = os.path.join(self._dir, '%s_win32_%s.coverage' % 203 (test_name, self.revision)) 204 logging.info('.coverage file for test %s: %s' % (test_name, coverage_file)) 205 206 # After all the binaries have been instrumented, we start the counters. 207 counters_command = ('%s -start:coverage -output:%s' % 208 (os.path.join(self.tools_path, 'vsperfcmd.exe'), 209 coverage_file)) 210 # Here we use subprocess.call() instead of the RunCommandFull because the 211 # VSPerfCmd spawns another process before terminating and this confuses 212 # the subprocess.Popen() used by RunCommandFull. 213 retcode = subprocess.call(counters_command) 214 215 # Run the test binary 216 logging.info('Executing test %s: ' % test_command) 217 (retcode, output) = proc.RunCommandFull(test_command, collect_output=True) 218 if retcode != 0: # Return error if the tests fail 219 logging.error('One or more tests failed in %s.' % test_command) 220 return None 221 222 # Stop the counters 223 counters_command = ('%s -shutdown' % 224 (os.path.join(self.tools_path, 'vsperfcmd.exe'))) 225 (retcode, output) = proc.RunCommandFull(counters_command, 226 collect_output=True) 227 logging.info('Counters shut down: %s' % (output)) 228 # Return the intermediate .coverage file 229 return coverage_file 230 231 def Upload(self, list_coverage, upload_path, sym_path=None, src_root=None): 232 """Upload the results to the dashboard. 233 234 This method uploads the coverage data to a dashboard where it will be 235 processed. On Windows, this method will first convert the .coverage file to 236 the lcov format. This method needs to be called before the TearDown method. 237 238 Args: 239 list_coverage: The list of coverage data files to consoliate and upload. 240 upload_path: Destination where the coverage data will be processed. 241 sym_path: Symbol path for the build (Win32 only) 242 src_root: Root folder of the source tree (Win32 only) 243 244 Returns: 245 True on success. 246 False on failure. 247 """ 248 if upload_path == None: 249 logging.info('Upload path not specified. Will not convert to LCOV') 250 return True 251 252 if IsWindows(): 253 # Stop counters 254 counters_command = ('%s -shutdown' % 255 (os.path.join(self.tools_path, 'vsperfcmd.exe'))) 256 (retcode, output) = proc.RunCommandFull(counters_command, 257 collect_output=True) 258 logging.info('Counters shut down: %s' % (output)) 259 lcov_file = os.path.join(upload_path, 'chrome_win32_%s.lcov' % 260 (self.revision)) 261 lcov = open(lcov_file, 'w') 262 for coverage_file in list_coverage: 263 # Convert the intermediate .coverage file to lcov format 264 if self.tools_path == None: 265 logging.error('Lcov converter tool not found') 266 return False 267 self.tools_path = self.tools_path.rstrip('\\') 268 convert_command = ('%s -sym_path=%s -src_root=%s %s' % 269 (os.path.join(self.tools_path, 270 'coverage_analyzer.exe'), 271 sym_path, 272 src_root, 273 coverage_file)) 274 (retcode, output) = proc.RunCommandFull(convert_command, 275 collect_output=True) 276 # TODO(niranjan): Fix this to check for the correct return code. 277 # if output != 0: 278 # logging.error('Conversion to LCOV failed. Exiting.') 279 tmp_lcov_file = coverage_file + '.lcov' 280 logging.info('Conversion to lcov complete for %s' % (coverage_file)) 281 # Now append this .lcov file to the cumulative lcov file 282 logging.info('Consolidating LCOV file: %s' % (tmp_lcov_file)) 283 tmp_lcov = open(tmp_lcov_file, 'r') 284 lcov.write(tmp_lcov.read()) 285 tmp_lcov.close() 286 lcov.close() 287 logging.info('LCOV file uploaded to %s' % (upload_path)) 288 289 290 def main(): 291 # Command line parsing 292 parser = optparse.OptionParser() 293 # Path where the .coverage to .lcov converter tools are stored. 294 parser.add_option('-t', 295 '--tools_path', 296 dest='tools_path', 297 default=None, 298 help='Location of the coverage tools (windows only)') 299 parser.add_option('-u', 300 '--upload', 301 dest='upload_path', 302 default=None, 303 help='Location where the results should be uploaded') 304 # We need the revision number so that we can generate the output file of the 305 # format chrome_<platform>_<revision>.lcov 306 parser.add_option('-r', 307 '--revision', 308 dest='revision', 309 default=None, 310 help='Revision number of the Chromium source repo') 311 # Root of the source tree. Needed for converting the generated .coverage file 312 # on Windows to the open source lcov format. 313 parser.add_option('-s', 314 '--src_root', 315 dest='src_root', 316 default=None, 317 help='Root of the source repository') 318 parser.add_option('-a', 319 '--archive', 320 dest='archive', 321 default=None, 322 help='Archive location of the intermediate .coverage data') 323 324 (options, args) = parser.parse_args() 325 326 if options.revision == None: 327 parser.error('Revision number not specified') 328 if options.src_root == None: 329 parser.error('Source root not specified') 330 331 if IsWindows(): 332 # Initialize coverage 333 cov = Coverage(options.revision, 334 options.src_root, 335 options.tools_path, 336 options.archive) 337 list_coverage = [] 338 # Instrument the binaries 339 if cov.SetUp(windows_binaries): 340 # Run all the tests 341 for test in windows_tests: 342 coverage = cov.RunTest(options.src_root, test) 343 if coverage == None: # Indicate failure to the buildbots. 344 return 1 345 # Collect the intermediate file 346 list_coverage.append(coverage) 347 else: 348 logging.error('Error during instrumentation.') 349 sys.exit(1) 350 351 cov.Upload(list_coverage, 352 options.upload_path, 353 os.path.join(options.src_root, 'chrome', 'Release'), 354 options.src_root) 355 cov.TearDown() 356 357 358 if __name__ == '__main__': 359 sys.exit(main()) 360
