1 #!/usr/bin/env python2.7 2 3 # Copyright 2015, ARM Limited 4 # All rights reserved. 5 # 6 # Redistribution and use in source and binary forms, with or without 7 # modification, are permitted provided that the following conditions are met: 8 # 9 # * Redistributions of source code must retain the above copyright notice, 10 # this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above copyright notice, 12 # this list of conditions and the following disclaimer in the documentation 13 # and/or other materials provided with the distribution. 14 # * Neither the name of ARM Limited nor the names of its contributors may be 15 # used to endorse or promote products derived from this software without 16 # specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 19 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import argparse 30 import fcntl 31 import git 32 import itertools 33 import multiprocessing 34 import os 35 from os.path import join 36 import platform 37 import re 38 import subprocess 39 import sys 40 import time 41 42 import config 43 import lint 44 import printer 45 import test 46 import threaded_tests 47 import util 48 49 50 dir_root = config.dir_root 51 52 def Optionify(name): 53 return '--' + name 54 55 56 # The options that can be tested are abstracted to provide an easy way to add 57 # new ones. 58 # Environment options influence the environment. They can be used for example to 59 # set the compiler used. 60 # Build options are options passed to scons, with a syntax like `scons opt=val` 61 # Runtime options are options passed to the test program. 62 # See the definition of `test_options` below. 63 64 # 'all' is a special value for the options. If specified, all other values of 65 # the option are tested. 66 class TestOption(object): 67 type_environment = 'type_environment' 68 type_build = 'type_build' 69 type_run = 'type_run' 70 71 def __init__(self, option_type, name, help, 72 val_test_choices, val_test_default = None, 73 # If unset, the user can pass any value. 74 strict_choices = True): 75 self.name = name 76 self.option_type = option_type 77 self.help = help 78 self.val_test_choices = val_test_choices 79 self.strict_choices = strict_choices 80 if val_test_default is not None: 81 self.val_test_default = val_test_default 82 else: 83 self.val_test_default = val_test_choices[0] 84 85 def ArgList(self, to_test): 86 res = [] 87 if to_test == 'all': 88 for value in self.val_test_choices: 89 if value != 'all': 90 res.append(self.GetOptionString(value)) 91 else: 92 for value in to_test: 93 res.append(self.GetOptionString(value)) 94 return res 95 96 class EnvironmentOption(TestOption): 97 option_type = TestOption.type_environment 98 def __init__(self, name, environment_variable_name, help, 99 val_test_choices, val_test_default = None, 100 strict_choices = True): 101 super(EnvironmentOption, self).__init__(EnvironmentOption.option_type, 102 name, 103 help, 104 val_test_choices, 105 val_test_default, 106 strict_choices = strict_choices) 107 self.environment_variable_name = environment_variable_name 108 109 def GetOptionString(self, value): 110 return self.environment_variable_name + '=' + value 111 112 113 class BuildOption(TestOption): 114 option_type = TestOption.type_build 115 def __init__(self, name, help, 116 val_test_choices, val_test_default = None, 117 strict_choices = True): 118 super(BuildOption, self).__init__(BuildOption.option_type, 119 name, 120 help, 121 val_test_choices, 122 val_test_default, 123 strict_choices = strict_choices) 124 def GetOptionString(self, value): 125 return self.name + '=' + value 126 127 128 class RuntimeOption(TestOption): 129 option_type = TestOption.type_run 130 def __init__(self, name, help, 131 val_test_choices, val_test_default = None): 132 super(RuntimeOption, self).__init__(RuntimeOption.option_type, 133 name, 134 help, 135 val_test_choices, 136 val_test_default) 137 def GetOptionString(self, value): 138 if value == 'on': 139 return Optionify(self.name) 140 else: 141 return None 142 143 144 145 environment_option_compiler = \ 146 EnvironmentOption('compiler', 'CXX', 'Test for the specified compilers.', 147 val_test_choices=['all'] + config.tested_compilers, 148 strict_choices = False) 149 test_environment_options = [ 150 environment_option_compiler 151 ] 152 153 build_option_mode = \ 154 BuildOption('mode', 'Test with the specified build modes.', 155 val_test_choices=['all'] + config.build_options_modes) 156 build_option_standard = \ 157 BuildOption('std', 'Test with the specified C++ standard.', 158 val_test_choices=['all'] + config.tested_cpp_standards, 159 strict_choices = False) 160 test_build_options = [ 161 build_option_mode, 162 build_option_standard 163 ] 164 165 runtime_option_debugger = \ 166 RuntimeOption('debugger', 167 '''Test with the specified configurations for the debugger. 168 Note that this is only tested if we are using the simulator.''', 169 val_test_choices=['all', 'on', 'off']) 170 test_runtime_options = [ 171 runtime_option_debugger 172 ] 173 174 test_options = \ 175 test_environment_options + test_build_options + test_runtime_options 176 177 178 def BuildOptions(): 179 args = argparse.ArgumentParser( 180 description = 181 '''This tool runs all tests matching the speficied filters for multiple 182 environment, build options, and runtime options configurations.''', 183 # Print default values. 184 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 185 186 args.add_argument('filters', metavar='filter', nargs='*', 187 help='Run tests matching all of the (regexp) filters.') 188 189 # We automatically build the script options from the options to be tested. 190 test_arguments = args.add_argument_group( 191 'Test options', 192 'These options indicate what should be tested') 193 for option in test_options: 194 choices = option.val_test_choices if option.strict_choices else None 195 help = option.help 196 if not option.strict_choices: 197 help += ' Supported values: {' + ','.join(option.val_test_choices) + '}' 198 test_arguments.add_argument(Optionify(option.name), 199 nargs='+', 200 choices=choices, 201 default=option.val_test_default, 202 help=help, 203 action='store') 204 205 general_arguments = args.add_argument_group('General options') 206 general_arguments.add_argument('--fast', action='store_true', 207 help='''Skip the lint tests, and run only with 208 one compiler, in one mode, with one C++ 209 standard, and with an appropriate default for 210 runtime options. The compiler, mode, and C++ 211 standard used are the first ones provided to 212 the script or in the default arguments.''') 213 general_arguments.add_argument( 214 '--jobs', '-j', metavar='N', type=int, nargs='?', 215 default=multiprocessing.cpu_count(), 216 const=multiprocessing.cpu_count(), 217 help='''Runs the tests using N jobs. If the option is set but no value is 218 provided, the script will use as many jobs as it thinks useful.''') 219 general_arguments.add_argument('--nobench', action='store_true', 220 help='Do not run benchmarks.') 221 general_arguments.add_argument('--nolint', action='store_true', 222 help='Do not run the linter.') 223 general_arguments.add_argument('--notest', action='store_true', 224 help='Do not run tests.') 225 sim_default = 'off' if platform.machine() == 'aarch64' else 'on' 226 general_arguments.add_argument( 227 '--simulator', action='store', choices=['on', 'off'], 228 default=sim_default, 229 help='Explicitly enable or disable the simulator.') 230 general_arguments.add_argument( 231 '--under_valgrind', action='store_true', 232 help='''Run the test-runner commands under Valgrind. 233 Note that a few tests are known to fail because of 234 issues in Valgrind''') 235 return args.parse_args() 236 237 238 def RunCommand(command, environment_options = None): 239 # Create a copy of the environment. We do not want to pollute the environment 240 # of future commands run. 241 environment = os.environ 242 # Configure the environment. 243 # TODO: We currently pass the options as strings, so we need to parse them. We 244 # should instead pass them as a data structure and build the string option 245 # later. `environment_options` looks like `['CXX=compiler', 'OPT=val']`. 246 if environment_options: 247 for option in environment_options: 248 opt, val = option.split('=') 249 environment[opt] = val 250 251 printable_command = '' 252 if environment_options: 253 printable_command += ' '.join(environment_options) + ' ' 254 printable_command += ' '.join(command) 255 256 printable_command_orange = \ 257 printer.COLOUR_ORANGE + printable_command + printer.NO_COLOUR 258 printer.PrintOverwritableLine(printable_command_orange) 259 sys.stdout.flush() 260 261 # Start a process for the command. 262 # Interleave `stderr` and `stdout`. 263 p = subprocess.Popen(command, 264 stdout=subprocess.PIPE, 265 stderr=subprocess.STDOUT, 266 env=environment) 267 268 # We want to be able to display a continuously updated 'work indicator' while 269 # the process is running. Since the process can hang if the `stdout` pipe is 270 # full, we need to pull from it regularly. We cannot do so via the 271 # `readline()` function because it is blocking, and would thus cause the 272 # indicator to not be updated properly. So use file control mechanisms 273 # instead. 274 indicator = ' (still working: %d seconds elapsed)' 275 276 # Mark the process output as non-blocking. 277 flags = fcntl.fcntl(p.stdout, fcntl.F_GETFL) 278 fcntl.fcntl(p.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK) 279 280 t_start = time.time() 281 t_last_indication = t_start 282 process_output = '' 283 284 # Keep looping as long as the process is running. 285 while p.poll() is None: 286 # Avoid polling too often. 287 time.sleep(0.1) 288 # Update the progress indicator. 289 t_current = time.time() 290 if (t_current - t_start >= 2) and (t_current - t_last_indication >= 1): 291 printer.PrintOverwritableLine( 292 printable_command_orange + indicator % int(t_current - t_start)) 293 sys.stdout.flush() 294 t_last_indication = t_current 295 # Pull from the process output. 296 while True: 297 try: 298 line = os.read(p.stdout.fileno(), 1024) 299 except OSError: 300 line = '' 301 break 302 if line == '': break 303 process_output += line 304 305 # The process has exited. Don't forget to retrieve the rest of its output. 306 out, err = p.communicate() 307 rc = p.poll() 308 process_output += out 309 310 if rc == 0: 311 printer.Print(printer.COLOUR_GREEN + printable_command + printer.NO_COLOUR) 312 else: 313 printer.Print(printer.COLOUR_RED + printable_command + printer.NO_COLOUR) 314 printer.Print(process_output) 315 return rc 316 317 318 def RunLinter(): 319 rc, default_tracked_files = lint.GetDefaultTrackedFiles() 320 if rc: 321 return rc 322 return lint.LintFiles(map(lambda x: join(dir_root, x), default_tracked_files), 323 jobs = args.jobs, progress_prefix = 'cpp lint: ') 324 325 326 327 def BuildAll(build_options, jobs): 328 scons_command = ["scons", "-C", dir_root, 'all', '-j', str(jobs)] 329 scons_command += list(build_options) 330 return RunCommand(scons_command, list(environment_options)) 331 332 333 def RunBenchmarks(): 334 rc = 0 335 benchmark_names = util.ListCCFilesWithoutExt(config.dir_benchmarks) 336 for bench in benchmark_names: 337 rc |= RunCommand( 338 [os.path.realpath(join(config.dir_build_latest, 'benchmarks', bench))]) 339 return rc 340 341 342 def PrintStatus(success): 343 printer.Print('\n$ ' + ' '.join(sys.argv)) 344 if success: 345 printer.Print('SUCCESS') 346 else: 347 printer.Print('FAILURE') 348 349 350 351 if __name__ == '__main__': 352 util.require_program('scons') 353 rc = 0 354 355 args = BuildOptions() 356 357 if args.under_valgrind: 358 util.require_program('valgrind') 359 360 if args.fast: 361 def SetFast(option, specified, default): 362 option.val_test_choices = \ 363 [default[0] if specified == 'all' else specified[0]] 364 SetFast(environment_option_compiler, args.compiler, config.tested_compilers) 365 SetFast(build_option_mode, args.mode, config.build_options_modes) 366 SetFast(build_option_standard, args.std, config.tested_cpp_standards) 367 SetFast(runtime_option_debugger, args.debugger, ['on', 'off']) 368 369 if not args.nolint and not args.fast: 370 rc |= RunLinter() 371 372 # Don't try to test the debugger if we are not running with the simulator. 373 if not args.simulator: 374 test_runtime_options = \ 375 filter(lambda x: x.name != 'debugger', test_runtime_options) 376 377 # List all combinations of options that will be tested. 378 def ListCombinations(args, options): 379 opts_list = map(lambda opt : opt.ArgList(args.__dict__[opt.name]), options) 380 return list(itertools.product(*opts_list)) 381 test_env_combinations = ListCombinations(args, test_environment_options) 382 test_build_combinations = ListCombinations(args, test_build_options) 383 test_runtime_combinations = ListCombinations(args, test_runtime_options) 384 385 for environment_options in test_env_combinations: 386 for build_options in test_build_combinations: 387 # Avoid going through the build stage if we are not using the build 388 # result. 389 if not (args.notest and args.nobench): 390 build_rc = BuildAll(build_options, args.jobs) 391 # Don't run the tests for this configuration if the build failed. 392 if build_rc != 0: 393 rc |= build_rc 394 continue 395 396 # Use the realpath of the test executable so that the commands printed 397 # can be copy-pasted and run. 398 test_executable = os.path.realpath( 399 join(config.dir_build_latest, 'test', 'test-runner')) 400 401 if not args.notest: 402 printer.Print(test_executable) 403 404 for runtime_options in test_runtime_combinations: 405 if not args.notest: 406 runtime_options = [x for x in runtime_options if x is not None] 407 prefix = ' ' + ' '.join(runtime_options) + ' ' 408 rc |= threaded_tests.RunTests(test_executable, 409 args.filters, 410 list(runtime_options), 411 args.under_valgrind, 412 jobs = args.jobs, prefix = prefix) 413 414 if not args.nobench: 415 rc |= RunBenchmarks() 416 417 PrintStatus(rc == 0) 418
