1 import argparse 2 import os 3 import sys 4 from test import support 5 6 7 USAGE = """\ 8 python -m test [options] [test_name1 [test_name2 ...]] 9 python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] 10 """ 11 12 DESCRIPTION = """\ 13 Run Python regression tests. 14 15 If no arguments or options are provided, finds all files matching 16 the pattern "test_*" in the Lib/test subdirectory and runs 17 them in alphabetical order (but see -M and -u, below, for exceptions). 18 19 For more rigorous testing, it is useful to use the following 20 command line: 21 22 python -E -Wd -m test [options] [test_name1 ...] 23 """ 24 25 EPILOG = """\ 26 Additional option details: 27 28 -r randomizes test execution order. You can use --randseed=int to provide an 29 int seed value for the randomizer; this is useful for reproducing troublesome 30 test orders. 31 32 -s On the first invocation of regrtest using -s, the first test file found 33 or the first test file given on the command line is run, and the name of 34 the next test is recorded in a file named pynexttest. If run from the 35 Python build directory, pynexttest is located in the 'build' subdirectory, 36 otherwise it is located in tempfile.gettempdir(). On subsequent runs, 37 the test in pynexttest is run, and the next test is written to pynexttest. 38 When the last test has been run, pynexttest is deleted. In this way it 39 is possible to single step through the test files. This is useful when 40 doing memory analysis on the Python interpreter, which process tends to 41 consume too many resources to run the full regression test non-stop. 42 43 -S is used to continue running tests after an aborted run. It will 44 maintain the order a standard run (ie, this assumes -r is not used). 45 This is useful after the tests have prematurely stopped for some external 46 reason and you want to start running from where you left off rather 47 than starting from the beginning. 48 49 -f reads the names of tests from the file given as f's argument, one 50 or more test names per line. Whitespace is ignored. Blank lines and 51 lines beginning with '#' are ignored. This is especially useful for 52 whittling down failures involving interactions among tests. 53 54 -L causes the leaks(1) command to be run just before exit if it exists. 55 leaks(1) is available on Mac OS X and presumably on some other 56 FreeBSD-derived systems. 57 58 -R runs each test several times and examines sys.gettotalrefcount() to 59 see if the test appears to be leaking references. The argument should 60 be of the form stab:run:fname where 'stab' is the number of times the 61 test is run to let gettotalrefcount settle down, 'run' is the number 62 of times further it is run and 'fname' is the name of the file the 63 reports are written to. These parameters all have defaults (5, 4 and 64 "reflog.txt" respectively), and the minimal invocation is '-R :'. 65 66 -M runs tests that require an exorbitant amount of memory. These tests 67 typically try to ascertain containers keep working when containing more than 68 2 billion objects, which only works on 64-bit systems. There are also some 69 tests that try to exhaust the address space of the process, which only makes 70 sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, 71 which is a string in the form of '2.5Gb', determines how much memory the 72 tests will limit themselves to (but they may go slightly over.) The number 73 shouldn't be more memory than the machine has (including swap memory). You 74 should also keep in mind that swap memory is generally much, much slower 75 than RAM, and setting memlimit to all available RAM or higher will heavily 76 tax the machine. On the other hand, it is no use running these tests with a 77 limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect 78 to use more than memlimit memory will be skipped. The big-memory tests 79 generally run very, very long. 80 81 -u is used to specify which special resource intensive tests to run, 82 such as those requiring large file support or network connectivity. 83 The argument is a comma-separated list of words indicating the 84 resources to test. Currently only the following are defined: 85 86 all - Enable all special resources. 87 88 none - Disable all special resources (this is the default). 89 90 audio - Tests that use the audio device. (There are known 91 cases of broken audio drivers that can crash Python or 92 even the Linux kernel.) 93 94 curses - Tests that use curses and will modify the terminal's 95 state and output modes. 96 97 largefile - It is okay to run some test that may create huge 98 files. These tests can take a long time and may 99 consume >2 GiB of disk space temporarily. 100 101 network - It is okay to run tests that use external network 102 resource, e.g. testing SSL support for sockets. 103 104 decimal - Test the decimal module against a large suite that 105 verifies compliance with standards. 106 107 cpu - Used for certain CPU-heavy tests. 108 109 subprocess Run all tests for the subprocess module. 110 111 urlfetch - It is okay to download files required on testing. 112 113 gui - Run tests that require a running GUI. 114 115 tzdata - Run tests that require timezone data. 116 117 To enable all resources except one, use '-uall,-<resource>'. For 118 example, to run all the tests except for the gui tests, give the 119 option '-uall,-gui'. 120 121 --matchfile filters tests using a text file, one pattern per line. 122 Pattern examples: 123 124 - test method: test_stat_attributes 125 - test class: FileTests 126 - test identifier: test_os.FileTests.test_stat_attributes 127 """ 128 129 130 ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', 131 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') 132 133 # Other resources excluded from --use=all: 134 # 135 # - extralagefile (ex: test_zipfile64): really too slow to be enabled 136 # "by default" 137 # - tzdata: while needed to validate fully test_datetime, it makes 138 # test_datetime too slow (15-20 min on some buildbots) and so is disabled by 139 # default (see bpo-30822). 140 RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') 141 142 class _ArgParser(argparse.ArgumentParser): 143 144 def error(self, message): 145 super().error(message + "\nPass -h or --help for complete help.") 146 147 148 def _create_parser(): 149 # Set prog to prevent the uninformative "__main__.py" from displaying in 150 # error messages when using "python -m test ...". 151 parser = _ArgParser(prog='regrtest.py', 152 usage=USAGE, 153 description=DESCRIPTION, 154 epilog=EPILOG, 155 add_help=False, 156 formatter_class=argparse.RawDescriptionHelpFormatter) 157 158 # Arguments with this clause added to its help are described further in 159 # the epilog's "Additional option details" section. 160 more_details = ' See the section at bottom for more details.' 161 162 group = parser.add_argument_group('General options') 163 # We add help explicitly to control what argument group it renders under. 164 group.add_argument('-h', '--help', action='help', 165 help='show this help message and exit') 166 group.add_argument('--timeout', metavar='TIMEOUT', type=float, 167 help='dump the traceback and exit if a test takes ' 168 'more than TIMEOUT seconds; disabled if TIMEOUT ' 169 'is negative or equals to zero') 170 group.add_argument('--wait', action='store_true', 171 help='wait for user input, e.g., allow a debugger ' 172 'to be attached') 173 group.add_argument('--worker-args', metavar='ARGS') 174 group.add_argument('-S', '--start', metavar='START', 175 help='the name of the test at which to start.' + 176 more_details) 177 178 group = parser.add_argument_group('Verbosity') 179 group.add_argument('-v', '--verbose', action='count', 180 help='run tests in verbose mode with output to stdout') 181 group.add_argument('-w', '--verbose2', action='store_true', 182 help='re-run failed tests in verbose mode') 183 group.add_argument('-W', '--verbose3', action='store_true', 184 help='display test output on failure') 185 group.add_argument('-q', '--quiet', action='store_true', 186 help='no output unless one or more tests fail') 187 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow', 188 help='print the slowest 10 tests') 189 group.add_argument('--header', action='store_true', 190 help='print header with interpreter info') 191 192 group = parser.add_argument_group('Selecting tests') 193 group.add_argument('-r', '--randomize', action='store_true', 194 help='randomize test execution order.' + more_details) 195 group.add_argument('--randseed', metavar='SEED', 196 dest='random_seed', type=int, 197 help='pass a random seed to reproduce a previous ' 198 'random run') 199 group.add_argument('-f', '--fromfile', metavar='FILE', 200 help='read names of tests to run from a file.' + 201 more_details) 202 group.add_argument('-x', '--exclude', action='store_true', 203 help='arguments are tests to *exclude*') 204 group.add_argument('-s', '--single', action='store_true', 205 help='single step through a set of tests.' + 206 more_details) 207 group.add_argument('-m', '--match', metavar='PAT', 208 dest='match_tests', action='append', 209 help='match test cases and methods with glob pattern PAT') 210 group.add_argument('--matchfile', metavar='FILENAME', 211 dest='match_filename', 212 help='similar to --match but get patterns from a ' 213 'text file, one pattern per line') 214 group.add_argument('-G', '--failfast', action='store_true', 215 help='fail as soon as a test fails (only with -v or -W)') 216 group.add_argument('-u', '--use', metavar='RES1,RES2,...', 217 action='append', type=resources_list, 218 help='specify which special resource intensive tests ' 219 'to run.' + more_details) 220 group.add_argument('-M', '--memlimit', metavar='LIMIT', 221 help='run very large memory-consuming tests.' + 222 more_details) 223 group.add_argument('--testdir', metavar='DIR', 224 type=relative_filename, 225 help='execute test files in the specified directory ' 226 '(instead of the Python stdlib test suite)') 227 228 group = parser.add_argument_group('Special runs') 229 group.add_argument('-l', '--findleaks', action='store_true', 230 help='if GC is available detect tests that leak memory') 231 group.add_argument('-L', '--runleaks', action='store_true', 232 help='run the leaks(1) command just before exit.' + 233 more_details) 234 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', 235 type=huntrleaks, 236 help='search for reference leaks (needs debug build, ' 237 'very slow).' + more_details) 238 group.add_argument('-j', '--multiprocess', metavar='PROCESSES', 239 dest='use_mp', type=int, 240 help='run PROCESSES processes at once') 241 group.add_argument('-T', '--coverage', action='store_true', 242 dest='trace', 243 help='turn on code coverage tracing using the trace ' 244 'module') 245 group.add_argument('-D', '--coverdir', metavar='DIR', 246 type=relative_filename, 247 help='directory where coverage files are put') 248 group.add_argument('-N', '--nocoverdir', 249 action='store_const', const=None, dest='coverdir', 250 help='put coverage files alongside modules') 251 group.add_argument('-t', '--threshold', metavar='THRESHOLD', 252 type=int, 253 help='call gc.set_threshold(THRESHOLD)') 254 group.add_argument('-n', '--nowindows', action='store_true', 255 help='suppress error message boxes on Windows') 256 group.add_argument('-F', '--forever', action='store_true', 257 help='run the specified tests in a loop, until an ' 258 'error happens') 259 group.add_argument('--list-tests', action='store_true', 260 help="only write the name of tests that will be run, " 261 "don't execute them") 262 group.add_argument('--list-cases', action='store_true', 263 help='only write the name of test cases that will be run' 264 ' , don\'t execute them') 265 group.add_argument('-P', '--pgo', dest='pgo', action='store_true', 266 help='enable Profile Guided Optimization training') 267 group.add_argument('--fail-env-changed', action='store_true', 268 help='if a test file alters the environment, mark ' 269 'the test as failed') 270 271 group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', 272 help='writes JUnit-style XML results to the specified ' 273 'file') 274 group.add_argument('--tempdir', dest='tempdir', metavar='PATH', 275 help='override the working directory for the test run') 276 return parser 277 278 279 def relative_filename(string): 280 # CWD is replaced with a temporary dir before calling main(), so we 281 # join it with the saved CWD so it ends up where the user expects. 282 return os.path.join(support.SAVEDCWD, string) 283 284 285 def huntrleaks(string): 286 args = string.split(':') 287 if len(args) not in (2, 3): 288 raise argparse.ArgumentTypeError( 289 'needs 2 or 3 colon-separated arguments') 290 nwarmup = int(args[0]) if args[0] else 5 291 ntracked = int(args[1]) if args[1] else 4 292 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' 293 return nwarmup, ntracked, fname 294 295 296 def resources_list(string): 297 u = [x.lower() for x in string.split(',')] 298 for r in u: 299 if r == 'all' or r == 'none': 300 continue 301 if r[0] == '-': 302 r = r[1:] 303 if r not in RESOURCE_NAMES: 304 raise argparse.ArgumentTypeError('invalid resource: ' + r) 305 return u 306 307 308 def _parse_args(args, **kwargs): 309 # Defaults 310 ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, 311 exclude=False, single=False, randomize=False, fromfile=None, 312 findleaks=False, use_resources=None, trace=False, coverdir='coverage', 313 runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, 314 random_seed=None, use_mp=None, verbose3=False, forever=False, 315 header=False, failfast=False, match_tests=None, pgo=False) 316 for k, v in kwargs.items(): 317 if not hasattr(ns, k): 318 raise TypeError('%r is an invalid keyword argument ' 319 'for this function' % k) 320 setattr(ns, k, v) 321 if ns.use_resources is None: 322 ns.use_resources = [] 323 324 parser = _create_parser() 325 # Issue #14191: argparse doesn't support "intermixed" positional and 326 # optional arguments. Use parse_known_args() as workaround. 327 ns.args = parser.parse_known_args(args=args, namespace=ns)[1] 328 for arg in ns.args: 329 if arg.startswith('-'): 330 parser.error("unrecognized arguments: %s" % arg) 331 sys.exit(1) 332 333 if ns.single and ns.fromfile: 334 parser.error("-s and -f don't go together!") 335 if ns.use_mp is not None and ns.trace: 336 parser.error("-T and -j don't go together!") 337 if ns.use_mp is not None and ns.findleaks: 338 parser.error("-l and -j don't go together!") 339 if ns.failfast and not (ns.verbose or ns.verbose3): 340 parser.error("-G/--failfast needs either -v or -W") 341 if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): 342 parser.error("--pgo/-v don't go together!") 343 344 if ns.nowindows: 345 print("Warning: the --nowindows (-n) option is deprecated. " 346 "Use -vv to display assertions in stderr.", file=sys.stderr) 347 348 if ns.quiet: 349 ns.verbose = 0 350 if ns.timeout is not None: 351 if ns.timeout <= 0: 352 ns.timeout = None 353 if ns.use_mp is not None: 354 if ns.use_mp <= 0: 355 # Use all cores + extras for tests that like to sleep 356 ns.use_mp = 2 + (os.cpu_count() or 1) 357 if ns.use: 358 for a in ns.use: 359 for r in a: 360 if r == 'all': 361 ns.use_resources[:] = ALL_RESOURCES 362 continue 363 if r == 'none': 364 del ns.use_resources[:] 365 continue 366 remove = False 367 if r[0] == '-': 368 remove = True 369 r = r[1:] 370 if remove: 371 if r in ns.use_resources: 372 ns.use_resources.remove(r) 373 elif r not in ns.use_resources: 374 ns.use_resources.append(r) 375 if ns.random_seed is not None: 376 ns.randomize = True 377 if ns.verbose: 378 ns.header = True 379 if ns.huntrleaks and ns.verbose3: 380 ns.verbose3 = False 381 print("WARNING: Disable --verbose3 because it's incompatible with " 382 "--huntrleaks: see http://bugs.python.org/issue27103", 383 file=sys.stderr) 384 if ns.match_filename: 385 if ns.match_tests is None: 386 ns.match_tests = [] 387 with open(ns.match_filename) as fp: 388 for line in fp: 389 ns.match_tests.append(line.strip()) 390 391 return ns 392
