1 import argparse 2 import os 3 import sys 4 from test import support 5 6 7 USAGE = """\ 8 python -m test [options] [test_name1 [test_name2 ...]] 9 python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] 10 """ 11 12 DESCRIPTION = """\ 13 Run Python regression tests. 14 15 If no arguments or options are provided, finds all files matching 16 the pattern "test_*" in the Lib/test subdirectory and runs 17 them in alphabetical order (but see -M and -u, below, for exceptions). 18 19 For more rigorous testing, it is useful to use the following 20 command line: 21 22 python -E -Wd -m test [options] [test_name1 ...] 23 """ 24 25 EPILOG = """\ 26 Additional option details: 27 28 -r randomizes test execution order. You can use --randseed=int to provide an 29 int seed value for the randomizer; this is useful for reproducing troublesome 30 test orders. 31 32 -s On the first invocation of regrtest using -s, the first test file found 33 or the first test file given on the command line is run, and the name of 34 the next test is recorded in a file named pynexttest. If run from the 35 Python build directory, pynexttest is located in the 'build' subdirectory, 36 otherwise it is located in tempfile.gettempdir(). On subsequent runs, 37 the test in pynexttest is run, and the next test is written to pynexttest. 38 When the last test has been run, pynexttest is deleted. In this way it 39 is possible to single step through the test files. This is useful when 40 doing memory analysis on the Python interpreter, which process tends to 41 consume too many resources to run the full regression test non-stop. 42 43 -S is used to continue running tests after an aborted run. It will 44 maintain the order a standard run (ie, this assumes -r is not used). 45 This is useful after the tests have prematurely stopped for some external 46 reason and you want to start running from where you left off rather 47 than starting from the beginning. 48 49 -f reads the names of tests from the file given as f's argument, one 50 or more test names per line. Whitespace is ignored. Blank lines and 51 lines beginning with '#' are ignored. This is especially useful for 52 whittling down failures involving interactions among tests. 53 54 -L causes the leaks(1) command to be run just before exit if it exists. 55 leaks(1) is available on Mac OS X and presumably on some other 56 FreeBSD-derived systems. 57 58 -R runs each test several times and examines sys.gettotalrefcount() to 59 see if the test appears to be leaking references. The argument should 60 be of the form stab:run:fname where 'stab' is the number of times the 61 test is run to let gettotalrefcount settle down, 'run' is the number 62 of times further it is run and 'fname' is the name of the file the 63 reports are written to. These parameters all have defaults (5, 4 and 64 "reflog.txt" respectively), and the minimal invocation is '-R :'. 65 66 -M runs tests that require an exorbitant amount of memory. These tests 67 typically try to ascertain containers keep working when containing more than 68 2 billion objects, which only works on 64-bit systems. There are also some 69 tests that try to exhaust the address space of the process, which only makes 70 sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, 71 which is a string in the form of '2.5Gb', determines howmuch memory the 72 tests will limit themselves to (but they may go slightly over.) The number 73 shouldn't be more memory than the machine has (including swap memory). You 74 should also keep in mind that swap memory is generally much, much slower 75 than RAM, and setting memlimit to all available RAM or higher will heavily 76 tax the machine. On the other hand, it is no use running these tests with a 77 limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect 78 to use more than memlimit memory will be skipped. The big-memory tests 79 generally run very, very long. 80 81 -u is used to specify which special resource intensive tests to run, 82 such as those requiring large file support or network connectivity. 83 The argument is a comma-separated list of words indicating the 84 resources to test. Currently only the following are defined: 85 86 all - Enable all special resources. 87 88 none - Disable all special resources (this is the default). 89 90 audio - Tests that use the audio device. (There are known 91 cases of broken audio drivers that can crash Python or 92 even the Linux kernel.) 93 94 curses - Tests that use curses and will modify the terminal's 95 state and output modes. 96 97 largefile - It is okay to run some test that may create huge 98 files. These tests can take a long time and may 99 consume >2GB of disk space temporarily. 100 101 network - It is okay to run tests that use external network 102 resource, e.g. testing SSL support for sockets. 103 104 decimal - Test the decimal module against a large suite that 105 verifies compliance with standards. 106 107 cpu - Used for certain CPU-heavy tests. 108 109 subprocess Run all tests for the subprocess module. 110 111 urlfetch - It is okay to download files required on testing. 112 113 gui - Run tests that require a running GUI. 114 115 tzdata - Run tests that require timezone data. 116 117 To enable all resources except one, use '-uall,-<resource>'. For 118 example, to run all the tests except for the gui tests, give the 119 option '-uall,-gui'. 120 """ 121 122 123 RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 124 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'tzdata') 125 126 class _ArgParser(argparse.ArgumentParser): 127 128 def error(self, message): 129 super().error(message + "\nPass -h or --help for complete help.") 130 131 132 def _create_parser(): 133 # Set prog to prevent the uninformative "__main__.py" from displaying in 134 # error messages when using "python -m test ...". 135 parser = _ArgParser(prog='regrtest.py', 136 usage=USAGE, 137 description=DESCRIPTION, 138 epilog=EPILOG, 139 add_help=False, 140 formatter_class=argparse.RawDescriptionHelpFormatter) 141 142 # Arguments with this clause added to its help are described further in 143 # the epilog's "Additional option details" section. 144 more_details = ' See the section at bottom for more details.' 145 146 group = parser.add_argument_group('General options') 147 # We add help explicitly to control what argument group it renders under. 148 group.add_argument('-h', '--help', action='help', 149 help='show this help message and exit') 150 group.add_argument('--timeout', metavar='TIMEOUT', type=float, 151 help='dump the traceback and exit if a test takes ' 152 'more than TIMEOUT seconds; disabled if TIMEOUT ' 153 'is negative or equals to zero') 154 group.add_argument('--wait', action='store_true', 155 help='wait for user input, e.g., allow a debugger ' 156 'to be attached') 157 group.add_argument('--slaveargs', metavar='ARGS') 158 group.add_argument('-S', '--start', metavar='START', 159 help='the name of the test at which to start.' + 160 more_details) 161 162 group = parser.add_argument_group('Verbosity') 163 group.add_argument('-v', '--verbose', action='count', 164 help='run tests in verbose mode with output to stdout') 165 group.add_argument('-w', '--verbose2', action='store_true', 166 help='re-run failed tests in verbose mode') 167 group.add_argument('-W', '--verbose3', action='store_true', 168 help='display test output on failure') 169 group.add_argument('-q', '--quiet', action='store_true', 170 help='no output unless one or more tests fail') 171 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow', 172 help='print the slowest 10 tests') 173 group.add_argument('--header', action='store_true', 174 help='print header with interpreter info') 175 176 group = parser.add_argument_group('Selecting tests') 177 group.add_argument('-r', '--randomize', action='store_true', 178 help='randomize test execution order.' + more_details) 179 group.add_argument('--randseed', metavar='SEED', 180 dest='random_seed', type=int, 181 help='pass a random seed to reproduce a previous ' 182 'random run') 183 group.add_argument('-f', '--fromfile', metavar='FILE', 184 help='read names of tests to run from a file.' + 185 more_details) 186 group.add_argument('-x', '--exclude', action='store_true', 187 help='arguments are tests to *exclude*') 188 group.add_argument('-s', '--single', action='store_true', 189 help='single step through a set of tests.' + 190 more_details) 191 group.add_argument('-m', '--match', metavar='PAT', 192 dest='match_tests', 193 help='match test cases and methods with glob pattern PAT') 194 group.add_argument('-G', '--failfast', action='store_true', 195 help='fail as soon as a test fails (only with -v or -W)') 196 group.add_argument('-u', '--use', metavar='RES1,RES2,...', 197 action='append', type=resources_list, 198 help='specify which special resource intensive tests ' 199 'to run.' + more_details) 200 group.add_argument('-M', '--memlimit', metavar='LIMIT', 201 help='run very large memory-consuming tests.' + 202 more_details) 203 group.add_argument('--testdir', metavar='DIR', 204 type=relative_filename, 205 help='execute test files in the specified directory ' 206 '(instead of the Python stdlib test suite)') 207 208 group = parser.add_argument_group('Special runs') 209 group.add_argument('-l', '--findleaks', action='store_true', 210 help='if GC is available detect tests that leak memory') 211 group.add_argument('-L', '--runleaks', action='store_true', 212 help='run the leaks(1) command just before exit.' + 213 more_details) 214 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', 215 type=huntrleaks, 216 help='search for reference leaks (needs debug build, ' 217 'very slow).' + more_details) 218 group.add_argument('-j', '--multiprocess', metavar='PROCESSES', 219 dest='use_mp', type=int, 220 help='run PROCESSES processes at once') 221 group.add_argument('-T', '--coverage', action='store_true', 222 dest='trace', 223 help='turn on code coverage tracing using the trace ' 224 'module') 225 group.add_argument('-D', '--coverdir', metavar='DIR', 226 type=relative_filename, 227 help='directory where coverage files are put') 228 group.add_argument('-N', '--nocoverdir', 229 action='store_const', const=None, dest='coverdir', 230 help='put coverage files alongside modules') 231 group.add_argument('-t', '--threshold', metavar='THRESHOLD', 232 type=int, 233 help='call gc.set_threshold(THRESHOLD)') 234 group.add_argument('-n', '--nowindows', action='store_true', 235 help='suppress error message boxes on Windows') 236 group.add_argument('-F', '--forever', action='store_true', 237 help='run the specified tests in a loop, until an ' 238 'error happens') 239 group.add_argument('--list-tests', action='store_true', 240 help="only write the name of tests that will be run, " 241 "don't execute them") 242 group.add_argument('-P', '--pgo', dest='pgo', action='store_true', 243 help='enable Profile Guided Optimization training') 244 245 return parser 246 247 248 def relative_filename(string): 249 # CWD is replaced with a temporary dir before calling main(), so we 250 # join it with the saved CWD so it ends up where the user expects. 251 return os.path.join(support.SAVEDCWD, string) 252 253 254 def huntrleaks(string): 255 args = string.split(':') 256 if len(args) not in (2, 3): 257 raise argparse.ArgumentTypeError( 258 'needs 2 or 3 colon-separated arguments') 259 nwarmup = int(args[0]) if args[0] else 5 260 ntracked = int(args[1]) if args[1] else 4 261 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' 262 return nwarmup, ntracked, fname 263 264 265 def resources_list(string): 266 u = [x.lower() for x in string.split(',')] 267 for r in u: 268 if r == 'all' or r == 'none': 269 continue 270 if r[0] == '-': 271 r = r[1:] 272 if r not in RESOURCE_NAMES: 273 raise argparse.ArgumentTypeError('invalid resource: ' + r) 274 return u 275 276 277 def _parse_args(args, **kwargs): 278 # Defaults 279 ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, 280 exclude=False, single=False, randomize=False, fromfile=None, 281 findleaks=False, use_resources=None, trace=False, coverdir='coverage', 282 runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, 283 random_seed=None, use_mp=None, verbose3=False, forever=False, 284 header=False, failfast=False, match_tests=None, pgo=False) 285 for k, v in kwargs.items(): 286 if not hasattr(ns, k): 287 raise TypeError('%r is an invalid keyword argument ' 288 'for this function' % k) 289 setattr(ns, k, v) 290 if ns.use_resources is None: 291 ns.use_resources = [] 292 293 parser = _create_parser() 294 # Issue #14191: argparse doesn't support "intermixed" positional and 295 # optional arguments. Use parse_known_args() as workaround. 296 ns.args = parser.parse_known_args(args=args, namespace=ns)[1] 297 for arg in ns.args: 298 if arg.startswith('-'): 299 parser.error("unrecognized arguments: %s" % arg) 300 sys.exit(1) 301 302 if ns.single and ns.fromfile: 303 parser.error("-s and -f don't go together!") 304 if ns.use_mp is not None and ns.trace: 305 parser.error("-T and -j don't go together!") 306 if ns.use_mp is not None and ns.findleaks: 307 parser.error("-l and -j don't go together!") 308 if ns.failfast and not (ns.verbose or ns.verbose3): 309 parser.error("-G/--failfast needs either -v or -W") 310 if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): 311 parser.error("--pgo/-v don't go together!") 312 313 if ns.nowindows: 314 print("Warning: the --nowindows (-n) option is deprecated. " 315 "Use -vv to display assertions in stderr.", file=sys.stderr) 316 317 if ns.quiet: 318 ns.verbose = 0 319 if ns.timeout is not None: 320 if ns.timeout <= 0: 321 ns.timeout = None 322 if ns.use_mp is not None: 323 if ns.use_mp <= 0: 324 # Use all cores + extras for tests that like to sleep 325 ns.use_mp = 2 + (os.cpu_count() or 1) 326 if ns.use: 327 for a in ns.use: 328 for r in a: 329 if r == 'all': 330 ns.use_resources[:] = RESOURCE_NAMES 331 continue 332 if r == 'none': 333 del ns.use_resources[:] 334 continue 335 remove = False 336 if r[0] == '-': 337 remove = True 338 r = r[1:] 339 if remove: 340 if r in ns.use_resources: 341 ns.use_resources.remove(r) 342 elif r not in ns.use_resources: 343 ns.use_resources.append(r) 344 if ns.random_seed is not None: 345 ns.randomize = True 346 347 return ns 348
