1 # Module doctest. 2 # Released to the public domain 16-Jan-2001, by Tim Peters (tim (at] python.org). 3 # Major enhancements and refactoring by: 4 # Jim Fulton 5 # Edward Loper 6 7 # Provided as-is; use at your own risk; no warranty; no promises; enjoy! 8 9 r"""Module doctest -- a framework for running examples in docstrings. 10 11 In simplest use, end each module M to be tested with: 12 13 def _test(): 14 import doctest 15 doctest.testmod() 16 17 if __name__ == "__main__": 18 _test() 19 20 Then running the module as a script will cause the examples in the 21 docstrings to get executed and verified: 22 23 python M.py 24 25 This won't display anything unless an example fails, in which case the 26 failing example(s) and the cause(s) of the failure(s) are printed to stdout 27 (why not stderr? because stderr is a lame hack <0.2 wink>), and the final 28 line of output is "Test failed.". 29 30 Run it with the -v switch instead: 31 32 python M.py -v 33 34 and a detailed report of all examples tried is printed to stdout, along 35 with assorted summaries at the end. 36 37 You can force verbose mode by passing "verbose=True" to testmod, or prohibit 38 it by passing "verbose=False". In either of those cases, sys.argv is not 39 examined by testmod. 40 41 There are a variety of other ways to run doctests, including integration 42 with the unittest framework, and support for running non-Python text 43 files containing doctests. There are also many ways to override parts 44 of doctest's default behaviors. See the Library Reference Manual for 45 details. 46 """ 47 48 __docformat__ = 'reStructuredText en' 49 50 __all__ = [ 51 # 0, Option Flags 52 'register_optionflag', 53 'DONT_ACCEPT_TRUE_FOR_1', 54 'DONT_ACCEPT_BLANKLINE', 55 'NORMALIZE_WHITESPACE', 56 'ELLIPSIS', 57 'SKIP', 58 'IGNORE_EXCEPTION_DETAIL', 59 'COMPARISON_FLAGS', 60 'REPORT_UDIFF', 61 'REPORT_CDIFF', 62 'REPORT_NDIFF', 63 'REPORT_ONLY_FIRST_FAILURE', 64 'REPORTING_FLAGS', 65 'FAIL_FAST', 66 # 1. Utility Functions 67 # 2. Example & DocTest 68 'Example', 69 'DocTest', 70 # 3. Doctest Parser 71 'DocTestParser', 72 # 4. Doctest Finder 73 'DocTestFinder', 74 # 5. Doctest Runner 75 'DocTestRunner', 76 'OutputChecker', 77 'DocTestFailure', 78 'UnexpectedException', 79 'DebugRunner', 80 # 6. Test Functions 81 'testmod', 82 'testfile', 83 'run_docstring_examples', 84 # 7. Unittest Support 85 'DocTestSuite', 86 'DocFileSuite', 87 'set_unittest_reportflags', 88 # 8. Debugging Support 89 'script_from_examples', 90 'testsource', 91 'debug_src', 92 'debug', 93 ] 94 95 import __future__ 96 import argparse 97 import difflib 98 import inspect 99 import linecache 100 import os 101 import pdb 102 import re 103 import sys 104 import traceback 105 import unittest 106 from io import StringIO 107 from collections import namedtuple 108 109 TestResults = namedtuple('TestResults', 'failed attempted') 110 111 # There are 4 basic classes: 112 # - Example: a <source, want> pair, plus an intra-docstring line number. 113 # - DocTest: a collection of examples, parsed from a docstring, plus 114 # info about where the docstring came from (name, filename, lineno). 115 # - DocTestFinder: extracts DocTests from a given object's docstring and 116 # its contained objects' docstrings. 117 # - DocTestRunner: runs DocTest cases, and accumulates statistics. 118 # 119 # So the basic picture is: 120 # 121 # list of: 122 # +------+ +---------+ +-------+ 123 # |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| 124 # +------+ +---------+ +-------+ 125 # | Example | 126 # | ... | 127 # | Example | 128 # +---------+ 129 130 # Option constants. 131 132 OPTIONFLAGS_BY_NAME = {} 133 def register_optionflag(name): 134 # Create a new flag unless `name` is already known. 135 return OPTIONFLAGS_BY_NAME.setdefault(name, 1 << len(OPTIONFLAGS_BY_NAME)) 136 137 DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') 138 DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') 139 NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') 140 ELLIPSIS = register_optionflag('ELLIPSIS') 141 SKIP = register_optionflag('SKIP') 142 IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') 143 144 COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | 145 DONT_ACCEPT_BLANKLINE | 146 NORMALIZE_WHITESPACE | 147 ELLIPSIS | 148 SKIP | 149 IGNORE_EXCEPTION_DETAIL) 150 151 REPORT_UDIFF = register_optionflag('REPORT_UDIFF') 152 REPORT_CDIFF = register_optionflag('REPORT_CDIFF') 153 REPORT_NDIFF = register_optionflag('REPORT_NDIFF') 154 REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') 155 FAIL_FAST = register_optionflag('FAIL_FAST') 156 157 REPORTING_FLAGS = (REPORT_UDIFF | 158 REPORT_CDIFF | 159 REPORT_NDIFF | 160 REPORT_ONLY_FIRST_FAILURE | 161 FAIL_FAST) 162 163 # Special string markers for use in `want` strings: 164 BLANKLINE_MARKER = '<BLANKLINE>' 165 ELLIPSIS_MARKER = '...' 166 167 ###################################################################### 168 ## Table of Contents 169 ###################################################################### 170 # 1. Utility Functions 171 # 2. Example & DocTest -- store test cases 172 # 3. DocTest Parser -- extracts examples from strings 173 # 4. DocTest Finder -- extracts test cases from objects 174 # 5. DocTest Runner -- runs test cases 175 # 6. Test Functions -- convenient wrappers for testing 176 # 7. Unittest Support 177 # 8. Debugging Support 178 # 9. Example Usage 179 180 ###################################################################### 181 ## 1. Utility Functions 182 ###################################################################### 183 184 def _extract_future_flags(globs): 185 """ 186 Return the compiler-flags associated with the future features that 187 have been imported into the given namespace (globs). 188 """ 189 flags = 0 190 for fname in __future__.all_feature_names: 191 feature = globs.get(fname, None) 192 if feature is getattr(__future__, fname): 193 flags |= feature.compiler_flag 194 return flags 195 196 def _normalize_module(module, depth=2): 197 """ 198 Return the module specified by `module`. In particular: 199 - If `module` is a module, then return module. 200 - If `module` is a string, then import and return the 201 module with that name. 202 - If `module` is None, then return the calling module. 203 The calling module is assumed to be the module of 204 the stack frame at the given depth in the call stack. 205 """ 206 if inspect.ismodule(module): 207 return module 208 elif isinstance(module, str): 209 return __import__(module, globals(), locals(), ["*"]) 210 elif module is None: 211 return sys.modules[sys._getframe(depth).f_globals['__name__']] 212 else: 213 raise TypeError("Expected a module, string, or None") 214 215 def _load_testfile(filename, package, module_relative, encoding): 216 if module_relative: 217 package = _normalize_module(package, 3) 218 filename = _module_relative_path(package, filename) 219 if getattr(package, '__loader__', None) is not None: 220 if hasattr(package.__loader__, 'get_data'): 221 file_contents = package.__loader__.get_data(filename) 222 file_contents = file_contents.decode(encoding) 223 # get_data() opens files as 'rb', so one must do the equivalent 224 # conversion as universal newlines would do. 225 return file_contents.replace(os.linesep, '\n'), filename 226 with open(filename, encoding=encoding) as f: 227 return f.read(), filename 228 229 def _indent(s, indent=4): 230 """ 231 Add the given number of space characters to the beginning of 232 every non-blank line in `s`, and return the result. 233 """ 234 # This regexp matches the start of non-blank lines: 235 return re.sub('(?m)^(?!$)', indent*' ', s) 236 237 def _exception_traceback(exc_info): 238 """ 239 Return a string containing a traceback message for the given 240 exc_info tuple (as returned by sys.exc_info()). 241 """ 242 # Get a traceback message. 243 excout = StringIO() 244 exc_type, exc_val, exc_tb = exc_info 245 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) 246 return excout.getvalue() 247 248 # Override some StringIO methods. 249 class _SpoofOut(StringIO): 250 def getvalue(self): 251 result = StringIO.getvalue(self) 252 # If anything at all was written, make sure there's a trailing 253 # newline. There's no way for the expected output to indicate 254 # that a trailing newline is missing. 255 if result and not result.endswith("\n"): 256 result += "\n" 257 return result 258 259 def truncate(self, size=None): 260 self.seek(size) 261 StringIO.truncate(self) 262 263 # Worst-case linear-time ellipsis matching. 264 def _ellipsis_match(want, got): 265 """ 266 Essentially the only subtle case: 267 >>> _ellipsis_match('aa...aa', 'aaa') 268 False 269 """ 270 if ELLIPSIS_MARKER not in want: 271 return want == got 272 273 # Find "the real" strings. 274 ws = want.split(ELLIPSIS_MARKER) 275 assert len(ws) >= 2 276 277 # Deal with exact matches possibly needed at one or both ends. 278 startpos, endpos = 0, len(got) 279 w = ws[0] 280 if w: # starts with exact match 281 if got.startswith(w): 282 startpos = len(w) 283 del ws[0] 284 else: 285 return False 286 w = ws[-1] 287 if w: # ends with exact match 288 if got.endswith(w): 289 endpos -= len(w) 290 del ws[-1] 291 else: 292 return False 293 294 if startpos > endpos: 295 # Exact end matches required more characters than we have, as in 296 # _ellipsis_match('aa...aa', 'aaa') 297 return False 298 299 # For the rest, we only need to find the leftmost non-overlapping 300 # match for each piece. If there's no overall match that way alone, 301 # there's no overall match period. 302 for w in ws: 303 # w may be '' at times, if there are consecutive ellipses, or 304 # due to an ellipsis at the start or end of `want`. That's OK. 305 # Search for an empty string succeeds, and doesn't change startpos. 306 startpos = got.find(w, startpos, endpos) 307 if startpos < 0: 308 return False 309 startpos += len(w) 310 311 return True 312 313 def _comment_line(line): 314 "Return a commented form of the given line" 315 line = line.rstrip() 316 if line: 317 return '# '+line 318 else: 319 return '#' 320 321 def _strip_exception_details(msg): 322 # Support for IGNORE_EXCEPTION_DETAIL. 323 # Get rid of everything except the exception name; in particular, drop 324 # the possibly dotted module path (if any) and the exception message (if 325 # any). We assume that a colon is never part of a dotted name, or of an 326 # exception name. 327 # E.g., given 328 # "foo.bar.MyError: la di da" 329 # return "MyError" 330 # Or for "abc.def" or "abc.def:\n" return "def". 331 332 start, end = 0, len(msg) 333 # The exception name must appear on the first line. 334 i = msg.find("\n") 335 if i >= 0: 336 end = i 337 # retain up to the first colon (if any) 338 i = msg.find(':', 0, end) 339 if i >= 0: 340 end = i 341 # retain just the exception name 342 i = msg.rfind('.', 0, end) 343 if i >= 0: 344 start = i+1 345 return msg[start: end] 346 347 class _OutputRedirectingPdb(pdb.Pdb): 348 """ 349 A specialized version of the python debugger that redirects stdout 350 to a given stream when interacting with the user. Stdout is *not* 351 redirected when traced code is executed. 352 """ 353 def __init__(self, out): 354 self.__out = out 355 self.__debugger_used = False 356 # do not play signal games in the pdb 357 pdb.Pdb.__init__(self, stdout=out, nosigint=True) 358 # still use input() to get user input 359 self.use_rawinput = 1 360 361 def set_trace(self, frame=None): 362 self.__debugger_used = True 363 if frame is None: 364 frame = sys._getframe().f_back 365 pdb.Pdb.set_trace(self, frame) 366 367 def set_continue(self): 368 # Calling set_continue unconditionally would break unit test 369 # coverage reporting, as Bdb.set_continue calls sys.settrace(None). 370 if self.__debugger_used: 371 pdb.Pdb.set_continue(self) 372 373 def trace_dispatch(self, *args): 374 # Redirect stdout to the given stream. 375 save_stdout = sys.stdout 376 sys.stdout = self.__out 377 # Call Pdb's trace dispatch method. 378 try: 379 return pdb.Pdb.trace_dispatch(self, *args) 380 finally: 381 sys.stdout = save_stdout 382 383 # [XX] Normalize with respect to os.path.pardir? 384 def _module_relative_path(module, test_path): 385 if not inspect.ismodule(module): 386 raise TypeError('Expected a module: %r' % module) 387 if test_path.startswith('/'): 388 raise ValueError('Module-relative files may not have absolute paths') 389 390 # Normalize the path. On Windows, replace "/" with "\". 391 test_path = os.path.join(*(test_path.split('/'))) 392 393 # Find the base directory for the path. 394 if hasattr(module, '__file__'): 395 # A normal module/package 396 basedir = os.path.split(module.__file__)[0] 397 elif module.__name__ == '__main__': 398 # An interactive session. 399 if len(sys.argv)>0 and sys.argv[0] != '': 400 basedir = os.path.split(sys.argv[0])[0] 401 else: 402 basedir = os.curdir 403 else: 404 if hasattr(module, '__path__'): 405 for directory in module.__path__: 406 fullpath = os.path.join(directory, test_path) 407 if os.path.exists(fullpath): 408 return fullpath 409 410 # A module w/o __file__ (this includes builtins) 411 raise ValueError("Can't resolve paths relative to the module " 412 "%r (it has no __file__)" 413 % module.__name__) 414 415 # Combine the base directory and the test path. 416 return os.path.join(basedir, test_path) 417 418 ###################################################################### 419 ## 2. Example & DocTest 420 ###################################################################### 421 ## - An "example" is a <source, want> pair, where "source" is a 422 ## fragment of source code, and "want" is the expected output for 423 ## "source." The Example class also includes information about 424 ## where the example was extracted from. 425 ## 426 ## - A "doctest" is a collection of examples, typically extracted from 427 ## a string (such as an object's docstring). The DocTest class also 428 ## includes information about where the string was extracted from. 429 430 class Example: 431 """ 432 A single doctest example, consisting of source code and expected 433 output. `Example` defines the following attributes: 434 435 - source: A single Python statement, always ending with a newline. 436 The constructor adds a newline if needed. 437 438 - want: The expected output from running the source code (either 439 from stdout, or a traceback in case of exception). `want` ends 440 with a newline unless it's empty, in which case it's an empty 441 string. The constructor adds a newline if needed. 442 443 - exc_msg: The exception message generated by the example, if 444 the example is expected to generate an exception; or `None` if 445 it is not expected to generate an exception. This exception 446 message is compared against the return value of 447 `traceback.format_exception_only()`. `exc_msg` ends with a 448 newline unless it's `None`. The constructor adds a newline 449 if needed. 450 451 - lineno: The line number within the DocTest string containing 452 this Example where the Example begins. This line number is 453 zero-based, with respect to the beginning of the DocTest. 454 455 - indent: The example's indentation in the DocTest string. 456 I.e., the number of space characters that precede the 457 example's first prompt. 458 459 - options: A dictionary mapping from option flags to True or 460 False, which is used to override default options for this 461 example. Any option flags not contained in this dictionary 462 are left at their default value (as specified by the 463 DocTestRunner's optionflags). By default, no options are set. 464 """ 465 def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, 466 options=None): 467 # Normalize inputs. 468 if not source.endswith('\n'): 469 source += '\n' 470 if want and not want.endswith('\n'): 471 want += '\n' 472 if exc_msg is not None and not exc_msg.endswith('\n'): 473 exc_msg += '\n' 474 # Store properties. 475 self.source = source 476 self.want = want 477 self.lineno = lineno 478 self.indent = indent 479 if options is None: options = {} 480 self.options = options 481 self.exc_msg = exc_msg 482 483 def __eq__(self, other): 484 if type(self) is not type(other): 485 return NotImplemented 486 487 return self.source == other.source and \ 488 self.want == other.want and \ 489 self.lineno == other.lineno and \ 490 self.indent == other.indent and \ 491 self.options == other.options and \ 492 self.exc_msg == other.exc_msg 493 494 def __hash__(self): 495 return hash((self.source, self.want, self.lineno, self.indent, 496 self.exc_msg)) 497 498 class DocTest: 499 """ 500 A collection of doctest examples that should be run in a single 501 namespace. Each `DocTest` defines the following attributes: 502 503 - examples: the list of examples. 504 505 - globs: The namespace (aka globals) that the examples should 506 be run in. 507 508 - name: A name identifying the DocTest (typically, the name of 509 the object whose docstring this DocTest was extracted from). 510 511 - filename: The name of the file that this DocTest was extracted 512 from, or `None` if the filename is unknown. 513 514 - lineno: The line number within filename where this DocTest 515 begins, or `None` if the line number is unavailable. This 516 line number is zero-based, with respect to the beginning of 517 the file. 518 519 - docstring: The string that the examples were extracted from, 520 or `None` if the string is unavailable. 521 """ 522 def __init__(self, examples, globs, name, filename, lineno, docstring): 523 """ 524 Create a new DocTest containing the given examples. The 525 DocTest's globals are initialized with a copy of `globs`. 526 """ 527 assert not isinstance(examples, str), \ 528 "DocTest no longer accepts str; use DocTestParser instead" 529 self.examples = examples 530 self.docstring = docstring 531 self.globs = globs.copy() 532 self.name = name 533 self.filename = filename 534 self.lineno = lineno 535 536 def __repr__(self): 537 if len(self.examples) == 0: 538 examples = 'no examples' 539 elif len(self.examples) == 1: 540 examples = '1 example' 541 else: 542 examples = '%d examples' % len(self.examples) 543 return ('<%s %s from %s:%s (%s)>' % 544 (self.__class__.__name__, 545 self.name, self.filename, self.lineno, examples)) 546 547 def __eq__(self, other): 548 if type(self) is not type(other): 549 return NotImplemented 550 551 return self.examples == other.examples and \ 552 self.docstring == other.docstring and \ 553 self.globs == other.globs and \ 554 self.name == other.name and \ 555 self.filename == other.filename and \ 556 self.lineno == other.lineno 557 558 def __hash__(self): 559 return hash((self.docstring, self.name, self.filename, self.lineno)) 560 561 # This lets us sort tests by name: 562 def __lt__(self, other): 563 if not isinstance(other, DocTest): 564 return NotImplemented 565 return ((self.name, self.filename, self.lineno, id(self)) 566 < 567 (other.name, other.filename, other.lineno, id(other))) 568 569 ###################################################################### 570 ## 3. DocTestParser 571 ###################################################################### 572 573 class DocTestParser: 574 """ 575 A class used to parse strings containing doctest examples. 576 """ 577 # This regular expression is used to find doctest examples in a 578 # string. It defines three groups: `source` is the source code 579 # (including leading indentation and prompts); `indent` is the 580 # indentation of the first (PS1) line of the source code; and 581 # `want` is the expected output (including leading indentation). 582 _EXAMPLE_RE = re.compile(r''' 583 # Source consists of a PS1 line followed by zero or more PS2 lines. 584 (?P<source> 585 (?:^(?P<indent> [ ]*) >>> .*) # PS1 line 586 (?:\n [ ]* \.\.\. .*)*) # PS2 lines 587 \n? 588 # Want consists of any non-blank lines that do not start with PS1. 589 (?P<want> (?:(?![ ]*$) # Not a blank line 590 (?![ ]*>>>) # Not a line starting with PS1 591 .+$\n? # But any other line 592 )*) 593 ''', re.MULTILINE | re.VERBOSE) 594 595 # A regular expression for handling `want` strings that contain 596 # expected exceptions. It divides `want` into three pieces: 597 # - the traceback header line (`hdr`) 598 # - the traceback stack (`stack`) 599 # - the exception message (`msg`), as generated by 600 # traceback.format_exception_only() 601 # `msg` may have multiple lines. We assume/require that the 602 # exception message is the first non-indented line starting with a word 603 # character following the traceback header line. 604 _EXCEPTION_RE = re.compile(r""" 605 # Grab the traceback header. Different versions of Python have 606 # said different things on the first traceback line. 607 ^(?P<hdr> Traceback\ \( 608 (?: most\ recent\ call\ last 609 | innermost\ last 610 ) \) : 611 ) 612 \s* $ # toss trailing whitespace on the header. 613 (?P<stack> .*?) # don't blink: absorb stuff until... 614 ^ (?P<msg> \w+ .*) # a line *starts* with alphanum. 615 """, re.VERBOSE | re.MULTILINE | re.DOTALL) 616 617 # A callable returning a true value iff its argument is a blank line 618 # or contains a single comment. 619 _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match 620 621 def parse(self, string, name='<string>'): 622 """ 623 Divide the given string into examples and intervening text, 624 and return them as a list of alternating Examples and strings. 625 Line numbers for the Examples are 0-based. The optional 626 argument `name` is a name identifying this string, and is only 627 used for error messages. 628 """ 629 string = string.expandtabs() 630 # If all lines begin with the same indentation, then strip it. 631 min_indent = self._min_indent(string) 632 if min_indent > 0: 633 string = '\n'.join([l[min_indent:] for l in string.split('\n')]) 634 635 output = [] 636 charno, lineno = 0, 0 637 # Find all doctest examples in the string: 638 for m in self._EXAMPLE_RE.finditer(string): 639 # Add the pre-example text to `output`. 640 output.append(string[charno:m.start()]) 641 # Update lineno (lines before this example) 642 lineno += string.count('\n', charno, m.start()) 643 # Extract info from the regexp match. 644 (source, options, want, exc_msg) = \ 645 self._parse_example(m, name, lineno) 646 # Create an Example, and add it to the list. 647 if not self._IS_BLANK_OR_COMMENT(source): 648 output.append( Example(source, want, exc_msg, 649 lineno=lineno, 650 indent=min_indent+len(m.group('indent')), 651 options=options) ) 652 # Update lineno (lines inside this example) 653 lineno += string.count('\n', m.start(), m.end()) 654 # Update charno. 655 charno = m.end() 656 # Add any remaining post-example text to `output`. 657 output.append(string[charno:]) 658 return output 659 660 def get_doctest(self, string, globs, name, filename, lineno): 661 """ 662 Extract all doctest examples from the given string, and 663 collect them into a `DocTest` object. 664 665 `globs`, `name`, `filename`, and `lineno` are attributes for 666 the new `DocTest` object. See the documentation for `DocTest` 667 for more information. 668 """ 669 return DocTest(self.get_examples(string, name), globs, 670 name, filename, lineno, string) 671 672 def get_examples(self, string, name='<string>'): 673 """ 674 Extract all doctest examples from the given string, and return 675 them as a list of `Example` objects. Line numbers are 676 0-based, because it's most common in doctests that nothing 677 interesting appears on the same line as opening triple-quote, 678 and so the first interesting line is called \"line 1\" then. 679 680 The optional argument `name` is a name identifying this 681 string, and is only used for error messages. 682 """ 683 return [x for x in self.parse(string, name) 684 if isinstance(x, Example)] 685 686 def _parse_example(self, m, name, lineno): 687 """ 688 Given a regular expression match from `_EXAMPLE_RE` (`m`), 689 return a pair `(source, want)`, where `source` is the matched 690 example's source code (with prompts and indentation stripped); 691 and `want` is the example's expected output (with indentation 692 stripped). 693 694 `name` is the string's name, and `lineno` is the line number 695 where the example starts; both are used for error messages. 696 """ 697 # Get the example's indentation level. 698 indent = len(m.group('indent')) 699 700 # Divide source into lines; check that they're properly 701 # indented; and then strip their indentation & prompts. 702 source_lines = m.group('source').split('\n') 703 self._check_prompt_blank(source_lines, indent, name, lineno) 704 self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) 705 source = '\n'.join([sl[indent+4:] for sl in source_lines]) 706 707 # Divide want into lines; check that it's properly indented; and 708 # then strip the indentation. Spaces before the last newline should 709 # be preserved, so plain rstrip() isn't good enough. 710 want = m.group('want') 711 want_lines = want.split('\n') 712 if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): 713 del want_lines[-1] # forget final newline & spaces after it 714 self._check_prefix(want_lines, ' '*indent, name, 715 lineno + len(source_lines)) 716 want = '\n'.join([wl[indent:] for wl in want_lines]) 717 718 # If `want` contains a traceback message, then extract it. 719 m = self._EXCEPTION_RE.match(want) 720 if m: 721 exc_msg = m.group('msg') 722 else: 723 exc_msg = None 724 725 # Extract options from the source. 726 options = self._find_options(source, name, lineno) 727 728 return source, options, want, exc_msg 729 730 # This regular expression looks for option directives in the 731 # source code of an example. Option directives are comments 732 # starting with "doctest:". Warning: this may give false 733 # positives for string-literals that contain the string 734 # "#doctest:". Eliminating these false positives would require 735 # actually parsing the string; but we limit them by ignoring any 736 # line containing "#doctest:" that is *followed* by a quote mark. 737 _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', 738 re.MULTILINE) 739 740 def _find_options(self, source, name, lineno): 741 """ 742 Return a dictionary containing option overrides extracted from 743 option directives in the given source string. 744 745 `name` is the string's name, and `lineno` is the line number 746 where the example starts; both are used for error messages. 747 """ 748 options = {} 749 # (note: with the current regexp, this will match at most once:) 750 for m in self._OPTION_DIRECTIVE_RE.finditer(source): 751 option_strings = m.group(1).replace(',', ' ').split() 752 for option in option_strings: 753 if (option[0] not in '+-' or 754 option[1:] not in OPTIONFLAGS_BY_NAME): 755 raise ValueError('line %r of the doctest for %s ' 756 'has an invalid option: %r' % 757 (lineno+1, name, option)) 758 flag = OPTIONFLAGS_BY_NAME[option[1:]] 759 options[flag] = (option[0] == '+') 760 if options and self._IS_BLANK_OR_COMMENT(source): 761 raise ValueError('line %r of the doctest for %s has an option ' 762 'directive on a line with no example: %r' % 763 (lineno, name, source)) 764 return options 765 766 # This regular expression finds the indentation of every non-blank 767 # line in a string. 768 _INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) 769 770 def _min_indent(self, s): 771 "Return the minimum indentation of any non-blank line in `s`" 772 indents = [len(indent) for indent in self._INDENT_RE.findall(s)] 773 if len(indents) > 0: 774 return min(indents) 775 else: 776 return 0 777 778 def _check_prompt_blank(self, lines, indent, name, lineno): 779 """ 780 Given the lines of a source string (including prompts and 781 leading indentation), check to make sure that every prompt is 782 followed by a space character. If any line is not followed by 783 a space character, then raise ValueError. 784 """ 785 for i, line in enumerate(lines): 786 if len(line) >= indent+4 and line[indent+3] != ' ': 787 raise ValueError('line %r of the docstring for %s ' 788 'lacks blank after %s: %r' % 789 (lineno+i+1, name, 790 line[indent:indent+3], line)) 791 792 def _check_prefix(self, lines, prefix, name, lineno): 793 """ 794 Check that every line in the given list starts with the given 795 prefix; if any line does not, then raise a ValueError. 796 """ 797 for i, line in enumerate(lines): 798 if line and not line.startswith(prefix): 799 raise ValueError('line %r of the docstring for %s has ' 800 'inconsistent leading whitespace: %r' % 801 (lineno+i+1, name, line)) 802 803 804 ###################################################################### 805 ## 4. DocTest Finder 806 ###################################################################### 807 808 class DocTestFinder: 809 """ 810 A class used to extract the DocTests that are relevant to a given 811 object, from its docstring and the docstrings of its contained 812 objects. Doctests can currently be extracted from the following 813 object types: modules, functions, classes, methods, staticmethods, 814 classmethods, and properties. 815 """ 816 817 def __init__(self, verbose=False, parser=DocTestParser(), 818 recurse=True, exclude_empty=True): 819 """ 820 Create a new doctest finder. 821 822 The optional argument `parser` specifies a class or 823 function that should be used to create new DocTest objects (or 824 objects that implement the same interface as DocTest). The 825 signature for this factory function should match the signature 826 of the DocTest constructor. 827 828 If the optional argument `recurse` is false, then `find` will 829 only examine the given object, and not any contained objects. 830 831 If the optional argument `exclude_empty` is false, then `find` 832 will include tests for objects with empty docstrings. 833 """ 834 self._parser = parser 835 self._verbose = verbose 836 self._recurse = recurse 837 self._exclude_empty = exclude_empty 838 839 def find(self, obj, name=None, module=None, globs=None, extraglobs=None): 840 """ 841 Return a list of the DocTests that are defined by the given 842 object's docstring, or by any of its contained objects' 843 docstrings. 844 845 The optional parameter `module` is the module that contains 846 the given object. If the module is not specified or is None, then 847 the test finder will attempt to automatically determine the 848 correct module. The object's module is used: 849 850 - As a default namespace, if `globs` is not specified. 851 - To prevent the DocTestFinder from extracting DocTests 852 from objects that are imported from other modules. 853 - To find the name of the file containing the object. 854 - To help find the line number of the object within its 855 file. 856 857 Contained objects whose module does not match `module` are ignored. 858 859 If `module` is False, no attempt to find the module will be made. 860 This is obscure, of use mostly in tests: if `module` is False, or 861 is None but cannot be found automatically, then all objects are 862 considered to belong to the (non-existent) module, so all contained 863 objects will (recursively) be searched for doctests. 864 865 The globals for each DocTest is formed by combining `globs` 866 and `extraglobs` (bindings in `extraglobs` override bindings 867 in `globs`). A new copy of the globals dictionary is created 868 for each DocTest. If `globs` is not specified, then it 869 defaults to the module's `__dict__`, if specified, or {} 870 otherwise. If `extraglobs` is not specified, then it defaults 871 to {}. 872 873 """ 874 # If name was not specified, then extract it from the object. 875 if name is None: 876 name = getattr(obj, '__name__', None) 877 if name is None: 878 raise ValueError("DocTestFinder.find: name must be given " 879 "when obj.__name__ doesn't exist: %r" % 880 (type(obj),)) 881 882 # Find the module that contains the given object (if obj is 883 # a module, then module=obj.). Note: this may fail, in which 884 # case module will be None. 885 if module is False: 886 module = None 887 elif module is None: 888 module = inspect.getmodule(obj) 889 890 # Read the module's source code. This is used by 891 # DocTestFinder._find_lineno to find the line number for a 892 # given object's docstring. 893 try: 894 file = inspect.getsourcefile(obj) 895 except TypeError: 896 source_lines = None 897 else: 898 if not file: 899 # Check to see if it's one of our special internal "files" 900 # (see __patched_linecache_getlines). 901 file = inspect.getfile(obj) 902 if not file[0]+file[-2:] == '<]>': file = None 903 if file is None: 904 source_lines = None 905 else: 906 if module is not None: 907 # Supply the module globals in case the module was 908 # originally loaded via a PEP 302 loader and 909 # file is not a valid filesystem path 910 source_lines = linecache.getlines(file, module.__dict__) 911 else: 912 # No access to a loader, so assume it's a normal 913 # filesystem path 914 source_lines = linecache.getlines(file) 915 if not source_lines: 916 source_lines = None 917 918 # Initialize globals, and merge in extraglobs. 919 if globs is None: 920 if module is None: 921 globs = {} 922 else: 923 globs = module.__dict__.copy() 924 else: 925 globs = globs.copy() 926 if extraglobs is not None: 927 globs.update(extraglobs) 928 if '__name__' not in globs: 929 globs['__name__'] = '__main__' # provide a default module name 930 931 # Recursively explore `obj`, extracting DocTests. 932 tests = [] 933 self._find(tests, obj, name, module, source_lines, globs, {}) 934 # Sort the tests by alpha order of names, for consistency in 935 # verbose-mode output. This was a feature of doctest in Pythons 936 # <= 2.3 that got lost by accident in 2.4. It was repaired in 937 # 2.4.4 and 2.5. 938 tests.sort() 939 return tests 940 941 def _from_module(self, module, object): 942 """ 943 Return true if the given object is defined in the given 944 module. 945 """ 946 if module is None: 947 return True 948 elif inspect.getmodule(object) is not None: 949 return module is inspect.getmodule(object) 950 elif inspect.isfunction(object): 951 return module.__dict__ is object.__globals__ 952 elif inspect.ismethoddescriptor(object): 953 if hasattr(object, '__objclass__'): 954 obj_mod = object.__objclass__.__module__ 955 elif hasattr(object, '__module__'): 956 obj_mod = object.__module__ 957 else: 958 return True # [XX] no easy way to tell otherwise 959 return module.__name__ == obj_mod 960 elif inspect.isclass(object): 961 return module.__name__ == object.__module__ 962 elif hasattr(object, '__module__'): 963 return module.__name__ == object.__module__ 964 elif isinstance(object, property): 965 return True # [XX] no way not be sure. 966 else: 967 raise ValueError("object must be a class or function") 968 969 def _find(self, tests, obj, name, module, source_lines, globs, seen): 970 """ 971 Find tests for the given object and any contained objects, and 972 add them to `tests`. 973 """ 974 if self._verbose: 975 print('Finding tests in %s' % name) 976 977 # If we've already processed this object, then ignore it. 978 if id(obj) in seen: 979 return 980 seen[id(obj)] = 1 981 982 # Find a test for this object, and add it to the list of tests. 983 test = self._get_test(obj, name, module, globs, source_lines) 984 if test is not None: 985 tests.append(test) 986 987 # Look for tests in a module's contained objects. 988 if inspect.ismodule(obj) and self._recurse: 989 for valname, val in obj.__dict__.items(): 990 valname = '%s.%s' % (name, valname) 991 # Recurse to functions & classes. 992 if ((inspect.isroutine(inspect.unwrap(val)) 993 or inspect.isclass(val)) and 994 self._from_module(module, val)): 995 self._find(tests, val, valname, module, source_lines, 996 globs, seen) 997 998 # Look for tests in a module's __test__ dictionary. 999 if inspect.ismodule(obj) and self._recurse: 1000 for valname, val in getattr(obj, '__test__', {}).items(): 1001 if not isinstance(valname, str): 1002 raise ValueError("DocTestFinder.find: __test__ keys " 1003 "must be strings: %r" % 1004 (type(valname),)) 1005 if not (inspect.isroutine(val) or inspect.isclass(val) or 1006 inspect.ismodule(val) or isinstance(val, str)): 1007 raise ValueError("DocTestFinder.find: __test__ values " 1008 "must be strings, functions, methods, " 1009 "classes, or modules: %r" % 1010 (type(val),)) 1011 valname = '%s.__test__.%s' % (name, valname) 1012 self._find(tests, val, valname, module, source_lines, 1013 globs, seen) 1014 1015 # Look for tests in a class's contained objects. 1016 if inspect.isclass(obj) and self._recurse: 1017 for valname, val in obj.__dict__.items(): 1018 # Special handling for staticmethod/classmethod. 1019 if isinstance(val, staticmethod): 1020 val = getattr(obj, valname) 1021 if isinstance(val, classmethod): 1022 val = getattr(obj, valname).__func__ 1023 1024 # Recurse to methods, properties, and nested classes. 1025 if ((inspect.isroutine(val) or inspect.isclass(val) or 1026 isinstance(val, property)) and 1027 self._from_module(module, val)): 1028 valname = '%s.%s' % (name, valname) 1029 self._find(tests, val, valname, module, source_lines, 1030 globs, seen) 1031 1032 def _get_test(self, obj, name, module, globs, source_lines): 1033 """ 1034 Return a DocTest for the given object, if it defines a docstring; 1035 otherwise, return None. 1036 """ 1037 # Extract the object's docstring. If it doesn't have one, 1038 # then return None (no test for this object). 1039 if isinstance(obj, str): 1040 docstring = obj 1041 else: 1042 try: 1043 if obj.__doc__ is None: 1044 docstring = '' 1045 else: 1046 docstring = obj.__doc__ 1047 if not isinstance(docstring, str): 1048 docstring = str(docstring) 1049 except (TypeError, AttributeError): 1050 docstring = '' 1051 1052 # Find the docstring's location in the file. 1053 lineno = self._find_lineno(obj, source_lines) 1054 1055 # Don't bother if the docstring is empty. 1056 if self._exclude_empty and not docstring: 1057 return None 1058 1059 # Return a DocTest for this object. 1060 if module is None: 1061 filename = None 1062 else: 1063 filename = getattr(module, '__file__', module.__name__) 1064 if filename[-4:] == ".pyc": 1065 filename = filename[:-1] 1066 return self._parser.get_doctest(docstring, globs, name, 1067 filename, lineno) 1068 1069 def _find_lineno(self, obj, source_lines): 1070 """ 1071 Return a line number of the given object's docstring. Note: 1072 this method assumes that the object has a docstring. 1073 """ 1074 lineno = None 1075 1076 # Find the line number for modules. 1077 if inspect.ismodule(obj): 1078 lineno = 0 1079 1080 # Find the line number for classes. 1081 # Note: this could be fooled if a class is defined multiple 1082 # times in a single file. 1083 if inspect.isclass(obj): 1084 if source_lines is None: 1085 return None 1086 pat = re.compile(r'^\s*class\s*%s\b' % 1087 getattr(obj, '__name__', '-')) 1088 for i, line in enumerate(source_lines): 1089 if pat.match(line): 1090 lineno = i 1091 break 1092 1093 # Find the line number for functions & methods. 1094 if inspect.ismethod(obj): obj = obj.__func__ 1095 if inspect.isfunction(obj): obj = obj.__code__ 1096 if inspect.istraceback(obj): obj = obj.tb_frame 1097 if inspect.isframe(obj): obj = obj.f_code 1098 if inspect.iscode(obj): 1099 lineno = getattr(obj, 'co_firstlineno', None)-1 1100 1101 # Find the line number where the docstring starts. Assume 1102 # that it's the first line that begins with a quote mark. 1103 # Note: this could be fooled by a multiline function 1104 # signature, where a continuation line begins with a quote 1105 # mark. 1106 if lineno is not None: 1107 if source_lines is None: 1108 return lineno+1 1109 pat = re.compile(r'(^|.*:)\s*\w*("|\')') 1110 for lineno in range(lineno, len(source_lines)): 1111 if pat.match(source_lines[lineno]): 1112 return lineno 1113 1114 # We couldn't find the line number. 1115 return None 1116 1117 ###################################################################### 1118 ## 5. DocTest Runner 1119 ###################################################################### 1120 1121 class DocTestRunner: 1122 """ 1123 A class used to run DocTest test cases, and accumulate statistics. 1124 The `run` method is used to process a single DocTest case. It 1125 returns a tuple `(f, t)`, where `t` is the number of test cases 1126 tried, and `f` is the number of test cases that failed. 1127 1128 >>> tests = DocTestFinder().find(_TestClass) 1129 >>> runner = DocTestRunner(verbose=False) 1130 >>> tests.sort(key = lambda test: test.name) 1131 >>> for test in tests: 1132 ... print(test.name, '->', runner.run(test)) 1133 _TestClass -> TestResults(failed=0, attempted=2) 1134 _TestClass.__init__ -> TestResults(failed=0, attempted=2) 1135 _TestClass.get -> TestResults(failed=0, attempted=2) 1136 _TestClass.square -> TestResults(failed=0, attempted=1) 1137 1138 The `summarize` method prints a summary of all the test cases that 1139 have been run by the runner, and returns an aggregated `(f, t)` 1140 tuple: 1141 1142 >>> runner.summarize(verbose=1) 1143 4 items passed all tests: 1144 2 tests in _TestClass 1145 2 tests in _TestClass.__init__ 1146 2 tests in _TestClass.get 1147 1 tests in _TestClass.square 1148 7 tests in 4 items. 1149 7 passed and 0 failed. 1150 Test passed. 1151 TestResults(failed=0, attempted=7) 1152 1153 The aggregated number of tried examples and failed examples is 1154 also available via the `tries` and `failures` attributes: 1155 1156 >>> runner.tries 1157 7 1158 >>> runner.failures 1159 0 1160 1161 The comparison between expected outputs and actual outputs is done 1162 by an `OutputChecker`. This comparison may be customized with a 1163 number of option flags; see the documentation for `testmod` for 1164 more information. If the option flags are insufficient, then the 1165 comparison may also be customized by passing a subclass of 1166 `OutputChecker` to the constructor. 1167 1168 The test runner's display output can be controlled in two ways. 1169 First, an output function (`out) can be passed to 1170 `TestRunner.run`; this function will be called with strings that 1171 should be displayed. It defaults to `sys.stdout.write`. If 1172 capturing the output is not sufficient, then the display output 1173 can be also customized by subclassing DocTestRunner, and 1174 overriding the methods `report_start`, `report_success`, 1175 `report_unexpected_exception`, and `report_failure`. 1176 """ 1177 # This divider string is used to separate failure messages, and to 1178 # separate sections of the summary. 1179 DIVIDER = "*" * 70 1180 1181 def __init__(self, checker=None, verbose=None, optionflags=0): 1182 """ 1183 Create a new test runner. 1184 1185 Optional keyword arg `checker` is the `OutputChecker` that 1186 should be used to compare the expected outputs and actual 1187 outputs of doctest examples. 1188 1189 Optional keyword arg 'verbose' prints lots of stuff if true, 1190 only failures if false; by default, it's true iff '-v' is in 1191 sys.argv. 1192 1193 Optional argument `optionflags` can be used to control how the 1194 test runner compares expected output to actual output, and how 1195 it displays failures. See the documentation for `testmod` for 1196 more information. 1197 """ 1198 self._checker = checker or OutputChecker() 1199 if verbose is None: 1200 verbose = '-v' in sys.argv 1201 self._verbose = verbose 1202 self.optionflags = optionflags 1203 self.original_optionflags = optionflags 1204 1205 # Keep track of the examples we've run. 1206 self.tries = 0 1207 self.failures = 0 1208 self._name2ft = {} 1209 1210 # Create a fake output target for capturing doctest output. 1211 self._fakeout = _SpoofOut() 1212 1213 #///////////////////////////////////////////////////////////////// 1214 # Reporting methods 1215 #///////////////////////////////////////////////////////////////// 1216 1217 def report_start(self, out, test, example): 1218 """ 1219 Report that the test runner is about to process the given 1220 example. (Only displays a message if verbose=True) 1221 """ 1222 if self._verbose: 1223 if example.want: 1224 out('Trying:\n' + _indent(example.source) + 1225 'Expecting:\n' + _indent(example.want)) 1226 else: 1227 out('Trying:\n' + _indent(example.source) + 1228 'Expecting nothing\n') 1229 1230 def report_success(self, out, test, example, got): 1231 """ 1232 Report that the given example ran successfully. (Only 1233 displays a message if verbose=True) 1234 """ 1235 if self._verbose: 1236 out("ok\n") 1237 1238 def report_failure(self, out, test, example, got): 1239 """ 1240 Report that the given example failed. 1241 """ 1242 out(self._failure_header(test, example) + 1243 self._checker.output_difference(example, got, self.optionflags)) 1244 1245 def report_unexpected_exception(self, out, test, example, exc_info): 1246 """ 1247 Report that the given example raised an unexpected exception. 1248 """ 1249 out(self._failure_header(test, example) + 1250 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) 1251 1252 def _failure_header(self, test, example): 1253 out = [self.DIVIDER] 1254 if test.filename: 1255 if test.lineno is not None and example.lineno is not None: 1256 lineno = test.lineno + example.lineno + 1 1257 else: 1258 lineno = '?' 1259 out.append('File "%s", line %s, in %s' % 1260 (test.filename, lineno, test.name)) 1261 else: 1262 out.append('Line %s, in %s' % (example.lineno+1, test.name)) 1263 out.append('Failed example:') 1264 source = example.source 1265 out.append(_indent(source)) 1266 return '\n'.join(out) 1267 1268 #///////////////////////////////////////////////////////////////// 1269 # DocTest Running 1270 #///////////////////////////////////////////////////////////////// 1271 1272 def __run(self, test, compileflags, out): 1273 """ 1274 Run the examples in `test`. Write the outcome of each example 1275 with one of the `DocTestRunner.report_*` methods, using the 1276 writer function `out`. `compileflags` is the set of compiler 1277 flags that should be used to execute examples. Return a tuple 1278 `(f, t)`, where `t` is the number of examples tried, and `f` 1279 is the number of examples that failed. The examples are run 1280 in the namespace `test.globs`. 1281 """ 1282 # Keep track of the number of failures and tries. 1283 failures = tries = 0 1284 1285 # Save the option flags (since option directives can be used 1286 # to modify them). 1287 original_optionflags = self.optionflags 1288 1289 SUCCESS, FAILURE, BOOM = range(3) # `outcome` state 1290 1291 check = self._checker.check_output 1292 1293 # Process each example. 1294 for examplenum, example in enumerate(test.examples): 1295 1296 # If REPORT_ONLY_FIRST_FAILURE is set, then suppress 1297 # reporting after the first failure. 1298 quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and 1299 failures > 0) 1300 1301 # Merge in the example's options. 1302 self.optionflags = original_optionflags 1303 if example.options: 1304 for (optionflag, val) in example.options.items(): 1305 if val: 1306 self.optionflags |= optionflag 1307 else: 1308 self.optionflags &= ~optionflag 1309 1310 # If 'SKIP' is set, then skip this example. 1311 if self.optionflags & SKIP: 1312 continue 1313 1314 # Record that we started this example. 1315 tries += 1 1316 if not quiet: 1317 self.report_start(out, test, example) 1318 1319 # Use a special filename for compile(), so we can retrieve 1320 # the source code during interactive debugging (see 1321 # __patched_linecache_getlines). 1322 filename = '<doctest %s[%d]>' % (test.name, examplenum) 1323 1324 # Run the example in the given context (globs), and record 1325 # any exception that gets raised. (But don't intercept 1326 # keyboard interrupts.) 1327 try: 1328 # Don't blink! This is where the user's code gets run. 1329 exec(compile(example.source, filename, "single", 1330 compileflags, 1), test.globs) 1331 self.debugger.set_continue() # ==== Example Finished ==== 1332 exception = None 1333 except KeyboardInterrupt: 1334 raise 1335 except: 1336 exception = sys.exc_info() 1337 self.debugger.set_continue() # ==== Example Finished ==== 1338 1339 got = self._fakeout.getvalue() # the actual output 1340 self._fakeout.truncate(0) 1341 outcome = FAILURE # guilty until proved innocent or insane 1342 1343 # If the example executed without raising any exceptions, 1344 # verify its output. 1345 if exception is None: 1346 if check(example.want, got, self.optionflags): 1347 outcome = SUCCESS 1348 1349 # The example raised an exception: check if it was expected. 1350 else: 1351 exc_msg = traceback.format_exception_only(*exception[:2])[-1] 1352 if not quiet: 1353 got += _exception_traceback(exception) 1354 1355 # If `example.exc_msg` is None, then we weren't expecting 1356 # an exception. 1357 if example.exc_msg is None: 1358 outcome = BOOM 1359 1360 # We expected an exception: see whether it matches. 1361 elif check(example.exc_msg, exc_msg, self.optionflags): 1362 outcome = SUCCESS 1363 1364 # Another chance if they didn't care about the detail. 1365 elif self.optionflags & IGNORE_EXCEPTION_DETAIL: 1366 if check(_strip_exception_details(example.exc_msg), 1367 _strip_exception_details(exc_msg), 1368 self.optionflags): 1369 outcome = SUCCESS 1370 1371 # Report the outcome. 1372 if outcome is SUCCESS: 1373 if not quiet: 1374 self.report_success(out, test, example, got) 1375 elif outcome is FAILURE: 1376 if not quiet: 1377 self.report_failure(out, test, example, got) 1378 failures += 1 1379 elif outcome is BOOM: 1380 if not quiet: 1381 self.report_unexpected_exception(out, test, example, 1382 exception) 1383 failures += 1 1384 else: 1385 assert False, ("unknown outcome", outcome) 1386 1387 if failures and self.optionflags & FAIL_FAST: 1388 break 1389 1390 # Restore the option flags (in case they were modified) 1391 self.optionflags = original_optionflags 1392 1393 # Record and return the number of failures and tries. 1394 self.__record_outcome(test, failures, tries) 1395 return TestResults(failures, tries) 1396 1397 def __record_outcome(self, test, f, t): 1398 """ 1399 Record the fact that the given DocTest (`test`) generated `f` 1400 failures out of `t` tried examples. 1401 """ 1402 f2, t2 = self._name2ft.get(test.name, (0,0)) 1403 self._name2ft[test.name] = (f+f2, t+t2) 1404 self.failures += f 1405 self.tries += t 1406 1407 __LINECACHE_FILENAME_RE = re.compile(r'<doctest ' 1408 r'(?P<name>.+)' 1409 r'\[(?P<examplenum>\d+)\]>$') 1410 def __patched_linecache_getlines(self, filename, module_globals=None): 1411 m = self.__LINECACHE_FILENAME_RE.match(filename) 1412 if m and m.group('name') == self.test.name: 1413 example = self.test.examples[int(m.group('examplenum'))] 1414 return example.source.splitlines(keepends=True) 1415 else: 1416 return self.save_linecache_getlines(filename, module_globals) 1417 1418 def run(self, test, compileflags=None, out=None, clear_globs=True): 1419 """ 1420 Run the examples in `test`, and display the results using the 1421 writer function `out`. 1422 1423 The examples are run in the namespace `test.globs`. If 1424 `clear_globs` is true (the default), then this namespace will 1425 be cleared after the test runs, to help with garbage 1426 collection. If you would like to examine the namespace after 1427 the test completes, then use `clear_globs=False`. 1428 1429 `compileflags` gives the set of flags that should be used by 1430 the Python compiler when running the examples. If not 1431 specified, then it will default to the set of future-import 1432 flags that apply to `globs`. 1433 1434 The output of each example is checked using 1435 `DocTestRunner.check_output`, and the results are formatted by 1436 the `DocTestRunner.report_*` methods. 1437 """ 1438 self.test = test 1439 1440 if compileflags is None: 1441 compileflags = _extract_future_flags(test.globs) 1442 1443 save_stdout = sys.stdout 1444 if out is None: 1445 encoding = save_stdout.encoding 1446 if encoding is None or encoding.lower() == 'utf-8': 1447 out = save_stdout.write 1448 else: 1449 # Use backslashreplace error handling on write 1450 def out(s): 1451 s = str(s.encode(encoding, 'backslashreplace'), encoding) 1452 save_stdout.write(s) 1453 sys.stdout = self._fakeout 1454 1455 # Patch pdb.set_trace to restore sys.stdout during interactive 1456 # debugging (so it's not still redirected to self._fakeout). 1457 # Note that the interactive output will go to *our* 1458 # save_stdout, even if that's not the real sys.stdout; this 1459 # allows us to write test cases for the set_trace behavior. 1460 save_trace = sys.gettrace() 1461 save_set_trace = pdb.set_trace 1462 self.debugger = _OutputRedirectingPdb(save_stdout) 1463 self.debugger.reset() 1464 pdb.set_trace = self.debugger.set_trace 1465 1466 # Patch linecache.getlines, so we can see the example's source 1467 # when we're inside the debugger. 1468 self.save_linecache_getlines = linecache.getlines 1469 linecache.getlines = self.__patched_linecache_getlines 1470 1471 # Make sure sys.displayhook just prints the value to stdout 1472 save_displayhook = sys.displayhook 1473 sys.displayhook = sys.__displayhook__ 1474 1475 try: 1476 return self.__run(test, compileflags, out) 1477 finally: 1478 sys.stdout = save_stdout 1479 pdb.set_trace = save_set_trace 1480 sys.settrace(save_trace) 1481 linecache.getlines = self.save_linecache_getlines 1482 sys.displayhook = save_displayhook 1483 if clear_globs: 1484 test.globs.clear() 1485 import builtins 1486 builtins._ = None 1487 1488 #///////////////////////////////////////////////////////////////// 1489 # Summarization 1490 #///////////////////////////////////////////////////////////////// 1491 def summarize(self, verbose=None): 1492 """ 1493 Print a summary of all the test cases that have been run by 1494 this DocTestRunner, and return a tuple `(f, t)`, where `f` is 1495 the total number of failed examples, and `t` is the total 1496 number of tried examples. 1497 1498 The optional `verbose` argument controls how detailed the 1499 summary is. If the verbosity is not specified, then the 1500 DocTestRunner's verbosity is used. 1501 """ 1502 if verbose is None: 1503 verbose = self._verbose 1504 notests = [] 1505 passed = [] 1506 failed = [] 1507 totalt = totalf = 0 1508 for x in self._name2ft.items(): 1509 name, (f, t) = x 1510 assert f <= t 1511 totalt += t 1512 totalf += f 1513 if t == 0: 1514 notests.append(name) 1515 elif f == 0: 1516 passed.append( (name, t) ) 1517 else: 1518 failed.append(x) 1519 if verbose: 1520 if notests: 1521 print(len(notests), "items had no tests:") 1522 notests.sort() 1523 for thing in notests: 1524 print(" ", thing) 1525 if passed: 1526 print(len(passed), "items passed all tests:") 1527 passed.sort() 1528 for thing, count in passed: 1529 print(" %3d tests in %s" % (count, thing)) 1530 if failed: 1531 print(self.DIVIDER) 1532 print(len(failed), "items had failures:") 1533 failed.sort() 1534 for thing, (f, t) in failed: 1535 print(" %3d of %3d in %s" % (f, t, thing)) 1536 if verbose: 1537 print(totalt, "tests in", len(self._name2ft), "items.") 1538 print(totalt - totalf, "passed and", totalf, "failed.") 1539 if totalf: 1540 print("***Test Failed***", totalf, "failures.") 1541 elif verbose: 1542 print("Test passed.") 1543 return TestResults(totalf, totalt) 1544 1545 #///////////////////////////////////////////////////////////////// 1546 # Backward compatibility cruft to maintain doctest.master. 1547 #///////////////////////////////////////////////////////////////// 1548 def merge(self, other): 1549 d = self._name2ft 1550 for name, (f, t) in other._name2ft.items(): 1551 if name in d: 1552 # Don't print here by default, since doing 1553 # so breaks some of the buildbots 1554 #print("*** DocTestRunner.merge: '" + name + "' in both" \ 1555 # " testers; summing outcomes.") 1556 f2, t2 = d[name] 1557 f = f + f2 1558 t = t + t2 1559 d[name] = f, t 1560 1561 class OutputChecker: 1562 """ 1563 A class used to check the whether the actual output from a doctest 1564 example matches the expected output. `OutputChecker` defines two 1565 methods: `check_output`, which compares a given pair of outputs, 1566 and returns true if they match; and `output_difference`, which 1567 returns a string describing the differences between two outputs. 1568 """ 1569 def _toAscii(self, s): 1570 """ 1571 Convert string to hex-escaped ASCII string. 1572 """ 1573 return str(s.encode('ASCII', 'backslashreplace'), "ASCII") 1574 1575 def check_output(self, want, got, optionflags): 1576 """ 1577 Return True iff the actual output from an example (`got`) 1578 matches the expected output (`want`). These strings are 1579 always considered to match if they are identical; but 1580 depending on what option flags the test runner is using, 1581 several non-exact match types are also possible. See the 1582 documentation for `TestRunner` for more information about 1583 option flags. 1584 """ 1585 1586 # If `want` contains hex-escaped character such as "\u1234", 1587 # then `want` is a string of six characters(e.g. [\,u,1,2,3,4]). 1588 # On the other hand, `got` could be another sequence of 1589 # characters such as [\u1234], so `want` and `got` should 1590 # be folded to hex-escaped ASCII string to compare. 1591 got = self._toAscii(got) 1592 want = self._toAscii(want) 1593 1594 # Handle the common case first, for efficiency: 1595 # if they're string-identical, always return true. 1596 if got == want: 1597 return True 1598 1599 # The values True and False replaced 1 and 0 as the return 1600 # value for boolean comparisons in Python 2.3. 1601 if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): 1602 if (got,want) == ("True\n", "1\n"): 1603 return True 1604 if (got,want) == ("False\n", "0\n"): 1605 return True 1606 1607 # <BLANKLINE> can be used as a special sequence to signify a 1608 # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. 1609 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1610 # Replace <BLANKLINE> in want with a blank line. 1611 want = re.sub(r'(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), 1612 '', want) 1613 # If a line in got contains only spaces, then remove the 1614 # spaces. 1615 got = re.sub(r'(?m)^\s*?$', '', got) 1616 if got == want: 1617 return True 1618 1619 # This flag causes doctest to ignore any differences in the 1620 # contents of whitespace strings. Note that this can be used 1621 # in conjunction with the ELLIPSIS flag. 1622 if optionflags & NORMALIZE_WHITESPACE: 1623 got = ' '.join(got.split()) 1624 want = ' '.join(want.split()) 1625 if got == want: 1626 return True 1627 1628 # The ELLIPSIS flag says to let the sequence "..." in `want` 1629 # match any substring in `got`. 1630 if optionflags & ELLIPSIS: 1631 if _ellipsis_match(want, got): 1632 return True 1633 1634 # We didn't find any match; return false. 1635 return False 1636 1637 # Should we do a fancy diff? 1638 def _do_a_fancy_diff(self, want, got, optionflags): 1639 # Not unless they asked for a fancy diff. 1640 if not optionflags & (REPORT_UDIFF | 1641 REPORT_CDIFF | 1642 REPORT_NDIFF): 1643 return False 1644 1645 # If expected output uses ellipsis, a meaningful fancy diff is 1646 # too hard ... or maybe not. In two real-life failures Tim saw, 1647 # a diff was a major help anyway, so this is commented out. 1648 # [todo] _ellipsis_match() knows which pieces do and don't match, 1649 # and could be the basis for a kick-ass diff in this case. 1650 ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: 1651 ## return False 1652 1653 # ndiff does intraline difference marking, so can be useful even 1654 # for 1-line differences. 1655 if optionflags & REPORT_NDIFF: 1656 return True 1657 1658 # The other diff types need at least a few lines to be helpful. 1659 return want.count('\n') > 2 and got.count('\n') > 2 1660 1661 def output_difference(self, example, got, optionflags): 1662 """ 1663 Return a string describing the differences between the 1664 expected output for a given example (`example`) and the actual 1665 output (`got`). `optionflags` is the set of option flags used 1666 to compare `want` and `got`. 1667 """ 1668 want = example.want 1669 # If <BLANKLINE>s are being used, then replace blank lines 1670 # with <BLANKLINE> in the actual output string. 1671 if not (optionflags & DONT_ACCEPT_BLANKLINE): 1672 got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) 1673 1674 # Check if we should use diff. 1675 if self._do_a_fancy_diff(want, got, optionflags): 1676 # Split want & got into lines. 1677 want_lines = want.splitlines(keepends=True) 1678 got_lines = got.splitlines(keepends=True) 1679 # Use difflib to find their differences. 1680 if optionflags & REPORT_UDIFF: 1681 diff = difflib.unified_diff(want_lines, got_lines, n=2) 1682 diff = list(diff)[2:] # strip the diff header 1683 kind = 'unified diff with -expected +actual' 1684 elif optionflags & REPORT_CDIFF: 1685 diff = difflib.context_diff(want_lines, got_lines, n=2) 1686 diff = list(diff)[2:] # strip the diff header 1687 kind = 'context diff with expected followed by actual' 1688 elif optionflags & REPORT_NDIFF: 1689 engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) 1690 diff = list(engine.compare(want_lines, got_lines)) 1691 kind = 'ndiff with -expected +actual' 1692 else: 1693 assert 0, 'Bad diff option' 1694 # Remove trailing whitespace on diff output. 1695 diff = [line.rstrip() + '\n' for line in diff] 1696 return 'Differences (%s):\n' % kind + _indent(''.join(diff)) 1697 1698 # If we're not using diff, then simply list the expected 1699 # output followed by the actual output. 1700 if want and got: 1701 return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) 1702 elif want: 1703 return 'Expected:\n%sGot nothing\n' % _indent(want) 1704 elif got: 1705 return 'Expected nothing\nGot:\n%s' % _indent(got) 1706 else: 1707 return 'Expected nothing\nGot nothing\n' 1708 1709 class DocTestFailure(Exception): 1710 """A DocTest example has failed in debugging mode. 1711 1712 The exception instance has variables: 1713 1714 - test: the DocTest object being run 1715 1716 - example: the Example object that failed 1717 1718 - got: the actual output 1719 """ 1720 def __init__(self, test, example, got): 1721 self.test = test 1722 self.example = example 1723 self.got = got 1724 1725 def __str__(self): 1726 return str(self.test) 1727 1728 class UnexpectedException(Exception): 1729 """A DocTest example has encountered an unexpected exception 1730 1731 The exception instance has variables: 1732 1733 - test: the DocTest object being run 1734 1735 - example: the Example object that failed 1736 1737 - exc_info: the exception info 1738 """ 1739 def __init__(self, test, example, exc_info): 1740 self.test = test 1741 self.example = example 1742 self.exc_info = exc_info 1743 1744 def __str__(self): 1745 return str(self.test) 1746 1747 class DebugRunner(DocTestRunner): 1748 r"""Run doc tests but raise an exception as soon as there is a failure. 1749 1750 If an unexpected exception occurs, an UnexpectedException is raised. 1751 It contains the test, the example, and the original exception: 1752 1753 >>> runner = DebugRunner(verbose=False) 1754 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 1755 ... {}, 'foo', 'foo.py', 0) 1756 >>> try: 1757 ... runner.run(test) 1758 ... except UnexpectedException as f: 1759 ... failure = f 1760 1761 >>> failure.test is test 1762 True 1763 1764 >>> failure.example.want 1765 '42\n' 1766 1767 >>> exc_info = failure.exc_info 1768 >>> raise exc_info[1] # Already has the traceback 1769 Traceback (most recent call last): 1770 ... 1771 KeyError 1772 1773 We wrap the original exception to give the calling application 1774 access to the test and example information. 1775 1776 If the output doesn't match, then a DocTestFailure is raised: 1777 1778 >>> test = DocTestParser().get_doctest(''' 1779 ... >>> x = 1 1780 ... >>> x 1781 ... 2 1782 ... ''', {}, 'foo', 'foo.py', 0) 1783 1784 >>> try: 1785 ... runner.run(test) 1786 ... except DocTestFailure as f: 1787 ... failure = f 1788 1789 DocTestFailure objects provide access to the test: 1790 1791 >>> failure.test is test 1792 True 1793 1794 As well as to the example: 1795 1796 >>> failure.example.want 1797 '2\n' 1798 1799 and the actual output: 1800 1801 >>> failure.got 1802 '1\n' 1803 1804 If a failure or error occurs, the globals are left intact: 1805 1806 >>> del test.globs['__builtins__'] 1807 >>> test.globs 1808 {'x': 1} 1809 1810 >>> test = DocTestParser().get_doctest(''' 1811 ... >>> x = 2 1812 ... >>> raise KeyError 1813 ... ''', {}, 'foo', 'foo.py', 0) 1814 1815 >>> runner.run(test) 1816 Traceback (most recent call last): 1817 ... 1818 doctest.UnexpectedException: <DocTest foo from foo.py:0 (2 examples)> 1819 1820 >>> del test.globs['__builtins__'] 1821 >>> test.globs 1822 {'x': 2} 1823 1824 But the globals are cleared if there is no error: 1825 1826 >>> test = DocTestParser().get_doctest(''' 1827 ... >>> x = 2 1828 ... ''', {}, 'foo', 'foo.py', 0) 1829 1830 >>> runner.run(test) 1831 TestResults(failed=0, attempted=1) 1832 1833 >>> test.globs 1834 {} 1835 1836 """ 1837 1838 def run(self, test, compileflags=None, out=None, clear_globs=True): 1839 r = DocTestRunner.run(self, test, compileflags, out, False) 1840 if clear_globs: 1841 test.globs.clear() 1842 return r 1843 1844 def report_unexpected_exception(self, out, test, example, exc_info): 1845 raise UnexpectedException(test, example, exc_info) 1846 1847 def report_failure(self, out, test, example, got): 1848 raise DocTestFailure(test, example, got) 1849 1850 ###################################################################### 1851 ## 6. Test Functions 1852 ###################################################################### 1853 # These should be backwards compatible. 1854 1855 # For backward compatibility, a global instance of a DocTestRunner 1856 # class, updated by testmod. 1857 master = None 1858 1859 def testmod(m=None, name=None, globs=None, verbose=None, 1860 report=True, optionflags=0, extraglobs=None, 1861 raise_on_error=False, exclude_empty=False): 1862 """m=None, name=None, globs=None, verbose=None, report=True, 1863 optionflags=0, extraglobs=None, raise_on_error=False, 1864 exclude_empty=False 1865 1866 Test examples in docstrings in functions and classes reachable 1867 from module m (or the current module if m is not supplied), starting 1868 with m.__doc__. 1869 1870 Also test examples reachable from dict m.__test__ if it exists and is 1871 not None. m.__test__ maps names to functions, classes and strings; 1872 function and class docstrings are tested even if the name is private; 1873 strings are tested directly, as if they were docstrings. 1874 1875 Return (#failures, #tests). 1876 1877 See help(doctest) for an overview. 1878 1879 Optional keyword arg "name" gives the name of the module; by default 1880 use m.__name__. 1881 1882 Optional keyword arg "globs" gives a dict to be used as the globals 1883 when executing examples; by default, use m.__dict__. A copy of this 1884 dict is actually used for each docstring, so that each docstring's 1885 examples start with a clean slate. 1886 1887 Optional keyword arg "extraglobs" gives a dictionary that should be 1888 merged into the globals that are used to execute examples. By 1889 default, no extra globals are used. This is new in 2.4. 1890 1891 Optional keyword arg "verbose" prints lots of stuff if true, prints 1892 only failures if false; by default, it's true iff "-v" is in sys.argv. 1893 1894 Optional keyword arg "report" prints a summary at the end when true, 1895 else prints nothing at the end. In verbose mode, the summary is 1896 detailed, else very brief (in fact, empty if all tests passed). 1897 1898 Optional keyword arg "optionflags" or's together module constants, 1899 and defaults to 0. This is new in 2.3. Possible values (see the 1900 docs for details): 1901 1902 DONT_ACCEPT_TRUE_FOR_1 1903 DONT_ACCEPT_BLANKLINE 1904 NORMALIZE_WHITESPACE 1905 ELLIPSIS 1906 SKIP 1907 IGNORE_EXCEPTION_DETAIL 1908 REPORT_UDIFF 1909 REPORT_CDIFF 1910 REPORT_NDIFF 1911 REPORT_ONLY_FIRST_FAILURE 1912 1913 Optional keyword arg "raise_on_error" raises an exception on the 1914 first unexpected exception or failure. This allows failures to be 1915 post-mortem debugged. 1916 1917 Advanced tomfoolery: testmod runs methods of a local instance of 1918 class doctest.Tester, then merges the results into (or creates) 1919 global Tester instance doctest.master. Methods of doctest.master 1920 can be called directly too, if you want to do something unusual. 1921 Passing report=0 to testmod is especially useful then, to delay 1922 displaying a summary. Invoke doctest.master.summarize(verbose) 1923 when you're done fiddling. 1924 """ 1925 global master 1926 1927 # If no module was given, then use __main__. 1928 if m is None: 1929 # DWA - m will still be None if this wasn't invoked from the command 1930 # line, in which case the following TypeError is about as good an error 1931 # as we should expect 1932 m = sys.modules.get('__main__') 1933 1934 # Check that we were actually given a module. 1935 if not inspect.ismodule(m): 1936 raise TypeError("testmod: module required; %r" % (m,)) 1937 1938 # If no name was given, then use the module's name. 1939 if name is None: 1940 name = m.__name__ 1941 1942 # Find, parse, and run all tests in the given module. 1943 finder = DocTestFinder(exclude_empty=exclude_empty) 1944 1945 if raise_on_error: 1946 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 1947 else: 1948 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 1949 1950 for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): 1951 runner.run(test) 1952 1953 if report: 1954 runner.summarize() 1955 1956 if master is None: 1957 master = runner 1958 else: 1959 master.merge(runner) 1960 1961 return TestResults(runner.failures, runner.tries) 1962 1963 def testfile(filename, module_relative=True, name=None, package=None, 1964 globs=None, verbose=None, report=True, optionflags=0, 1965 extraglobs=None, raise_on_error=False, parser=DocTestParser(), 1966 encoding=None): 1967 """ 1968 Test examples in the given file. Return (#failures, #tests). 1969 1970 Optional keyword arg "module_relative" specifies how filenames 1971 should be interpreted: 1972 1973 - If "module_relative" is True (the default), then "filename" 1974 specifies a module-relative path. By default, this path is 1975 relative to the calling module's directory; but if the 1976 "package" argument is specified, then it is relative to that 1977 package. To ensure os-independence, "filename" should use 1978 "/" characters to separate path segments, and should not 1979 be an absolute path (i.e., it may not begin with "/"). 1980 1981 - If "module_relative" is False, then "filename" specifies an 1982 os-specific path. The path may be absolute or relative (to 1983 the current working directory). 1984 1985 Optional keyword arg "name" gives the name of the test; by default 1986 use the file's basename. 1987 1988 Optional keyword argument "package" is a Python package or the 1989 name of a Python package whose directory should be used as the 1990 base directory for a module relative filename. If no package is 1991 specified, then the calling module's directory is used as the base 1992 directory for module relative filenames. It is an error to 1993 specify "package" if "module_relative" is False. 1994 1995 Optional keyword arg "globs" gives a dict to be used as the globals 1996 when executing examples; by default, use {}. A copy of this dict 1997 is actually used for each docstring, so that each docstring's 1998 examples start with a clean slate. 1999 2000 Optional keyword arg "extraglobs" gives a dictionary that should be 2001 merged into the globals that are used to execute examples. By 2002 default, no extra globals are used. 2003 2004 Optional keyword arg "verbose" prints lots of stuff if true, prints 2005 only failures if false; by default, it's true iff "-v" is in sys.argv. 2006 2007 Optional keyword arg "report" prints a summary at the end when true, 2008 else prints nothing at the end. In verbose mode, the summary is 2009 detailed, else very brief (in fact, empty if all tests passed). 2010 2011 Optional keyword arg "optionflags" or's together module constants, 2012 and defaults to 0. Possible values (see the docs for details): 2013 2014 DONT_ACCEPT_TRUE_FOR_1 2015 DONT_ACCEPT_BLANKLINE 2016 NORMALIZE_WHITESPACE 2017 ELLIPSIS 2018 SKIP 2019 IGNORE_EXCEPTION_DETAIL 2020 REPORT_UDIFF 2021 REPORT_CDIFF 2022 REPORT_NDIFF 2023 REPORT_ONLY_FIRST_FAILURE 2024 2025 Optional keyword arg "raise_on_error" raises an exception on the 2026 first unexpected exception or failure. This allows failures to be 2027 post-mortem debugged. 2028 2029 Optional keyword arg "parser" specifies a DocTestParser (or 2030 subclass) that should be used to extract tests from the files. 2031 2032 Optional keyword arg "encoding" specifies an encoding that should 2033 be used to convert the file to unicode. 2034 2035 Advanced tomfoolery: testmod runs methods of a local instance of 2036 class doctest.Tester, then merges the results into (or creates) 2037 global Tester instance doctest.master. Methods of doctest.master 2038 can be called directly too, if you want to do something unusual. 2039 Passing report=0 to testmod is especially useful then, to delay 2040 displaying a summary. Invoke doctest.master.summarize(verbose) 2041 when you're done fiddling. 2042 """ 2043 global master 2044 2045 if package and not module_relative: 2046 raise ValueError("Package may only be specified for module-" 2047 "relative paths.") 2048 2049 # Relativize the path 2050 text, filename = _load_testfile(filename, package, module_relative, 2051 encoding or "utf-8") 2052 2053 # If no name was given, then use the file's name. 2054 if name is None: 2055 name = os.path.basename(filename) 2056 2057 # Assemble the globals. 2058 if globs is None: 2059 globs = {} 2060 else: 2061 globs = globs.copy() 2062 if extraglobs is not None: 2063 globs.update(extraglobs) 2064 if '__name__' not in globs: 2065 globs['__name__'] = '__main__' 2066 2067 if raise_on_error: 2068 runner = DebugRunner(verbose=verbose, optionflags=optionflags) 2069 else: 2070 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2071 2072 # Read the file, convert it to a test, and run it. 2073 test = parser.get_doctest(text, globs, name, filename, 0) 2074 runner.run(test) 2075 2076 if report: 2077 runner.summarize() 2078 2079 if master is None: 2080 master = runner 2081 else: 2082 master.merge(runner) 2083 2084 return TestResults(runner.failures, runner.tries) 2085 2086 def run_docstring_examples(f, globs, verbose=False, name="NoName", 2087 compileflags=None, optionflags=0): 2088 """ 2089 Test examples in the given object's docstring (`f`), using `globs` 2090 as globals. Optional argument `name` is used in failure messages. 2091 If the optional argument `verbose` is true, then generate output 2092 even if there are no failures. 2093 2094 `compileflags` gives the set of flags that should be used by the 2095 Python compiler when running the examples. If not specified, then 2096 it will default to the set of future-import flags that apply to 2097 `globs`. 2098 2099 Optional keyword arg `optionflags` specifies options for the 2100 testing and output. See the documentation for `testmod` for more 2101 information. 2102 """ 2103 # Find, parse, and run all tests in the given module. 2104 finder = DocTestFinder(verbose=verbose, recurse=False) 2105 runner = DocTestRunner(verbose=verbose, optionflags=optionflags) 2106 for test in finder.find(f, name, globs=globs): 2107 runner.run(test, compileflags=compileflags) 2108 2109 ###################################################################### 2110 ## 7. Unittest Support 2111 ###################################################################### 2112 2113 _unittest_reportflags = 0 2114 2115 def set_unittest_reportflags(flags): 2116 """Sets the unittest option flags. 2117 2118 The old flag is returned so that a runner could restore the old 2119 value if it wished to: 2120 2121 >>> import doctest 2122 >>> old = doctest._unittest_reportflags 2123 >>> doctest.set_unittest_reportflags(REPORT_NDIFF | 2124 ... REPORT_ONLY_FIRST_FAILURE) == old 2125 True 2126 2127 >>> doctest._unittest_reportflags == (REPORT_NDIFF | 2128 ... REPORT_ONLY_FIRST_FAILURE) 2129 True 2130 2131 Only reporting flags can be set: 2132 2133 >>> doctest.set_unittest_reportflags(ELLIPSIS) 2134 Traceback (most recent call last): 2135 ... 2136 ValueError: ('Only reporting flags allowed', 8) 2137 2138 >>> doctest.set_unittest_reportflags(old) == (REPORT_NDIFF | 2139 ... REPORT_ONLY_FIRST_FAILURE) 2140 True 2141 """ 2142 global _unittest_reportflags 2143 2144 if (flags & REPORTING_FLAGS) != flags: 2145 raise ValueError("Only reporting flags allowed", flags) 2146 old = _unittest_reportflags 2147 _unittest_reportflags = flags 2148 return old 2149 2150 2151 class DocTestCase(unittest.TestCase): 2152 2153 def __init__(self, test, optionflags=0, setUp=None, tearDown=None, 2154 checker=None): 2155 2156 unittest.TestCase.__init__(self) 2157 self._dt_optionflags = optionflags 2158 self._dt_checker = checker 2159 self._dt_test = test 2160 self._dt_setUp = setUp 2161 self._dt_tearDown = tearDown 2162 2163 def setUp(self): 2164 test = self._dt_test 2165 2166 if self._dt_setUp is not None: 2167 self._dt_setUp(test) 2168 2169 def tearDown(self): 2170 test = self._dt_test 2171 2172 if self._dt_tearDown is not None: 2173 self._dt_tearDown(test) 2174 2175 test.globs.clear() 2176 2177 def runTest(self): 2178 test = self._dt_test 2179 old = sys.stdout 2180 new = StringIO() 2181 optionflags = self._dt_optionflags 2182 2183 if not (optionflags & REPORTING_FLAGS): 2184 # The option flags don't include any reporting flags, 2185 # so add the default reporting flags 2186 optionflags |= _unittest_reportflags 2187 2188 runner = DocTestRunner(optionflags=optionflags, 2189 checker=self._dt_checker, verbose=False) 2190 2191 try: 2192 runner.DIVIDER = "-"*70 2193 failures, tries = runner.run( 2194 test, out=new.write, clear_globs=False) 2195 finally: 2196 sys.stdout = old 2197 2198 if failures: 2199 raise self.failureException(self.format_failure(new.getvalue())) 2200 2201 def format_failure(self, err): 2202 test = self._dt_test 2203 if test.lineno is None: 2204 lineno = 'unknown line number' 2205 else: 2206 lineno = '%s' % test.lineno 2207 lname = '.'.join(test.name.split('.')[-1:]) 2208 return ('Failed doctest test for %s\n' 2209 ' File "%s", line %s, in %s\n\n%s' 2210 % (test.name, test.filename, lineno, lname, err) 2211 ) 2212 2213 def debug(self): 2214 r"""Run the test case without results and without catching exceptions 2215 2216 The unit test framework includes a debug method on test cases 2217 and test suites to support post-mortem debugging. The test code 2218 is run in such a way that errors are not caught. This way a 2219 caller can catch the errors and initiate post-mortem debugging. 2220 2221 The DocTestCase provides a debug method that raises 2222 UnexpectedException errors if there is an unexpected 2223 exception: 2224 2225 >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', 2226 ... {}, 'foo', 'foo.py', 0) 2227 >>> case = DocTestCase(test) 2228 >>> try: 2229 ... case.debug() 2230 ... except UnexpectedException as f: 2231 ... failure = f 2232 2233 The UnexpectedException contains the test, the example, and 2234 the original exception: 2235 2236 >>> failure.test is test 2237 True 2238 2239 >>> failure.example.want 2240 '42\n' 2241 2242 >>> exc_info = failure.exc_info 2243 >>> raise exc_info[1] # Already has the traceback 2244 Traceback (most recent call last): 2245 ... 2246 KeyError 2247 2248 If the output doesn't match, then a DocTestFailure is raised: 2249 2250 >>> test = DocTestParser().get_doctest(''' 2251 ... >>> x = 1 2252 ... >>> x 2253 ... 2 2254 ... ''', {}, 'foo', 'foo.py', 0) 2255 >>> case = DocTestCase(test) 2256 2257 >>> try: 2258 ... case.debug() 2259 ... except DocTestFailure as f: 2260 ... failure = f 2261 2262 DocTestFailure objects provide access to the test: 2263 2264 >>> failure.test is test 2265 True 2266 2267 As well as to the example: 2268 2269 >>> failure.example.want 2270 '2\n' 2271 2272 and the actual output: 2273 2274 >>> failure.got 2275 '1\n' 2276 2277 """ 2278 2279 self.setUp() 2280 runner = DebugRunner(optionflags=self._dt_optionflags, 2281 checker=self._dt_checker, verbose=False) 2282 runner.run(self._dt_test, clear_globs=False) 2283 self.tearDown() 2284 2285 def id(self): 2286 return self._dt_test.name 2287 2288 def __eq__(self, other): 2289 if type(self) is not type(other): 2290 return NotImplemented 2291 2292 return self._dt_test == other._dt_test and \ 2293 self._dt_optionflags == other._dt_optionflags and \ 2294 self._dt_setUp == other._dt_setUp and \ 2295 self._dt_tearDown == other._dt_tearDown and \ 2296 self._dt_checker == other._dt_checker 2297 2298 def __hash__(self): 2299 return hash((self._dt_optionflags, self._dt_setUp, self._dt_tearDown, 2300 self._dt_checker)) 2301 2302 def __repr__(self): 2303 name = self._dt_test.name.split('.') 2304 return "%s (%s)" % (name[-1], '.'.join(name[:-1])) 2305 2306 __str__ = __repr__ 2307 2308 def shortDescription(self): 2309 return "Doctest: " + self._dt_test.name 2310 2311 class SkipDocTestCase(DocTestCase): 2312 def __init__(self, module): 2313 self.module = module 2314 DocTestCase.__init__(self, None) 2315 2316 def setUp(self): 2317 self.skipTest("DocTestSuite will not work with -O2 and above") 2318 2319 def test_skip(self): 2320 pass 2321 2322 def shortDescription(self): 2323 return "Skipping tests from %s" % self.module.__name__ 2324 2325 __str__ = shortDescription 2326 2327 2328 class _DocTestSuite(unittest.TestSuite): 2329 2330 def _removeTestAtIndex(self, index): 2331 pass 2332 2333 2334 def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, 2335 **options): 2336 """ 2337 Convert doctest tests for a module to a unittest test suite. 2338 2339 This converts each documentation string in a module that 2340 contains doctest tests to a unittest test case. If any of the 2341 tests in a doc string fail, then the test case fails. An exception 2342 is raised showing the name of the file containing the test and a 2343 (sometimes approximate) line number. 2344 2345 The `module` argument provides the module to be tested. The argument 2346 can be either a module or a module name. 2347 2348 If no argument is given, the calling module is used. 2349 2350 A number of options may be provided as keyword arguments: 2351 2352 setUp 2353 A set-up function. This is called before running the 2354 tests in each file. The setUp function will be passed a DocTest 2355 object. The setUp function can access the test globals as the 2356 globs attribute of the test passed. 2357 2358 tearDown 2359 A tear-down function. This is called after running the 2360 tests in each file. The tearDown function will be passed a DocTest 2361 object. The tearDown function can access the test globals as the 2362 globs attribute of the test passed. 2363 2364 globs 2365 A dictionary containing initial global variables for the tests. 2366 2367 optionflags 2368 A set of doctest option flags expressed as an integer. 2369 """ 2370 2371 if test_finder is None: 2372 test_finder = DocTestFinder() 2373 2374 module = _normalize_module(module) 2375 tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) 2376 2377 if not tests and sys.flags.optimize >=2: 2378 # Skip doctests when running with -O2 2379 suite = _DocTestSuite() 2380 suite.addTest(SkipDocTestCase(module)) 2381 return suite 2382 2383 tests.sort() 2384 suite = _DocTestSuite() 2385 2386 for test in tests: 2387 if len(test.examples) == 0: 2388 continue 2389 if not test.filename: 2390 filename = module.__file__ 2391 if filename[-4:] == ".pyc": 2392 filename = filename[:-1] 2393 test.filename = filename 2394 suite.addTest(DocTestCase(test, **options)) 2395 2396 return suite 2397 2398 class DocFileCase(DocTestCase): 2399 2400 def id(self): 2401 return '_'.join(self._dt_test.name.split('.')) 2402 2403 def __repr__(self): 2404 return self._dt_test.filename 2405 __str__ = __repr__ 2406 2407 def format_failure(self, err): 2408 return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' 2409 % (self._dt_test.name, self._dt_test.filename, err) 2410 ) 2411 2412 def DocFileTest(path, module_relative=True, package=None, 2413 globs=None, parser=DocTestParser(), 2414 encoding=None, **options): 2415 if globs is None: 2416 globs = {} 2417 else: 2418 globs = globs.copy() 2419 2420 if package and not module_relative: 2421 raise ValueError("Package may only be specified for module-" 2422 "relative paths.") 2423 2424 # Relativize the path. 2425 doc, path = _load_testfile(path, package, module_relative, 2426 encoding or "utf-8") 2427 2428 if "__file__" not in globs: 2429 globs["__file__"] = path 2430 2431 # Find the file and read it. 2432 name = os.path.basename(path) 2433 2434 # Convert it to a test, and wrap it in a DocFileCase. 2435 test = parser.get_doctest(doc, globs, name, path, 0) 2436 return DocFileCase(test, **options) 2437 2438 def DocFileSuite(*paths, **kw): 2439 """A unittest suite for one or more doctest files. 2440 2441 The path to each doctest file is given as a string; the 2442 interpretation of that string depends on the keyword argument 2443 "module_relative". 2444 2445 A number of options may be provided as keyword arguments: 2446 2447 module_relative 2448 If "module_relative" is True, then the given file paths are 2449 interpreted as os-independent module-relative paths. By 2450 default, these paths are relative to the calling module's 2451 directory; but if the "package" argument is specified, then 2452 they are relative to that package. To ensure os-independence, 2453 "filename" should use "/" characters to separate path 2454 segments, and may not be an absolute path (i.e., it may not 2455 begin with "/"). 2456 2457 If "module_relative" is False, then the given file paths are 2458 interpreted as os-specific paths. These paths may be absolute 2459 or relative (to the current working directory). 2460 2461 package 2462 A Python package or the name of a Python package whose directory 2463 should be used as the base directory for module relative paths. 2464 If "package" is not specified, then the calling module's 2465 directory is used as the base directory for module relative 2466 filenames. It is an error to specify "package" if 2467 "module_relative" is False. 2468 2469 setUp 2470 A set-up function. This is called before running the 2471 tests in each file. The setUp function will be passed a DocTest 2472 object. The setUp function can access the test globals as the 2473 globs attribute of the test passed. 2474 2475 tearDown 2476 A tear-down function. This is called after running the 2477 tests in each file. The tearDown function will be passed a DocTest 2478 object. The tearDown function can access the test globals as the 2479 globs attribute of the test passed. 2480 2481 globs 2482 A dictionary containing initial global variables for the tests. 2483 2484 optionflags 2485 A set of doctest option flags expressed as an integer. 2486 2487 parser 2488 A DocTestParser (or subclass) that should be used to extract 2489 tests from the files. 2490 2491 encoding 2492 An encoding that will be used to convert the files to unicode. 2493 """ 2494 suite = _DocTestSuite() 2495 2496 # We do this here so that _normalize_module is called at the right 2497 # level. If it were called in DocFileTest, then this function 2498 # would be the caller and we might guess the package incorrectly. 2499 if kw.get('module_relative', True): 2500 kw['package'] = _normalize_module(kw.get('package')) 2501 2502 for path in paths: 2503 suite.addTest(DocFileTest(path, **kw)) 2504 2505 return suite 2506 2507 ###################################################################### 2508 ## 8. Debugging Support 2509 ###################################################################### 2510 2511 def script_from_examples(s): 2512 r"""Extract script from text with examples. 2513 2514 Converts text with examples to a Python script. Example input is 2515 converted to regular code. Example output and all other words 2516 are converted to comments: 2517 2518 >>> text = ''' 2519 ... Here are examples of simple math. 2520 ... 2521 ... Python has super accurate integer addition 2522 ... 2523 ... >>> 2 + 2 2524 ... 5 2525 ... 2526 ... And very friendly error messages: 2527 ... 2528 ... >>> 1/0 2529 ... To Infinity 2530 ... And 2531 ... Beyond 2532 ... 2533 ... You can use logic if you want: 2534 ... 2535 ... >>> if 0: 2536 ... ... blah 2537 ... ... blah 2538 ... ... 2539 ... 2540 ... Ho hum 2541 ... ''' 2542 2543 >>> print(script_from_examples(text)) 2544 # Here are examples of simple math. 2545 # 2546 # Python has super accurate integer addition 2547 # 2548 2 + 2 2549 # Expected: 2550 ## 5 2551 # 2552 # And very friendly error messages: 2553 # 2554 1/0 2555 # Expected: 2556 ## To Infinity 2557 ## And 2558 ## Beyond 2559 # 2560 # You can use logic if you want: 2561 # 2562 if 0: 2563 blah 2564 blah 2565 # 2566 # Ho hum 2567 <BLANKLINE> 2568 """ 2569 output = [] 2570 for piece in DocTestParser().parse(s): 2571 if isinstance(piece, Example): 2572 # Add the example's source code (strip trailing NL) 2573 output.append(piece.source[:-1]) 2574 # Add the expected output: 2575 want = piece.want 2576 if want: 2577 output.append('# Expected:') 2578 output += ['## '+l for l in want.split('\n')[:-1]] 2579 else: 2580 # Add non-example text. 2581 output += [_comment_line(l) 2582 for l in piece.split('\n')[:-1]] 2583 2584 # Trim junk on both ends. 2585 while output and output[-1] == '#': 2586 output.pop() 2587 while output and output[0] == '#': 2588 output.pop(0) 2589 # Combine the output, and return it. 2590 # Add a courtesy newline to prevent exec from choking (see bug #1172785) 2591 return '\n'.join(output) + '\n' 2592 2593 def testsource(module, name): 2594 """Extract the test sources from a doctest docstring as a script. 2595 2596 Provide the module (or dotted name of the module) containing the 2597 test to be debugged and the name (within the module) of the object 2598 with the doc string with tests to be debugged. 2599 """ 2600 module = _normalize_module(module) 2601 tests = DocTestFinder().find(module) 2602 test = [t for t in tests if t.name == name] 2603 if not test: 2604 raise ValueError(name, "not found in tests") 2605 test = test[0] 2606 testsrc = script_from_examples(test.docstring) 2607 return testsrc 2608 2609 def debug_src(src, pm=False, globs=None): 2610 """Debug a single doctest docstring, in argument `src`'""" 2611 testsrc = script_from_examples(src) 2612 debug_script(testsrc, pm, globs) 2613 2614 def debug_script(src, pm=False, globs=None): 2615 "Debug a test script. `src` is the script, as a string." 2616 import pdb 2617 2618 if globs: 2619 globs = globs.copy() 2620 else: 2621 globs = {} 2622 2623 if pm: 2624 try: 2625 exec(src, globs, globs) 2626 except: 2627 print(sys.exc_info()[1]) 2628 p = pdb.Pdb(nosigint=True) 2629 p.reset() 2630 p.interaction(None, sys.exc_info()[2]) 2631 else: 2632 pdb.Pdb(nosigint=True).run("exec(%r)" % src, globs, globs) 2633 2634 def debug(module, name, pm=False): 2635 """Debug a single doctest docstring. 2636 2637 Provide the module (or dotted name of the module) containing the 2638 test to be debugged and the name (within the module) of the object 2639 with the docstring with tests to be debugged. 2640 """ 2641 module = _normalize_module(module) 2642 testsrc = testsource(module, name) 2643 debug_script(testsrc, pm, module.__dict__) 2644 2645 ###################################################################### 2646 ## 9. Example Usage 2647 ###################################################################### 2648 class _TestClass: 2649 """ 2650 A pointless class, for sanity-checking of docstring testing. 2651 2652 Methods: 2653 square() 2654 get() 2655 2656 >>> _TestClass(13).get() + _TestClass(-12).get() 2657 1 2658 >>> hex(_TestClass(13).square().get()) 2659 '0xa9' 2660 """ 2661 2662 def __init__(self, val): 2663 """val -> _TestClass object with associated value val. 2664 2665 >>> t = _TestClass(123) 2666 >>> print(t.get()) 2667 123 2668 """ 2669 2670 self.val = val 2671 2672 def square(self): 2673 """square() -> square TestClass's associated value 2674 2675 >>> _TestClass(13).square().get() 2676 169 2677 """ 2678 2679 self.val = self.val ** 2 2680 return self 2681 2682 def get(self): 2683 """get() -> return TestClass's associated value. 2684 2685 >>> x = _TestClass(-42) 2686 >>> print(x.get()) 2687 -42 2688 """ 2689 2690 return self.val 2691 2692 __test__ = {"_TestClass": _TestClass, 2693 "string": r""" 2694 Example of a string object, searched as-is. 2695 >>> x = 1; y = 2 2696 >>> x + y, x * y 2697 (3, 2) 2698 """, 2699 2700 "bool-int equivalence": r""" 2701 In 2.2, boolean expressions displayed 2702 0 or 1. By default, we still accept 2703 them. This can be disabled by passing 2704 DONT_ACCEPT_TRUE_FOR_1 to the new 2705 optionflags argument. 2706 >>> 4 == 4 2707 1 2708 >>> 4 == 4 2709 True 2710 >>> 4 > 4 2711 0 2712 >>> 4 > 4 2713 False 2714 """, 2715 2716 "blank lines": r""" 2717 Blank lines can be marked with <BLANKLINE>: 2718 >>> print('foo\n\nbar\n') 2719 foo 2720 <BLANKLINE> 2721 bar 2722 <BLANKLINE> 2723 """, 2724 2725 "ellipsis": r""" 2726 If the ellipsis flag is used, then '...' can be used to 2727 elide substrings in the desired output: 2728 >>> print(list(range(1000))) #doctest: +ELLIPSIS 2729 [0, 1, 2, ..., 999] 2730 """, 2731 2732 "whitespace normalization": r""" 2733 If the whitespace normalization flag is used, then 2734 differences in whitespace are ignored. 2735 >>> print(list(range(30))) #doctest: +NORMALIZE_WHITESPACE 2736 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 2737 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 2738 27, 28, 29] 2739 """, 2740 } 2741 2742 2743 def _test(): 2744 parser = argparse.ArgumentParser(description="doctest runner") 2745 parser.add_argument('-v', '--verbose', action='store_true', default=False, 2746 help='print very verbose output for all tests') 2747 parser.add_argument('-o', '--option', action='append', 2748 choices=OPTIONFLAGS_BY_NAME.keys(), default=[], 2749 help=('specify a doctest option flag to apply' 2750 ' to the test run; may be specified more' 2751 ' than once to apply multiple options')) 2752 parser.add_argument('-f', '--fail-fast', action='store_true', 2753 help=('stop running tests after first failure (this' 2754 ' is a shorthand for -o FAIL_FAST, and is' 2755 ' in addition to any other -o options)')) 2756 parser.add_argument('file', nargs='+', 2757 help='file containing the tests to run') 2758 args = parser.parse_args() 2759 testfiles = args.file 2760 # Verbose used to be handled by the "inspect argv" magic in DocTestRunner, 2761 # but since we are using argparse we are passing it manually now. 2762 verbose = args.verbose 2763 options = 0 2764 for option in args.option: 2765 options |= OPTIONFLAGS_BY_NAME[option] 2766 if args.fail_fast: 2767 options |= FAIL_FAST 2768 for filename in testfiles: 2769 if filename.endswith(".py"): 2770 # It is a module -- insert its dir into sys.path and try to 2771 # import it. If it is part of a package, that possibly 2772 # won't work because of package imports. 2773 dirname, filename = os.path.split(filename) 2774 sys.path.insert(0, dirname) 2775 m = __import__(filename[:-3]) 2776 del sys.path[0] 2777 failures, _ = testmod(m, verbose=verbose, optionflags=options) 2778 else: 2779 failures, _ = testfile(filename, module_relative=False, 2780 verbose=verbose, optionflags=options) 2781 if failures: 2782 return 1 2783 return 0 2784 2785 2786 if __name__ == "__main__": 2787 sys.exit(_test()) 2788