1 import parser 2 import unittest 3 import sys 4 import struct 5 from test import test_support as support 6 from test.script_helper import assert_python_failure 7 8 # 9 # First, we test that we can generate trees from valid source fragments, 10 # and that these valid trees are indeed allowed by the tree-loading side 11 # of the parser module. 12 # 13 14 class RoundtripLegalSyntaxTestCase(unittest.TestCase): 15 16 def roundtrip(self, f, s): 17 st1 = f(s) 18 t = st1.totuple() 19 try: 20 st2 = parser.sequence2st(t) 21 except parser.ParserError, why: 22 self.fail("could not roundtrip %r: %s" % (s, why)) 23 24 self.assertEqual(t, st2.totuple(), 25 "could not re-generate syntax tree") 26 27 def check_expr(self, s): 28 self.roundtrip(parser.expr, s) 29 30 def test_flags_passed(self): 31 # The unicode literals flags has to be passed from the paser to AST 32 # generation. 33 suite = parser.suite("from __future__ import unicode_literals; x = ''") 34 code = suite.compile() 35 scope = {} 36 exec code in scope 37 self.assertIsInstance(scope["x"], unicode) 38 39 def check_suite(self, s): 40 self.roundtrip(parser.suite, s) 41 42 def test_yield_statement(self): 43 self.check_suite("def f(): yield 1") 44 self.check_suite("def f(): yield") 45 self.check_suite("def f(): x += yield") 46 self.check_suite("def f(): x = yield 1") 47 self.check_suite("def f(): x = y = yield 1") 48 self.check_suite("def f(): x = yield") 49 self.check_suite("def f(): x = y = yield") 50 self.check_suite("def f(): 1 + (yield)*2") 51 self.check_suite("def f(): (yield 1)*2") 52 self.check_suite("def f(): return; yield 1") 53 self.check_suite("def f(): yield 1; return") 54 self.check_suite("def f():\n" 55 " for x in range(30):\n" 56 " yield x\n") 57 self.check_suite("def f():\n" 58 " if (yield):\n" 59 " yield x\n") 60 61 def test_expressions(self): 62 self.check_expr("foo(1)") 63 self.check_expr("{1:1}") 64 self.check_expr("{1:1, 2:2, 3:3}") 65 self.check_expr("{1:1, 2:2, 3:3,}") 66 self.check_expr("{1}") 67 self.check_expr("{1, 2, 3}") 68 self.check_expr("{1, 2, 3,}") 69 self.check_expr("[]") 70 self.check_expr("[1]") 71 self.check_expr("[1, 2, 3]") 72 self.check_expr("[1, 2, 3,]") 73 self.check_expr("()") 74 self.check_expr("(1,)") 75 self.check_expr("(1, 2, 3)") 76 self.check_expr("(1, 2, 3,)") 77 self.check_expr("[x**3 for x in range(20)]") 78 self.check_expr("[x**3 for x in range(20) if x % 3]") 79 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]") 80 self.check_expr("[x+y for x in range(30) for y in range(20) if x % 2 if y % 3]") 81 #self.check_expr("[x for x in lambda: True, lambda: False if x()]") 82 self.check_expr("list(x**3 for x in range(20))") 83 self.check_expr("list(x**3 for x in range(20) if x % 3)") 84 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)") 85 self.check_expr("list(x+y for x in range(30) for y in range(20) if x % 2 if y % 3)") 86 self.check_expr("{x**3 for x in range(30)}") 87 self.check_expr("{x**3 for x in range(30) if x % 3}") 88 self.check_expr("{x**3 for x in range(30) if x % 2 if x % 3}") 89 self.check_expr("{x+y for x in range(30) for y in range(20) if x % 2 if y % 3}") 90 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30))}") 91 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3}") 92 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3 if y % 3}") 93 self.check_expr("{x:y for x in range(30) for y in range(20) if x % 2 if y % 3}") 94 self.check_expr("foo(*args)") 95 self.check_expr("foo(*args, **kw)") 96 self.check_expr("foo(**kw)") 97 self.check_expr("foo(key=value)") 98 self.check_expr("foo(key=value, *args)") 99 self.check_expr("foo(key=value, *args, **kw)") 100 self.check_expr("foo(key=value, **kw)") 101 self.check_expr("foo(a, b, c, *args)") 102 self.check_expr("foo(a, b, c, *args, **kw)") 103 self.check_expr("foo(a, b, c, **kw)") 104 self.check_expr("foo(a, *args, keyword=23)") 105 self.check_expr("foo + bar") 106 self.check_expr("foo - bar") 107 self.check_expr("foo * bar") 108 self.check_expr("foo / bar") 109 self.check_expr("foo // bar") 110 self.check_expr("lambda: 0") 111 self.check_expr("lambda x: 0") 112 self.check_expr("lambda *y: 0") 113 self.check_expr("lambda *y, **z: 0") 114 self.check_expr("lambda **z: 0") 115 self.check_expr("lambda x, y: 0") 116 self.check_expr("lambda foo=bar: 0") 117 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0") 118 self.check_expr("lambda foo=bar, **z: 0") 119 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0") 120 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0") 121 self.check_expr("lambda x, *y, **z: 0") 122 self.check_expr("lambda x: 5 if x else 2") 123 self.check_expr("(x for x in range(10))") 124 self.check_expr("foo(x for x in range(10))") 125 126 def test_print(self): 127 self.check_suite("print") 128 self.check_suite("print 1") 129 self.check_suite("print 1,") 130 self.check_suite("print >>fp") 131 self.check_suite("print >>fp, 1") 132 self.check_suite("print >>fp, 1,") 133 134 def test_simple_expression(self): 135 # expr_stmt 136 self.check_suite("a") 137 138 def test_simple_assignments(self): 139 self.check_suite("a = b") 140 self.check_suite("a = b = c = d = e") 141 142 def test_simple_augmented_assignments(self): 143 self.check_suite("a += b") 144 self.check_suite("a -= b") 145 self.check_suite("a *= b") 146 self.check_suite("a /= b") 147 self.check_suite("a //= b") 148 self.check_suite("a %= b") 149 self.check_suite("a &= b") 150 self.check_suite("a |= b") 151 self.check_suite("a ^= b") 152 self.check_suite("a <<= b") 153 self.check_suite("a >>= b") 154 self.check_suite("a **= b") 155 156 def test_function_defs(self): 157 self.check_suite("def f(): pass") 158 self.check_suite("def f(*args): pass") 159 self.check_suite("def f(*args, **kw): pass") 160 self.check_suite("def f(**kw): pass") 161 self.check_suite("def f(foo=bar): pass") 162 self.check_suite("def f(foo=bar, *args): pass") 163 self.check_suite("def f(foo=bar, *args, **kw): pass") 164 self.check_suite("def f(foo=bar, **kw): pass") 165 166 self.check_suite("def f(a, b): pass") 167 self.check_suite("def f(a, b, *args): pass") 168 self.check_suite("def f(a, b, *args, **kw): pass") 169 self.check_suite("def f(a, b, **kw): pass") 170 self.check_suite("def f(a, b, foo=bar): pass") 171 self.check_suite("def f(a, b, foo=bar, *args): pass") 172 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass") 173 self.check_suite("def f(a, b, foo=bar, **kw): pass") 174 175 self.check_suite("@staticmethod\n" 176 "def f(): pass") 177 self.check_suite("@staticmethod\n" 178 "@funcattrs(x, y)\n" 179 "def f(): pass") 180 self.check_suite("@funcattrs()\n" 181 "def f(): pass") 182 183 def test_class_defs(self): 184 self.check_suite("class foo():pass") 185 self.check_suite("@class_decorator\n" 186 "class foo():pass") 187 self.check_suite("@class_decorator(arg)\n" 188 "class foo():pass") 189 self.check_suite("@decorator1\n" 190 "@decorator2\n" 191 "class foo():pass") 192 193 194 def test_import_from_statement(self): 195 self.check_suite("from sys.path import *") 196 self.check_suite("from sys.path import dirname") 197 self.check_suite("from sys.path import (dirname)") 198 self.check_suite("from sys.path import (dirname,)") 199 self.check_suite("from sys.path import dirname as my_dirname") 200 self.check_suite("from sys.path import (dirname as my_dirname)") 201 self.check_suite("from sys.path import (dirname as my_dirname,)") 202 self.check_suite("from sys.path import dirname, basename") 203 self.check_suite("from sys.path import (dirname, basename)") 204 self.check_suite("from sys.path import (dirname, basename,)") 205 self.check_suite( 206 "from sys.path import dirname as my_dirname, basename") 207 self.check_suite( 208 "from sys.path import (dirname as my_dirname, basename)") 209 self.check_suite( 210 "from sys.path import (dirname as my_dirname, basename,)") 211 self.check_suite( 212 "from sys.path import dirname, basename as my_basename") 213 self.check_suite( 214 "from sys.path import (dirname, basename as my_basename)") 215 self.check_suite( 216 "from sys.path import (dirname, basename as my_basename,)") 217 self.check_suite("from .bogus import x") 218 219 def test_basic_import_statement(self): 220 self.check_suite("import sys") 221 self.check_suite("import sys as system") 222 self.check_suite("import sys, math") 223 self.check_suite("import sys as system, math") 224 self.check_suite("import sys, math as my_math") 225 226 def test_relative_imports(self): 227 self.check_suite("from . import name") 228 self.check_suite("from .. import name") 229 self.check_suite("from .pkg import name") 230 self.check_suite("from ..pkg import name") 231 232 def test_pep263(self): 233 self.check_suite("# -*- coding: iso-8859-1 -*-\n" 234 "pass\n") 235 236 def test_assert(self): 237 self.check_suite("assert alo < ahi and blo < bhi\n") 238 239 def test_with(self): 240 self.check_suite("with open('x'): pass\n") 241 self.check_suite("with open('x') as f: pass\n") 242 self.check_suite("with open('x') as f, open('y') as g: pass\n") 243 244 def test_try_stmt(self): 245 self.check_suite("try: pass\nexcept: pass\n") 246 self.check_suite("try: pass\nfinally: pass\n") 247 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n") 248 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n" 249 "finally: pass\n") 250 self.check_suite("try: pass\nexcept: pass\nelse: pass\n") 251 self.check_suite("try: pass\nexcept: pass\nelse: pass\n" 252 "finally: pass\n") 253 254 def test_except_clause(self): 255 self.check_suite("try: pass\nexcept: pass\n") 256 self.check_suite("try: pass\nexcept A: pass\n") 257 self.check_suite("try: pass\nexcept A, e: pass\n") 258 self.check_suite("try: pass\nexcept A as e: pass\n") 259 260 def test_position(self): 261 # An absolutely minimal test of position information. Better 262 # tests would be a big project. 263 code = "def f(x):\n return x + 1" 264 st1 = parser.suite(code) 265 st2 = st1.totuple(line_info=1, col_info=1) 266 267 def walk(tree): 268 node_type = tree[0] 269 next = tree[1] 270 if isinstance(next, tuple): 271 for elt in tree[1:]: 272 for x in walk(elt): 273 yield x 274 else: 275 yield tree 276 277 terminals = list(walk(st2)) 278 self.assertEqual([ 279 (1, 'def', 1, 0), 280 (1, 'f', 1, 4), 281 (7, '(', 1, 5), 282 (1, 'x', 1, 6), 283 (8, ')', 1, 7), 284 (11, ':', 1, 8), 285 (4, '', 1, 9), 286 (5, '', 2, -1), 287 (1, 'return', 2, 4), 288 (1, 'x', 2, 11), 289 (14, '+', 2, 13), 290 (2, '1', 2, 15), 291 (4, '', 2, 16), 292 (6, '', 2, -1), 293 (4, '', 2, -1), 294 (0, '', 2, -1)], 295 terminals) 296 297 298 # 299 # Second, we take *invalid* trees and make sure we get ParserError 300 # rejections for them. 301 # 302 303 class IllegalSyntaxTestCase(unittest.TestCase): 304 305 def check_bad_tree(self, tree, label): 306 try: 307 parser.sequence2st(tree) 308 except parser.ParserError: 309 pass 310 else: 311 self.fail("did not detect invalid tree for %r" % label) 312 313 def test_junk(self): 314 # not even remotely valid: 315 self.check_bad_tree((1, 2, 3), "<junk>") 316 317 def test_illegal_yield_1(self): 318 # Illegal yield statement: def f(): return 1; yield 1 319 tree = \ 320 (257, 321 (264, 322 (285, 323 (259, 324 (1, 'def'), 325 (1, 'f'), 326 (260, (7, '('), (8, ')')), 327 (11, ':'), 328 (291, 329 (4, ''), 330 (5, ''), 331 (264, 332 (265, 333 (266, 334 (272, 335 (275, 336 (1, 'return'), 337 (313, 338 (292, 339 (293, 340 (294, 341 (295, 342 (297, 343 (298, 344 (299, 345 (300, 346 (301, 347 (302, (303, (304, (305, (2, '1')))))))))))))))))), 348 (264, 349 (265, 350 (266, 351 (272, 352 (276, 353 (1, 'yield'), 354 (313, 355 (292, 356 (293, 357 (294, 358 (295, 359 (297, 360 (298, 361 (299, 362 (300, 363 (301, 364 (302, 365 (303, (304, (305, (2, '1')))))))))))))))))), 366 (4, ''))), 367 (6, ''))))), 368 (4, ''), 369 (0, '')))) 370 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 371 372 def test_illegal_yield_2(self): 373 # Illegal return in generator: def f(): return 1; yield 1 374 tree = \ 375 (257, 376 (264, 377 (265, 378 (266, 379 (278, 380 (1, 'from'), 381 (281, (1, '__future__')), 382 (1, 'import'), 383 (279, (1, 'generators')))), 384 (4, ''))), 385 (264, 386 (285, 387 (259, 388 (1, 'def'), 389 (1, 'f'), 390 (260, (7, '('), (8, ')')), 391 (11, ':'), 392 (291, 393 (4, ''), 394 (5, ''), 395 (264, 396 (265, 397 (266, 398 (272, 399 (275, 400 (1, 'return'), 401 (313, 402 (292, 403 (293, 404 (294, 405 (295, 406 (297, 407 (298, 408 (299, 409 (300, 410 (301, 411 (302, (303, (304, (305, (2, '1')))))))))))))))))), 412 (264, 413 (265, 414 (266, 415 (272, 416 (276, 417 (1, 'yield'), 418 (313, 419 (292, 420 (293, 421 (294, 422 (295, 423 (297, 424 (298, 425 (299, 426 (300, 427 (301, 428 (302, 429 (303, (304, (305, (2, '1')))))))))))))))))), 430 (4, ''))), 431 (6, ''))))), 432 (4, ''), 433 (0, '')))) 434 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 435 436 def test_print_chevron_comma(self): 437 # Illegal input: print >>fp, 438 tree = \ 439 (257, 440 (264, 441 (265, 442 (266, 443 (268, 444 (1, 'print'), 445 (35, '>>'), 446 (290, 447 (291, 448 (292, 449 (293, 450 (295, 451 (296, 452 (297, 453 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))), 454 (12, ','))), 455 (4, ''))), 456 (0, '')) 457 self.check_bad_tree(tree, "print >>fp,") 458 459 def test_a_comma_comma_c(self): 460 # Illegal input: a,,c 461 tree = \ 462 (258, 463 (311, 464 (290, 465 (291, 466 (292, 467 (293, 468 (295, 469 (296, 470 (297, 471 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))), 472 (12, ','), 473 (12, ','), 474 (290, 475 (291, 476 (292, 477 (293, 478 (295, 479 (296, 480 (297, 481 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))), 482 (4, ''), 483 (0, '')) 484 self.check_bad_tree(tree, "a,,c") 485 486 def test_illegal_operator(self): 487 # Illegal input: a $= b 488 tree = \ 489 (257, 490 (264, 491 (265, 492 (266, 493 (267, 494 (312, 495 (291, 496 (292, 497 (293, 498 (294, 499 (296, 500 (297, 501 (298, 502 (299, 503 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))), 504 (268, (37, '$=')), 505 (312, 506 (291, 507 (292, 508 (293, 509 (294, 510 (296, 511 (297, 512 (298, 513 (299, 514 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))), 515 (4, ''))), 516 (0, '')) 517 self.check_bad_tree(tree, "a $= b") 518 519 def test_malformed_global(self): 520 #doesn't have global keyword in ast 521 tree = (257, 522 (264, 523 (265, 524 (266, 525 (282, (1, 'foo'))), (4, ''))), 526 (4, ''), 527 (0, '')) 528 self.check_bad_tree(tree, "malformed global ast") 529 530 def test_missing_import_source(self): 531 # from import a 532 tree = \ 533 (257, 534 (267, 535 (268, 536 (269, 537 (281, 538 (283, (1, 'from'), (1, 'import'), 539 (286, (284, (1, 'fred')))))), 540 (4, ''))), 541 (4, ''), (0, '')) 542 self.check_bad_tree(tree, "from import a") 543 544 545 class CompileTestCase(unittest.TestCase): 546 547 # These tests are very minimal. :-( 548 549 def test_compile_expr(self): 550 st = parser.expr('2 + 3') 551 code = parser.compilest(st) 552 self.assertEqual(eval(code), 5) 553 554 def test_compile_suite(self): 555 st = parser.suite('x = 2; y = x + 3') 556 code = parser.compilest(st) 557 globs = {} 558 exec code in globs 559 self.assertEqual(globs['y'], 5) 560 561 def test_compile_error(self): 562 st = parser.suite('1 = 3 + 4') 563 self.assertRaises(SyntaxError, parser.compilest, st) 564 565 def test_compile_badunicode(self): 566 st = parser.suite('a = u"\U12345678"') 567 self.assertRaises(SyntaxError, parser.compilest, st) 568 st = parser.suite('a = u"\u1"') 569 self.assertRaises(SyntaxError, parser.compilest, st) 570 571 def test_issue_9011(self): 572 # Issue 9011: compilation of an unary minus expression changed 573 # the meaning of the ST, so that a second compilation produced 574 # incorrect results. 575 st = parser.expr('-3') 576 code1 = parser.compilest(st) 577 self.assertEqual(eval(code1), -3) 578 code2 = parser.compilest(st) 579 self.assertEqual(eval(code2), -3) 580 581 582 class ParserStackLimitTestCase(unittest.TestCase): 583 """try to push the parser to/over its limits. 584 see http://bugs.python.org/issue1881 for a discussion 585 """ 586 def _nested_expression(self, level): 587 return "["*level+"]"*level 588 589 def test_deeply_nested_list(self): 590 e = self._nested_expression(99) 591 st = parser.expr(e) 592 st.compile() 593 594 def test_trigger_memory_error(self): 595 e = self._nested_expression(100) 596 rc, out, err = assert_python_failure('-c', e) 597 # parsing the expression will result in an error message 598 # followed by a MemoryError (see #11963) 599 self.assertIn(b's_push: parser stack overflow', err) 600 self.assertIn(b'MemoryError', err) 601 602 class STObjectTestCase(unittest.TestCase): 603 """Test operations on ST objects themselves""" 604 605 check_sizeof = support.check_sizeof 606 607 @support.cpython_only 608 def test_sizeof(self): 609 def XXXROUNDUP(n): 610 if n <= 1: 611 return n 612 if n <= 128: 613 return (n + 3) & ~3 614 return 1 << (n - 1).bit_length() 615 616 basesize = support.calcobjsize('Pii') 617 nodesize = struct.calcsize('hP3iP0h') 618 def sizeofchildren(node): 619 if node is None: 620 return 0 621 res = 0 622 hasstr = len(node) > 1 and isinstance(node[-1], str) 623 if hasstr: 624 res += len(node[-1]) + 1 625 children = node[1:-1] if hasstr else node[1:] 626 if children: 627 res += XXXROUNDUP(len(children)) * nodesize 628 for child in children: 629 res += sizeofchildren(child) 630 return res 631 632 def check_st_sizeof(st): 633 self.check_sizeof(st, basesize + nodesize + 634 sizeofchildren(st.totuple())) 635 636 check_st_sizeof(parser.expr('2 + 3')) 637 check_st_sizeof(parser.expr('2 + 3 + 4')) 638 check_st_sizeof(parser.suite('x = 2 + 3')) 639 check_st_sizeof(parser.suite('')) 640 check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-')) 641 check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']')) 642 643 644 # XXX tests for pickling and unpickling of ST objects should go here 645 646 def test_main(): 647 support.run_unittest( 648 RoundtripLegalSyntaxTestCase, 649 IllegalSyntaxTestCase, 650 CompileTestCase, 651 ParserStackLimitTestCase, 652 STObjectTestCase, 653 ) 654 655 656 if __name__ == "__main__": 657 test_main() 658
