1 import parser 2 import unittest 3 import sys 4 from test import test_support 5 6 # 7 # First, we test that we can generate trees from valid source fragments, 8 # and that these valid trees are indeed allowed by the tree-loading side 9 # of the parser module. 10 # 11 12 class RoundtripLegalSyntaxTestCase(unittest.TestCase): 13 14 def roundtrip(self, f, s): 15 st1 = f(s) 16 t = st1.totuple() 17 try: 18 st2 = parser.sequence2st(t) 19 except parser.ParserError, why: 20 self.fail("could not roundtrip %r: %s" % (s, why)) 21 22 self.assertEqual(t, st2.totuple(), 23 "could not re-generate syntax tree") 24 25 def check_expr(self, s): 26 self.roundtrip(parser.expr, s) 27 28 def test_flags_passed(self): 29 # The unicode literals flags has to be passed from the paser to AST 30 # generation. 31 suite = parser.suite("from __future__ import unicode_literals; x = ''") 32 code = suite.compile() 33 scope = {} 34 exec code in scope 35 self.assertIsInstance(scope["x"], unicode) 36 37 def check_suite(self, s): 38 self.roundtrip(parser.suite, s) 39 40 def test_yield_statement(self): 41 self.check_suite("def f(): yield 1") 42 self.check_suite("def f(): yield") 43 self.check_suite("def f(): x += yield") 44 self.check_suite("def f(): x = yield 1") 45 self.check_suite("def f(): x = y = yield 1") 46 self.check_suite("def f(): x = yield") 47 self.check_suite("def f(): x = y = yield") 48 self.check_suite("def f(): 1 + (yield)*2") 49 self.check_suite("def f(): (yield 1)*2") 50 self.check_suite("def f(): return; yield 1") 51 self.check_suite("def f(): yield 1; return") 52 self.check_suite("def f():\n" 53 " for x in range(30):\n" 54 " yield x\n") 55 self.check_suite("def f():\n" 56 " if (yield):\n" 57 " yield x\n") 58 59 def test_expressions(self): 60 self.check_expr("foo(1)") 61 self.check_expr("{1:1}") 62 self.check_expr("{1:1, 2:2, 3:3}") 63 self.check_expr("{1:1, 2:2, 3:3,}") 64 self.check_expr("{1}") 65 self.check_expr("{1, 2, 3}") 66 self.check_expr("{1, 2, 3,}") 67 self.check_expr("[]") 68 self.check_expr("[1]") 69 self.check_expr("[1, 2, 3]") 70 self.check_expr("[1, 2, 3,]") 71 self.check_expr("()") 72 self.check_expr("(1,)") 73 self.check_expr("(1, 2, 3)") 74 self.check_expr("(1, 2, 3,)") 75 self.check_expr("[x**3 for x in range(20)]") 76 self.check_expr("[x**3 for x in range(20) if x % 3]") 77 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]") 78 self.check_expr("[x+y for x in range(30) for y in range(20) if x % 2 if y % 3]") 79 #self.check_expr("[x for x in lambda: True, lambda: False if x()]") 80 self.check_expr("list(x**3 for x in range(20))") 81 self.check_expr("list(x**3 for x in range(20) if x % 3)") 82 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)") 83 self.check_expr("list(x+y for x in range(30) for y in range(20) if x % 2 if y % 3)") 84 self.check_expr("{x**3 for x in range(30)}") 85 self.check_expr("{x**3 for x in range(30) if x % 3}") 86 self.check_expr("{x**3 for x in range(30) if x % 2 if x % 3}") 87 self.check_expr("{x+y for x in range(30) for y in range(20) if x % 2 if y % 3}") 88 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30))}") 89 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3}") 90 self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3 if y % 3}") 91 self.check_expr("{x:y for x in range(30) for y in range(20) if x % 2 if y % 3}") 92 self.check_expr("foo(*args)") 93 self.check_expr("foo(*args, **kw)") 94 self.check_expr("foo(**kw)") 95 self.check_expr("foo(key=value)") 96 self.check_expr("foo(key=value, *args)") 97 self.check_expr("foo(key=value, *args, **kw)") 98 self.check_expr("foo(key=value, **kw)") 99 self.check_expr("foo(a, b, c, *args)") 100 self.check_expr("foo(a, b, c, *args, **kw)") 101 self.check_expr("foo(a, b, c, **kw)") 102 self.check_expr("foo(a, *args, keyword=23)") 103 self.check_expr("foo + bar") 104 self.check_expr("foo - bar") 105 self.check_expr("foo * bar") 106 self.check_expr("foo / bar") 107 self.check_expr("foo // bar") 108 self.check_expr("lambda: 0") 109 self.check_expr("lambda x: 0") 110 self.check_expr("lambda *y: 0") 111 self.check_expr("lambda *y, **z: 0") 112 self.check_expr("lambda **z: 0") 113 self.check_expr("lambda x, y: 0") 114 self.check_expr("lambda foo=bar: 0") 115 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0") 116 self.check_expr("lambda foo=bar, **z: 0") 117 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0") 118 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0") 119 self.check_expr("lambda x, *y, **z: 0") 120 self.check_expr("lambda x: 5 if x else 2") 121 self.check_expr("(x for x in range(10))") 122 self.check_expr("foo(x for x in range(10))") 123 124 def test_print(self): 125 self.check_suite("print") 126 self.check_suite("print 1") 127 self.check_suite("print 1,") 128 self.check_suite("print >>fp") 129 self.check_suite("print >>fp, 1") 130 self.check_suite("print >>fp, 1,") 131 132 def test_simple_expression(self): 133 # expr_stmt 134 self.check_suite("a") 135 136 def test_simple_assignments(self): 137 self.check_suite("a = b") 138 self.check_suite("a = b = c = d = e") 139 140 def test_simple_augmented_assignments(self): 141 self.check_suite("a += b") 142 self.check_suite("a -= b") 143 self.check_suite("a *= b") 144 self.check_suite("a /= b") 145 self.check_suite("a //= b") 146 self.check_suite("a %= b") 147 self.check_suite("a &= b") 148 self.check_suite("a |= b") 149 self.check_suite("a ^= b") 150 self.check_suite("a <<= b") 151 self.check_suite("a >>= b") 152 self.check_suite("a **= b") 153 154 def test_function_defs(self): 155 self.check_suite("def f(): pass") 156 self.check_suite("def f(*args): pass") 157 self.check_suite("def f(*args, **kw): pass") 158 self.check_suite("def f(**kw): pass") 159 self.check_suite("def f(foo=bar): pass") 160 self.check_suite("def f(foo=bar, *args): pass") 161 self.check_suite("def f(foo=bar, *args, **kw): pass") 162 self.check_suite("def f(foo=bar, **kw): pass") 163 164 self.check_suite("def f(a, b): pass") 165 self.check_suite("def f(a, b, *args): pass") 166 self.check_suite("def f(a, b, *args, **kw): pass") 167 self.check_suite("def f(a, b, **kw): pass") 168 self.check_suite("def f(a, b, foo=bar): pass") 169 self.check_suite("def f(a, b, foo=bar, *args): pass") 170 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass") 171 self.check_suite("def f(a, b, foo=bar, **kw): pass") 172 173 self.check_suite("@staticmethod\n" 174 "def f(): pass") 175 self.check_suite("@staticmethod\n" 176 "@funcattrs(x, y)\n" 177 "def f(): pass") 178 self.check_suite("@funcattrs()\n" 179 "def f(): pass") 180 181 def test_class_defs(self): 182 self.check_suite("class foo():pass") 183 self.check_suite("@class_decorator\n" 184 "class foo():pass") 185 self.check_suite("@class_decorator(arg)\n" 186 "class foo():pass") 187 self.check_suite("@decorator1\n" 188 "@decorator2\n" 189 "class foo():pass") 190 191 192 def test_import_from_statement(self): 193 self.check_suite("from sys.path import *") 194 self.check_suite("from sys.path import dirname") 195 self.check_suite("from sys.path import (dirname)") 196 self.check_suite("from sys.path import (dirname,)") 197 self.check_suite("from sys.path import dirname as my_dirname") 198 self.check_suite("from sys.path import (dirname as my_dirname)") 199 self.check_suite("from sys.path import (dirname as my_dirname,)") 200 self.check_suite("from sys.path import dirname, basename") 201 self.check_suite("from sys.path import (dirname, basename)") 202 self.check_suite("from sys.path import (dirname, basename,)") 203 self.check_suite( 204 "from sys.path import dirname as my_dirname, basename") 205 self.check_suite( 206 "from sys.path import (dirname as my_dirname, basename)") 207 self.check_suite( 208 "from sys.path import (dirname as my_dirname, basename,)") 209 self.check_suite( 210 "from sys.path import dirname, basename as my_basename") 211 self.check_suite( 212 "from sys.path import (dirname, basename as my_basename)") 213 self.check_suite( 214 "from sys.path import (dirname, basename as my_basename,)") 215 self.check_suite("from .bogus import x") 216 217 def test_basic_import_statement(self): 218 self.check_suite("import sys") 219 self.check_suite("import sys as system") 220 self.check_suite("import sys, math") 221 self.check_suite("import sys as system, math") 222 self.check_suite("import sys, math as my_math") 223 224 def test_relative_imports(self): 225 self.check_suite("from . import name") 226 self.check_suite("from .. import name") 227 self.check_suite("from .pkg import name") 228 self.check_suite("from ..pkg import name") 229 230 def test_pep263(self): 231 self.check_suite("# -*- coding: iso-8859-1 -*-\n" 232 "pass\n") 233 234 def test_assert(self): 235 self.check_suite("assert alo < ahi and blo < bhi\n") 236 237 def test_with(self): 238 self.check_suite("with open('x'): pass\n") 239 self.check_suite("with open('x') as f: pass\n") 240 self.check_suite("with open('x') as f, open('y') as g: pass\n") 241 242 def test_try_stmt(self): 243 self.check_suite("try: pass\nexcept: pass\n") 244 self.check_suite("try: pass\nfinally: pass\n") 245 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n") 246 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n" 247 "finally: pass\n") 248 self.check_suite("try: pass\nexcept: pass\nelse: pass\n") 249 self.check_suite("try: pass\nexcept: pass\nelse: pass\n" 250 "finally: pass\n") 251 252 def test_except_clause(self): 253 self.check_suite("try: pass\nexcept: pass\n") 254 self.check_suite("try: pass\nexcept A: pass\n") 255 self.check_suite("try: pass\nexcept A, e: pass\n") 256 self.check_suite("try: pass\nexcept A as e: pass\n") 257 258 def test_position(self): 259 # An absolutely minimal test of position information. Better 260 # tests would be a big project. 261 code = "def f(x):\n return x + 1" 262 st1 = parser.suite(code) 263 st2 = st1.totuple(line_info=1, col_info=1) 264 265 def walk(tree): 266 node_type = tree[0] 267 next = tree[1] 268 if isinstance(next, tuple): 269 for elt in tree[1:]: 270 for x in walk(elt): 271 yield x 272 else: 273 yield tree 274 275 terminals = list(walk(st2)) 276 self.assertEqual([ 277 (1, 'def', 1, 0), 278 (1, 'f', 1, 4), 279 (7, '(', 1, 5), 280 (1, 'x', 1, 6), 281 (8, ')', 1, 7), 282 (11, ':', 1, 8), 283 (4, '', 1, 9), 284 (5, '', 2, -1), 285 (1, 'return', 2, 4), 286 (1, 'x', 2, 11), 287 (14, '+', 2, 13), 288 (2, '1', 2, 15), 289 (4, '', 2, 16), 290 (6, '', 2, -1), 291 (4, '', 2, -1), 292 (0, '', 2, -1)], 293 terminals) 294 295 296 # 297 # Second, we take *invalid* trees and make sure we get ParserError 298 # rejections for them. 299 # 300 301 class IllegalSyntaxTestCase(unittest.TestCase): 302 303 def check_bad_tree(self, tree, label): 304 try: 305 parser.sequence2st(tree) 306 except parser.ParserError: 307 pass 308 else: 309 self.fail("did not detect invalid tree for %r" % label) 310 311 def test_junk(self): 312 # not even remotely valid: 313 self.check_bad_tree((1, 2, 3), "<junk>") 314 315 def test_illegal_yield_1(self): 316 # Illegal yield statement: def f(): return 1; yield 1 317 tree = \ 318 (257, 319 (264, 320 (285, 321 (259, 322 (1, 'def'), 323 (1, 'f'), 324 (260, (7, '('), (8, ')')), 325 (11, ':'), 326 (291, 327 (4, ''), 328 (5, ''), 329 (264, 330 (265, 331 (266, 332 (272, 333 (275, 334 (1, 'return'), 335 (313, 336 (292, 337 (293, 338 (294, 339 (295, 340 (297, 341 (298, 342 (299, 343 (300, 344 (301, 345 (302, (303, (304, (305, (2, '1')))))))))))))))))), 346 (264, 347 (265, 348 (266, 349 (272, 350 (276, 351 (1, 'yield'), 352 (313, 353 (292, 354 (293, 355 (294, 356 (295, 357 (297, 358 (298, 359 (299, 360 (300, 361 (301, 362 (302, 363 (303, (304, (305, (2, '1')))))))))))))))))), 364 (4, ''))), 365 (6, ''))))), 366 (4, ''), 367 (0, '')))) 368 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 369 370 def test_illegal_yield_2(self): 371 # Illegal return in generator: def f(): return 1; yield 1 372 tree = \ 373 (257, 374 (264, 375 (265, 376 (266, 377 (278, 378 (1, 'from'), 379 (281, (1, '__future__')), 380 (1, 'import'), 381 (279, (1, 'generators')))), 382 (4, ''))), 383 (264, 384 (285, 385 (259, 386 (1, 'def'), 387 (1, 'f'), 388 (260, (7, '('), (8, ')')), 389 (11, ':'), 390 (291, 391 (4, ''), 392 (5, ''), 393 (264, 394 (265, 395 (266, 396 (272, 397 (275, 398 (1, 'return'), 399 (313, 400 (292, 401 (293, 402 (294, 403 (295, 404 (297, 405 (298, 406 (299, 407 (300, 408 (301, 409 (302, (303, (304, (305, (2, '1')))))))))))))))))), 410 (264, 411 (265, 412 (266, 413 (272, 414 (276, 415 (1, 'yield'), 416 (313, 417 (292, 418 (293, 419 (294, 420 (295, 421 (297, 422 (298, 423 (299, 424 (300, 425 (301, 426 (302, 427 (303, (304, (305, (2, '1')))))))))))))))))), 428 (4, ''))), 429 (6, ''))))), 430 (4, ''), 431 (0, '')))) 432 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 433 434 def test_print_chevron_comma(self): 435 # Illegal input: print >>fp, 436 tree = \ 437 (257, 438 (264, 439 (265, 440 (266, 441 (268, 442 (1, 'print'), 443 (35, '>>'), 444 (290, 445 (291, 446 (292, 447 (293, 448 (295, 449 (296, 450 (297, 451 (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))), 452 (12, ','))), 453 (4, ''))), 454 (0, '')) 455 self.check_bad_tree(tree, "print >>fp,") 456 457 def test_a_comma_comma_c(self): 458 # Illegal input: a,,c 459 tree = \ 460 (258, 461 (311, 462 (290, 463 (291, 464 (292, 465 (293, 466 (295, 467 (296, 468 (297, 469 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))), 470 (12, ','), 471 (12, ','), 472 (290, 473 (291, 474 (292, 475 (293, 476 (295, 477 (296, 478 (297, 479 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))), 480 (4, ''), 481 (0, '')) 482 self.check_bad_tree(tree, "a,,c") 483 484 def test_illegal_operator(self): 485 # Illegal input: a $= b 486 tree = \ 487 (257, 488 (264, 489 (265, 490 (266, 491 (267, 492 (312, 493 (291, 494 (292, 495 (293, 496 (294, 497 (296, 498 (297, 499 (298, 500 (299, 501 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))), 502 (268, (37, '$=')), 503 (312, 504 (291, 505 (292, 506 (293, 507 (294, 508 (296, 509 (297, 510 (298, 511 (299, 512 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))), 513 (4, ''))), 514 (0, '')) 515 self.check_bad_tree(tree, "a $= b") 516 517 def test_malformed_global(self): 518 #doesn't have global keyword in ast 519 tree = (257, 520 (264, 521 (265, 522 (266, 523 (282, (1, 'foo'))), (4, ''))), 524 (4, ''), 525 (0, '')) 526 self.check_bad_tree(tree, "malformed global ast") 527 528 def test_missing_import_source(self): 529 # from import a 530 tree = \ 531 (257, 532 (267, 533 (268, 534 (269, 535 (281, 536 (283, (1, 'from'), (1, 'import'), 537 (286, (284, (1, 'fred')))))), 538 (4, ''))), 539 (4, ''), (0, '')) 540 self.check_bad_tree(tree, "from import a") 541 542 543 class CompileTestCase(unittest.TestCase): 544 545 # These tests are very minimal. :-( 546 547 def test_compile_expr(self): 548 st = parser.expr('2 + 3') 549 code = parser.compilest(st) 550 self.assertEqual(eval(code), 5) 551 552 def test_compile_suite(self): 553 st = parser.suite('x = 2; y = x + 3') 554 code = parser.compilest(st) 555 globs = {} 556 exec code in globs 557 self.assertEqual(globs['y'], 5) 558 559 def test_compile_error(self): 560 st = parser.suite('1 = 3 + 4') 561 self.assertRaises(SyntaxError, parser.compilest, st) 562 563 def test_compile_badunicode(self): 564 st = parser.suite('a = u"\U12345678"') 565 self.assertRaises(SyntaxError, parser.compilest, st) 566 st = parser.suite('a = u"\u1"') 567 self.assertRaises(SyntaxError, parser.compilest, st) 568 569 class ParserStackLimitTestCase(unittest.TestCase): 570 """try to push the parser to/over it's limits. 571 see http://bugs.python.org/issue1881 for a discussion 572 """ 573 def _nested_expression(self, level): 574 return "["*level+"]"*level 575 576 def test_deeply_nested_list(self): 577 e = self._nested_expression(99) 578 st = parser.expr(e) 579 st.compile() 580 581 def test_trigger_memory_error(self): 582 e = self._nested_expression(100) 583 print >>sys.stderr, "Expecting 's_push: parser stack overflow' in next line" 584 self.assertRaises(MemoryError, parser.expr, e) 585 586 def test_main(): 587 test_support.run_unittest( 588 RoundtripLegalSyntaxTestCase, 589 IllegalSyntaxTestCase, 590 CompileTestCase, 591 ParserStackLimitTestCase, 592 ) 593 594 595 if __name__ == "__main__": 596 test_main() 597
