1 import copy 2 import parser 3 import pickle 4 import unittest 5 import operator 6 import struct 7 from test import support 8 from test.support.script_helper import assert_python_failure 9 10 # 11 # First, we test that we can generate trees from valid source fragments, 12 # and that these valid trees are indeed allowed by the tree-loading side 13 # of the parser module. 14 # 15 16 class RoundtripLegalSyntaxTestCase(unittest.TestCase): 17 18 def roundtrip(self, f, s): 19 st1 = f(s) 20 t = st1.totuple() 21 try: 22 st2 = parser.sequence2st(t) 23 except parser.ParserError as why: 24 self.fail("could not roundtrip %r: %s" % (s, why)) 25 26 self.assertEqual(t, st2.totuple(), 27 "could not re-generate syntax tree") 28 29 def check_expr(self, s): 30 self.roundtrip(parser.expr, s) 31 32 def test_flags_passed(self): 33 # The unicode literals flags has to be passed from the parser to AST 34 # generation. 35 suite = parser.suite("from __future__ import unicode_literals; x = ''") 36 code = suite.compile() 37 scope = {} 38 exec(code, {}, scope) 39 self.assertIsInstance(scope["x"], str) 40 41 def check_suite(self, s): 42 self.roundtrip(parser.suite, s) 43 44 def test_yield_statement(self): 45 self.check_suite("def f(): yield 1") 46 self.check_suite("def f(): yield") 47 self.check_suite("def f(): x += yield") 48 self.check_suite("def f(): x = yield 1") 49 self.check_suite("def f(): x = y = yield 1") 50 self.check_suite("def f(): x = yield") 51 self.check_suite("def f(): x = y = yield") 52 self.check_suite("def f(): 1 + (yield)*2") 53 self.check_suite("def f(): (yield 1)*2") 54 self.check_suite("def f(): return; yield 1") 55 self.check_suite("def f(): yield 1; return") 56 self.check_suite("def f(): yield from 1") 57 self.check_suite("def f(): x = yield from 1") 58 self.check_suite("def f(): f((yield from 1))") 59 self.check_suite("def f(): yield 1; return 1") 60 self.check_suite("def f():\n" 61 " for x in range(30):\n" 62 " yield x\n") 63 self.check_suite("def f():\n" 64 " if (yield):\n" 65 " yield x\n") 66 67 def test_await_statement(self): 68 self.check_suite("async def f():\n await smth()") 69 self.check_suite("async def f():\n foo = await smth()") 70 self.check_suite("async def f():\n foo, bar = await smth()") 71 self.check_suite("async def f():\n (await smth())") 72 self.check_suite("async def f():\n foo((await smth()))") 73 self.check_suite("async def f():\n await foo(); return 42") 74 75 def test_async_with_statement(self): 76 self.check_suite("async def f():\n async with 1: pass") 77 self.check_suite("async def f():\n async with a as b, c as d: pass") 78 79 def test_async_for_statement(self): 80 self.check_suite("async def f():\n async for i in (): pass") 81 self.check_suite("async def f():\n async for i, b in (): pass") 82 83 def test_nonlocal_statement(self): 84 self.check_suite("def f():\n" 85 " x = 0\n" 86 " def g():\n" 87 " nonlocal x\n") 88 self.check_suite("def f():\n" 89 " x = y = 0\n" 90 " def g():\n" 91 " nonlocal x, y\n") 92 93 def test_expressions(self): 94 self.check_expr("foo(1)") 95 self.check_expr("[1, 2, 3]") 96 self.check_expr("[x**3 for x in range(20)]") 97 self.check_expr("[x**3 for x in range(20) if x % 3]") 98 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]") 99 self.check_expr("list(x**3 for x in range(20))") 100 self.check_expr("list(x**3 for x in range(20) if x % 3)") 101 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)") 102 self.check_expr("foo(*args)") 103 self.check_expr("foo(*args, **kw)") 104 self.check_expr("foo(**kw)") 105 self.check_expr("foo(key=value)") 106 self.check_expr("foo(key=value, *args)") 107 self.check_expr("foo(key=value, *args, **kw)") 108 self.check_expr("foo(key=value, **kw)") 109 self.check_expr("foo(a, b, c, *args)") 110 self.check_expr("foo(a, b, c, *args, **kw)") 111 self.check_expr("foo(a, b, c, **kw)") 112 self.check_expr("foo(a, *args, keyword=23)") 113 self.check_expr("foo + bar") 114 self.check_expr("foo - bar") 115 self.check_expr("foo * bar") 116 self.check_expr("foo / bar") 117 self.check_expr("foo // bar") 118 self.check_expr("lambda: 0") 119 self.check_expr("lambda x: 0") 120 self.check_expr("lambda *y: 0") 121 self.check_expr("lambda *y, **z: 0") 122 self.check_expr("lambda **z: 0") 123 self.check_expr("lambda x, y: 0") 124 self.check_expr("lambda foo=bar: 0") 125 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0") 126 self.check_expr("lambda foo=bar, **z: 0") 127 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0") 128 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0") 129 self.check_expr("lambda x, *y, **z: 0") 130 self.check_expr("(x for x in range(10))") 131 self.check_expr("foo(x for x in range(10))") 132 self.check_expr("...") 133 self.check_expr("a[...]") 134 135 def test_simple_expression(self): 136 # expr_stmt 137 self.check_suite("a") 138 139 def test_simple_assignments(self): 140 self.check_suite("a = b") 141 self.check_suite("a = b = c = d = e") 142 143 def test_var_annot(self): 144 self.check_suite("x: int = 5") 145 self.check_suite("y: List[T] = []; z: [list] = fun()") 146 self.check_suite("x: tuple = (1, 2)") 147 self.check_suite("d[f()]: int = 42") 148 self.check_suite("f(d[x]): str = 'abc'") 149 self.check_suite("x.y.z.w: complex = 42j") 150 self.check_suite("x: int") 151 self.check_suite("def f():\n" 152 " x: str\n" 153 " y: int = 5\n") 154 self.check_suite("class C:\n" 155 " x: str\n" 156 " y: int = 5\n") 157 self.check_suite("class C:\n" 158 " def __init__(self, x: int) -> None:\n" 159 " self.x: int = x\n") 160 # double check for nonsense 161 with self.assertRaises(SyntaxError): 162 exec("2+2: int", {}, {}) 163 with self.assertRaises(SyntaxError): 164 exec("[]: int = 5", {}, {}) 165 with self.assertRaises(SyntaxError): 166 exec("x, *y, z: int = range(5)", {}, {}) 167 with self.assertRaises(SyntaxError): 168 exec("t: tuple = 1, 2", {}, {}) 169 with self.assertRaises(SyntaxError): 170 exec("u = v: int", {}, {}) 171 with self.assertRaises(SyntaxError): 172 exec("False: int", {}, {}) 173 with self.assertRaises(SyntaxError): 174 exec("x.False: int", {}, {}) 175 with self.assertRaises(SyntaxError): 176 exec("x.y,: int", {}, {}) 177 with self.assertRaises(SyntaxError): 178 exec("[0]: int", {}, {}) 179 with self.assertRaises(SyntaxError): 180 exec("f(): int", {}, {}) 181 182 def test_simple_augmented_assignments(self): 183 self.check_suite("a += b") 184 self.check_suite("a -= b") 185 self.check_suite("a *= b") 186 self.check_suite("a /= b") 187 self.check_suite("a //= b") 188 self.check_suite("a %= b") 189 self.check_suite("a &= b") 190 self.check_suite("a |= b") 191 self.check_suite("a ^= b") 192 self.check_suite("a <<= b") 193 self.check_suite("a >>= b") 194 self.check_suite("a **= b") 195 196 def test_function_defs(self): 197 self.check_suite("def f(): pass") 198 self.check_suite("def f(*args): pass") 199 self.check_suite("def f(*args, **kw): pass") 200 self.check_suite("def f(**kw): pass") 201 self.check_suite("def f(foo=bar): pass") 202 self.check_suite("def f(foo=bar, *args): pass") 203 self.check_suite("def f(foo=bar, *args, **kw): pass") 204 self.check_suite("def f(foo=bar, **kw): pass") 205 206 self.check_suite("def f(a, b): pass") 207 self.check_suite("def f(a, b, *args): pass") 208 self.check_suite("def f(a, b, *args, **kw): pass") 209 self.check_suite("def f(a, b, **kw): pass") 210 self.check_suite("def f(a, b, foo=bar): pass") 211 self.check_suite("def f(a, b, foo=bar, *args): pass") 212 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass") 213 self.check_suite("def f(a, b, foo=bar, **kw): pass") 214 215 self.check_suite("@staticmethod\n" 216 "def f(): pass") 217 self.check_suite("@staticmethod\n" 218 "@funcattrs(x, y)\n" 219 "def f(): pass") 220 self.check_suite("@funcattrs()\n" 221 "def f(): pass") 222 223 # keyword-only arguments 224 self.check_suite("def f(*, a): pass") 225 self.check_suite("def f(*, a = 5): pass") 226 self.check_suite("def f(*, a = 5, b): pass") 227 self.check_suite("def f(*, a, b = 5): pass") 228 self.check_suite("def f(*, a, b = 5, **kwds): pass") 229 self.check_suite("def f(*args, a): pass") 230 self.check_suite("def f(*args, a = 5): pass") 231 self.check_suite("def f(*args, a = 5, b): pass") 232 self.check_suite("def f(*args, a, b = 5): pass") 233 self.check_suite("def f(*args, a, b = 5, **kwds): pass") 234 235 # function annotations 236 self.check_suite("def f(a: int): pass") 237 self.check_suite("def f(a: int = 5): pass") 238 self.check_suite("def f(*args: list): pass") 239 self.check_suite("def f(**kwds: dict): pass") 240 self.check_suite("def f(*, a: int): pass") 241 self.check_suite("def f(*, a: int = 5): pass") 242 self.check_suite("def f() -> int: pass") 243 244 def test_class_defs(self): 245 self.check_suite("class foo():pass") 246 self.check_suite("class foo(object):pass") 247 self.check_suite("@class_decorator\n" 248 "class foo():pass") 249 self.check_suite("@class_decorator(arg)\n" 250 "class foo():pass") 251 self.check_suite("@decorator1\n" 252 "@decorator2\n" 253 "class foo():pass") 254 255 def test_import_from_statement(self): 256 self.check_suite("from sys.path import *") 257 self.check_suite("from sys.path import dirname") 258 self.check_suite("from sys.path import (dirname)") 259 self.check_suite("from sys.path import (dirname,)") 260 self.check_suite("from sys.path import dirname as my_dirname") 261 self.check_suite("from sys.path import (dirname as my_dirname)") 262 self.check_suite("from sys.path import (dirname as my_dirname,)") 263 self.check_suite("from sys.path import dirname, basename") 264 self.check_suite("from sys.path import (dirname, basename)") 265 self.check_suite("from sys.path import (dirname, basename,)") 266 self.check_suite( 267 "from sys.path import dirname as my_dirname, basename") 268 self.check_suite( 269 "from sys.path import (dirname as my_dirname, basename)") 270 self.check_suite( 271 "from sys.path import (dirname as my_dirname, basename,)") 272 self.check_suite( 273 "from sys.path import dirname, basename as my_basename") 274 self.check_suite( 275 "from sys.path import (dirname, basename as my_basename)") 276 self.check_suite( 277 "from sys.path import (dirname, basename as my_basename,)") 278 self.check_suite("from .bogus import x") 279 280 def test_basic_import_statement(self): 281 self.check_suite("import sys") 282 self.check_suite("import sys as system") 283 self.check_suite("import sys, math") 284 self.check_suite("import sys as system, math") 285 self.check_suite("import sys, math as my_math") 286 287 def test_relative_imports(self): 288 self.check_suite("from . import name") 289 self.check_suite("from .. import name") 290 # check all the way up to '....', since '...' is tokenized 291 # differently from '.' (it's an ellipsis token). 292 self.check_suite("from ... import name") 293 self.check_suite("from .... import name") 294 self.check_suite("from .pkg import name") 295 self.check_suite("from ..pkg import name") 296 self.check_suite("from ...pkg import name") 297 self.check_suite("from ....pkg import name") 298 299 def test_pep263(self): 300 self.check_suite("# -*- coding: iso-8859-1 -*-\n" 301 "pass\n") 302 303 def test_assert(self): 304 self.check_suite("assert alo < ahi and blo < bhi\n") 305 306 def test_with(self): 307 self.check_suite("with open('x'): pass\n") 308 self.check_suite("with open('x') as f: pass\n") 309 self.check_suite("with open('x') as f, open('y') as g: pass\n") 310 311 def test_try_stmt(self): 312 self.check_suite("try: pass\nexcept: pass\n") 313 self.check_suite("try: pass\nfinally: pass\n") 314 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n") 315 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n" 316 "finally: pass\n") 317 self.check_suite("try: pass\nexcept: pass\nelse: pass\n") 318 self.check_suite("try: pass\nexcept: pass\nelse: pass\n" 319 "finally: pass\n") 320 321 def test_position(self): 322 # An absolutely minimal test of position information. Better 323 # tests would be a big project. 324 code = "def f(x):\n return x + 1" 325 st = parser.suite(code) 326 327 def walk(tree): 328 node_type = tree[0] 329 next = tree[1] 330 if isinstance(next, (tuple, list)): 331 for elt in tree[1:]: 332 for x in walk(elt): 333 yield x 334 else: 335 yield tree 336 337 expected = [ 338 (1, 'def', 1, 0), 339 (1, 'f', 1, 4), 340 (7, '(', 1, 5), 341 (1, 'x', 1, 6), 342 (8, ')', 1, 7), 343 (11, ':', 1, 8), 344 (4, '', 1, 9), 345 (5, '', 2, -1), 346 (1, 'return', 2, 4), 347 (1, 'x', 2, 11), 348 (14, '+', 2, 13), 349 (2, '1', 2, 15), 350 (4, '', 2, 16), 351 (6, '', 2, -1), 352 (4, '', 2, -1), 353 (0, '', 2, -1), 354 ] 355 356 self.assertEqual(list(walk(st.totuple(line_info=True, col_info=True))), 357 expected) 358 self.assertEqual(list(walk(st.totuple())), 359 [(t, n) for t, n, l, c in expected]) 360 self.assertEqual(list(walk(st.totuple(line_info=True))), 361 [(t, n, l) for t, n, l, c in expected]) 362 self.assertEqual(list(walk(st.totuple(col_info=True))), 363 [(t, n, c) for t, n, l, c in expected]) 364 self.assertEqual(list(walk(st.tolist(line_info=True, col_info=True))), 365 [list(x) for x in expected]) 366 self.assertEqual(list(walk(parser.st2tuple(st, line_info=True, 367 col_info=True))), 368 expected) 369 self.assertEqual(list(walk(parser.st2list(st, line_info=True, 370 col_info=True))), 371 [list(x) for x in expected]) 372 373 def test_extended_unpacking(self): 374 self.check_suite("*a = y") 375 self.check_suite("x, *b, = m") 376 self.check_suite("[*a, *b] = y") 377 self.check_suite("for [*x, b] in x: pass") 378 379 def test_raise_statement(self): 380 self.check_suite("raise\n") 381 self.check_suite("raise e\n") 382 self.check_suite("try:\n" 383 " suite\n" 384 "except Exception as e:\n" 385 " raise ValueError from e\n") 386 387 def test_list_displays(self): 388 self.check_expr('[]') 389 self.check_expr('[*{2}, 3, *[4]]') 390 391 def test_set_displays(self): 392 self.check_expr('{*{2}, 3, *[4]}') 393 self.check_expr('{2}') 394 self.check_expr('{2,}') 395 self.check_expr('{2, 3}') 396 self.check_expr('{2, 3,}') 397 398 def test_dict_displays(self): 399 self.check_expr('{}') 400 self.check_expr('{a:b}') 401 self.check_expr('{a:b,}') 402 self.check_expr('{a:b, c:d}') 403 self.check_expr('{a:b, c:d,}') 404 self.check_expr('{**{}}') 405 self.check_expr('{**{}, 3:4, **{5:6, 7:8}}') 406 407 def test_argument_unpacking(self): 408 self.check_expr("f(*a, **b)") 409 self.check_expr('f(a, *b, *c, *d)') 410 self.check_expr('f(**a, **b)') 411 self.check_expr('f(2, *a, *b, **b, **c, **d)') 412 self.check_expr("f(*b, *() or () and (), **{} and {}, **() or {})") 413 414 def test_set_comprehensions(self): 415 self.check_expr('{x for x in seq}') 416 self.check_expr('{f(x) for x in seq}') 417 self.check_expr('{f(x) for x in seq if condition(x)}') 418 419 def test_dict_comprehensions(self): 420 self.check_expr('{x:x for x in seq}') 421 self.check_expr('{x**2:x[3] for x in seq if condition(x)}') 422 self.check_expr('{x:x for x in seq1 for y in seq2 if condition(x, y)}') 423 424 425 # 426 # Second, we take *invalid* trees and make sure we get ParserError 427 # rejections for them. 428 # 429 430 class IllegalSyntaxTestCase(unittest.TestCase): 431 432 def check_bad_tree(self, tree, label): 433 try: 434 parser.sequence2st(tree) 435 except parser.ParserError: 436 pass 437 else: 438 self.fail("did not detect invalid tree for %r" % label) 439 440 def test_junk(self): 441 # not even remotely valid: 442 self.check_bad_tree((1, 2, 3), "<junk>") 443 444 def test_illegal_terminal(self): 445 tree = \ 446 (257, 447 (269, 448 (270, 449 (271, 450 (277, 451 (1,))), 452 (4, ''))), 453 (4, ''), 454 (0, '')) 455 self.check_bad_tree(tree, "too small items in terminal node") 456 tree = \ 457 (257, 458 (269, 459 (270, 460 (271, 461 (277, 462 (1, b'pass'))), 463 (4, ''))), 464 (4, ''), 465 (0, '')) 466 self.check_bad_tree(tree, "non-string second item in terminal node") 467 tree = \ 468 (257, 469 (269, 470 (270, 471 (271, 472 (277, 473 (1, 'pass', '0', 0))), 474 (4, ''))), 475 (4, ''), 476 (0, '')) 477 self.check_bad_tree(tree, "non-integer third item in terminal node") 478 tree = \ 479 (257, 480 (269, 481 (270, 482 (271, 483 (277, 484 (1, 'pass', 0, 0))), 485 (4, ''))), 486 (4, ''), 487 (0, '')) 488 self.check_bad_tree(tree, "too many items in terminal node") 489 490 def test_illegal_yield_1(self): 491 # Illegal yield statement: def f(): return 1; yield 1 492 tree = \ 493 (257, 494 (264, 495 (285, 496 (259, 497 (1, 'def'), 498 (1, 'f'), 499 (260, (7, '('), (8, ')')), 500 (11, ':'), 501 (291, 502 (4, ''), 503 (5, ''), 504 (264, 505 (265, 506 (266, 507 (272, 508 (275, 509 (1, 'return'), 510 (313, 511 (292, 512 (293, 513 (294, 514 (295, 515 (297, 516 (298, 517 (299, 518 (300, 519 (301, 520 (302, (303, (304, (305, (2, '1')))))))))))))))))), 521 (264, 522 (265, 523 (266, 524 (272, 525 (276, 526 (1, 'yield'), 527 (313, 528 (292, 529 (293, 530 (294, 531 (295, 532 (297, 533 (298, 534 (299, 535 (300, 536 (301, 537 (302, 538 (303, (304, (305, (2, '1')))))))))))))))))), 539 (4, ''))), 540 (6, ''))))), 541 (4, ''), 542 (0, '')))) 543 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 544 545 def test_illegal_yield_2(self): 546 # Illegal return in generator: def f(): return 1; yield 1 547 tree = \ 548 (257, 549 (264, 550 (265, 551 (266, 552 (278, 553 (1, 'from'), 554 (281, (1, '__future__')), 555 (1, 'import'), 556 (279, (1, 'generators')))), 557 (4, ''))), 558 (264, 559 (285, 560 (259, 561 (1, 'def'), 562 (1, 'f'), 563 (260, (7, '('), (8, ')')), 564 (11, ':'), 565 (291, 566 (4, ''), 567 (5, ''), 568 (264, 569 (265, 570 (266, 571 (272, 572 (275, 573 (1, 'return'), 574 (313, 575 (292, 576 (293, 577 (294, 578 (295, 579 (297, 580 (298, 581 (299, 582 (300, 583 (301, 584 (302, (303, (304, (305, (2, '1')))))))))))))))))), 585 (264, 586 (265, 587 (266, 588 (272, 589 (276, 590 (1, 'yield'), 591 (313, 592 (292, 593 (293, 594 (294, 595 (295, 596 (297, 597 (298, 598 (299, 599 (300, 600 (301, 601 (302, 602 (303, (304, (305, (2, '1')))))))))))))))))), 603 (4, ''))), 604 (6, ''))))), 605 (4, ''), 606 (0, '')))) 607 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 608 609 def test_a_comma_comma_c(self): 610 # Illegal input: a,,c 611 tree = \ 612 (258, 613 (311, 614 (290, 615 (291, 616 (292, 617 (293, 618 (295, 619 (296, 620 (297, 621 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))), 622 (12, ','), 623 (12, ','), 624 (290, 625 (291, 626 (292, 627 (293, 628 (295, 629 (296, 630 (297, 631 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))), 632 (4, ''), 633 (0, '')) 634 self.check_bad_tree(tree, "a,,c") 635 636 def test_illegal_operator(self): 637 # Illegal input: a $= b 638 tree = \ 639 (257, 640 (264, 641 (265, 642 (266, 643 (267, 644 (312, 645 (291, 646 (292, 647 (293, 648 (294, 649 (296, 650 (297, 651 (298, 652 (299, 653 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))), 654 (268, (37, '$=')), 655 (312, 656 (291, 657 (292, 658 (293, 659 (294, 660 (296, 661 (297, 662 (298, 663 (299, 664 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))), 665 (4, ''))), 666 (0, '')) 667 self.check_bad_tree(tree, "a $= b") 668 669 def test_malformed_global(self): 670 #doesn't have global keyword in ast 671 tree = (257, 672 (264, 673 (265, 674 (266, 675 (282, (1, 'foo'))), (4, ''))), 676 (4, ''), 677 (0, '')) 678 self.check_bad_tree(tree, "malformed global ast") 679 680 def test_missing_import_source(self): 681 # from import fred 682 tree = \ 683 (257, 684 (268, 685 (269, 686 (270, 687 (282, 688 (284, (1, 'from'), (1, 'import'), 689 (287, (285, (1, 'fred')))))), 690 (4, ''))), 691 (4, ''), (0, '')) 692 self.check_bad_tree(tree, "from import fred") 693 694 def test_illegal_encoding(self): 695 # Illegal encoding declaration 696 tree = \ 697 (340, 698 (257, (0, ''))) 699 self.check_bad_tree(tree, "missed encoding") 700 tree = \ 701 (340, 702 (257, (0, '')), 703 b'iso-8859-1') 704 self.check_bad_tree(tree, "non-string encoding") 705 tree = \ 706 (340, 707 (257, (0, '')), 708 '\udcff') 709 with self.assertRaises(UnicodeEncodeError): 710 parser.sequence2st(tree) 711 712 713 class CompileTestCase(unittest.TestCase): 714 715 # These tests are very minimal. :-( 716 717 def test_compile_expr(self): 718 st = parser.expr('2 + 3') 719 code = parser.compilest(st) 720 self.assertEqual(eval(code), 5) 721 722 def test_compile_suite(self): 723 st = parser.suite('x = 2; y = x + 3') 724 code = parser.compilest(st) 725 globs = {} 726 exec(code, globs) 727 self.assertEqual(globs['y'], 5) 728 729 def test_compile_error(self): 730 st = parser.suite('1 = 3 + 4') 731 self.assertRaises(SyntaxError, parser.compilest, st) 732 733 def test_compile_badunicode(self): 734 st = parser.suite('a = "\\U12345678"') 735 self.assertRaises(SyntaxError, parser.compilest, st) 736 st = parser.suite('a = "\\u1"') 737 self.assertRaises(SyntaxError, parser.compilest, st) 738 739 def test_issue_9011(self): 740 # Issue 9011: compilation of an unary minus expression changed 741 # the meaning of the ST, so that a second compilation produced 742 # incorrect results. 743 st = parser.expr('-3') 744 code1 = parser.compilest(st) 745 self.assertEqual(eval(code1), -3) 746 code2 = parser.compilest(st) 747 self.assertEqual(eval(code2), -3) 748 749 def test_compile_filename(self): 750 st = parser.expr('a + 5') 751 code = parser.compilest(st) 752 self.assertEqual(code.co_filename, '<syntax-tree>') 753 code = st.compile() 754 self.assertEqual(code.co_filename, '<syntax-tree>') 755 for filename in 'file.py', b'file.py': 756 code = parser.compilest(st, filename) 757 self.assertEqual(code.co_filename, 'file.py') 758 code = st.compile(filename) 759 self.assertEqual(code.co_filename, 'file.py') 760 for filename in bytearray(b'file.py'), memoryview(b'file.py'): 761 with self.assertWarns(DeprecationWarning): 762 code = parser.compilest(st, filename) 763 self.assertEqual(code.co_filename, 'file.py') 764 with self.assertWarns(DeprecationWarning): 765 code = st.compile(filename) 766 self.assertEqual(code.co_filename, 'file.py') 767 self.assertRaises(TypeError, parser.compilest, st, list(b'file.py')) 768 self.assertRaises(TypeError, st.compile, list(b'file.py')) 769 770 771 class ParserStackLimitTestCase(unittest.TestCase): 772 """try to push the parser to/over its limits. 773 see http://bugs.python.org/issue1881 for a discussion 774 """ 775 def _nested_expression(self, level): 776 return "["*level+"]"*level 777 778 def test_deeply_nested_list(self): 779 # XXX used to be 99 levels in 2.x 780 e = self._nested_expression(93) 781 st = parser.expr(e) 782 st.compile() 783 784 def test_trigger_memory_error(self): 785 e = self._nested_expression(100) 786 rc, out, err = assert_python_failure('-c', e) 787 # parsing the expression will result in an error message 788 # followed by a MemoryError (see #11963) 789 self.assertIn(b's_push: parser stack overflow', err) 790 self.assertIn(b'MemoryError', err) 791 792 class STObjectTestCase(unittest.TestCase): 793 """Test operations on ST objects themselves""" 794 795 def test_comparisons(self): 796 # ST objects should support order and equality comparisons 797 st1 = parser.expr('2 + 3') 798 st2 = parser.suite('x = 2; y = x + 3') 799 st3 = parser.expr('list(x**3 for x in range(20))') 800 st1_copy = parser.expr('2 + 3') 801 st2_copy = parser.suite('x = 2; y = x + 3') 802 st3_copy = parser.expr('list(x**3 for x in range(20))') 803 804 # exercise fast path for object identity 805 self.assertEqual(st1 == st1, True) 806 self.assertEqual(st2 == st2, True) 807 self.assertEqual(st3 == st3, True) 808 # slow path equality 809 self.assertEqual(st1, st1_copy) 810 self.assertEqual(st2, st2_copy) 811 self.assertEqual(st3, st3_copy) 812 self.assertEqual(st1 == st2, False) 813 self.assertEqual(st1 == st3, False) 814 self.assertEqual(st2 == st3, False) 815 self.assertEqual(st1 != st1, False) 816 self.assertEqual(st2 != st2, False) 817 self.assertEqual(st3 != st3, False) 818 self.assertEqual(st1 != st1_copy, False) 819 self.assertEqual(st2 != st2_copy, False) 820 self.assertEqual(st3 != st3_copy, False) 821 self.assertEqual(st2 != st1, True) 822 self.assertEqual(st1 != st3, True) 823 self.assertEqual(st3 != st2, True) 824 # we don't particularly care what the ordering is; just that 825 # it's usable and self-consistent 826 self.assertEqual(st1 < st2, not (st2 <= st1)) 827 self.assertEqual(st1 < st3, not (st3 <= st1)) 828 self.assertEqual(st2 < st3, not (st3 <= st2)) 829 self.assertEqual(st1 < st2, st2 > st1) 830 self.assertEqual(st1 < st3, st3 > st1) 831 self.assertEqual(st2 < st3, st3 > st2) 832 self.assertEqual(st1 <= st2, st2 >= st1) 833 self.assertEqual(st3 <= st1, st1 >= st3) 834 self.assertEqual(st2 <= st3, st3 >= st2) 835 # transitivity 836 bottom = min(st1, st2, st3) 837 top = max(st1, st2, st3) 838 mid = sorted([st1, st2, st3])[1] 839 self.assertTrue(bottom < mid) 840 self.assertTrue(bottom < top) 841 self.assertTrue(mid < top) 842 self.assertTrue(bottom <= mid) 843 self.assertTrue(bottom <= top) 844 self.assertTrue(mid <= top) 845 self.assertTrue(bottom <= bottom) 846 self.assertTrue(mid <= mid) 847 self.assertTrue(top <= top) 848 # interaction with other types 849 self.assertEqual(st1 == 1588.602459, False) 850 self.assertEqual('spanish armada' != st2, True) 851 self.assertRaises(TypeError, operator.ge, st3, None) 852 self.assertRaises(TypeError, operator.le, False, st1) 853 self.assertRaises(TypeError, operator.lt, st1, 1815) 854 self.assertRaises(TypeError, operator.gt, b'waterloo', st2) 855 856 def test_copy_pickle(self): 857 sts = [ 858 parser.expr('2 + 3'), 859 parser.suite('x = 2; y = x + 3'), 860 parser.expr('list(x**3 for x in range(20))') 861 ] 862 for st in sts: 863 st_copy = copy.copy(st) 864 self.assertEqual(st_copy.totuple(), st.totuple()) 865 st_copy = copy.deepcopy(st) 866 self.assertEqual(st_copy.totuple(), st.totuple()) 867 for proto in range(pickle.HIGHEST_PROTOCOL+1): 868 st_copy = pickle.loads(pickle.dumps(st, proto)) 869 self.assertEqual(st_copy.totuple(), st.totuple()) 870 871 check_sizeof = support.check_sizeof 872 873 @support.cpython_only 874 def test_sizeof(self): 875 def XXXROUNDUP(n): 876 if n <= 1: 877 return n 878 if n <= 128: 879 return (n + 3) & ~3 880 return 1 << (n - 1).bit_length() 881 882 basesize = support.calcobjsize('Pii') 883 nodesize = struct.calcsize('hP3iP0h') 884 def sizeofchildren(node): 885 if node is None: 886 return 0 887 res = 0 888 hasstr = len(node) > 1 and isinstance(node[-1], str) 889 if hasstr: 890 res += len(node[-1]) + 1 891 children = node[1:-1] if hasstr else node[1:] 892 if children: 893 res += XXXROUNDUP(len(children)) * nodesize 894 for child in children: 895 res += sizeofchildren(child) 896 return res 897 898 def check_st_sizeof(st): 899 self.check_sizeof(st, basesize + nodesize + 900 sizeofchildren(st.totuple())) 901 902 check_st_sizeof(parser.expr('2 + 3')) 903 check_st_sizeof(parser.expr('2 + 3 + 4')) 904 check_st_sizeof(parser.suite('x = 2 + 3')) 905 check_st_sizeof(parser.suite('')) 906 check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-')) 907 check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']')) 908 909 910 # XXX tests for pickling and unpickling of ST objects should go here 911 912 class OtherParserCase(unittest.TestCase): 913 914 def test_two_args_to_expr(self): 915 # See bug #12264 916 with self.assertRaises(TypeError): 917 parser.expr("a", "b") 918 919 if __name__ == "__main__": 920 unittest.main() 921
