Home | History | Annotate | Download | only in test 
      1 import parser
      2 import unittest
      3 import sys
      4 import struct
      5 from test import test_support as support
      6 from test.script_helper import assert_python_failure
      7 
      8 #
      9 #  First, we test that we can generate trees from valid source fragments,
     10 #  and that these valid trees are indeed allowed by the tree-loading side
     11 #  of the parser module.
     12 #
     13 
     14 class RoundtripLegalSyntaxTestCase(unittest.TestCase):
     15 
     16     def roundtrip(self, f, s):
     17         st1 = f(s)
     18         t = st1.totuple()
     19         try:
     20             st2 = parser.sequence2st(t)
     21         except parser.ParserError, why:
     22             self.fail("could not roundtrip %r: %s" % (s, why))
     23 
     24         self.assertEqual(t, st2.totuple(),
     25                          "could not re-generate syntax tree")
     26 
     27     def check_expr(self, s):
     28         self.roundtrip(parser.expr, s)
     29 
     30     def test_flags_passed(self):
     31         # The unicode literals flags has to be passed from the paser to AST
     32         # generation.
     33         suite = parser.suite("from __future__ import unicode_literals; x = ''")
     34         code = suite.compile()
     35         scope = {}
     36         exec code in scope
     37         self.assertIsInstance(scope["x"], unicode)
     38 
     39     def check_suite(self, s):
     40         self.roundtrip(parser.suite, s)
     41 
     42     def test_yield_statement(self):
     43         self.check_suite("def f(): yield 1")
     44         self.check_suite("def f(): yield")
     45         self.check_suite("def f(): x += yield")
     46         self.check_suite("def f(): x = yield 1")
     47         self.check_suite("def f(): x = y = yield 1")
     48         self.check_suite("def f(): x = yield")
     49         self.check_suite("def f(): x = y = yield")
     50         self.check_suite("def f(): 1 + (yield)*2")
     51         self.check_suite("def f(): (yield 1)*2")
     52         self.check_suite("def f(): return; yield 1")
     53         self.check_suite("def f(): yield 1; return")
     54         self.check_suite("def f():\n"
     55                          "    for x in range(30):\n"
     56                          "        yield x\n")
     57         self.check_suite("def f():\n"
     58                          "    if (yield):\n"
     59                          "        yield x\n")
     60 
     61     def test_expressions(self):
     62         self.check_expr("foo(1)")
     63         self.check_expr("{1:1}")
     64         self.check_expr("{1:1, 2:2, 3:3}")
     65         self.check_expr("{1:1, 2:2, 3:3,}")
     66         self.check_expr("{1}")
     67         self.check_expr("{1, 2, 3}")
     68         self.check_expr("{1, 2, 3,}")
     69         self.check_expr("[]")
     70         self.check_expr("[1]")
     71         self.check_expr("[1, 2, 3]")
     72         self.check_expr("[1, 2, 3,]")
     73         self.check_expr("()")
     74         self.check_expr("(1,)")
     75         self.check_expr("(1, 2, 3)")
     76         self.check_expr("(1, 2, 3,)")
     77         self.check_expr("[x**3 for x in range(20)]")
     78         self.check_expr("[x**3 for x in range(20) if x % 3]")
     79         self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
     80         self.check_expr("[x+y for x in range(30) for y in range(20) if x % 2 if y % 3]")
     81         #self.check_expr("[x for x in lambda: True, lambda: False if x()]")
     82         self.check_expr("list(x**3 for x in range(20))")
     83         self.check_expr("list(x**3 for x in range(20) if x % 3)")
     84         self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
     85         self.check_expr("list(x+y for x in range(30) for y in range(20) if x % 2 if y % 3)")
     86         self.check_expr("{x**3 for x in range(30)}")
     87         self.check_expr("{x**3 for x in range(30) if x % 3}")
     88         self.check_expr("{x**3 for x in range(30) if x % 2 if x % 3}")
     89         self.check_expr("{x+y for x in range(30) for y in range(20) if x % 2 if y % 3}")
     90         self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30))}")
     91         self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3}")
     92         self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3 if y % 3}")
     93         self.check_expr("{x:y for x in range(30) for y in range(20) if x % 2 if y % 3}")
     94         self.check_expr("foo(*args)")
     95         self.check_expr("foo(*args, **kw)")
     96         self.check_expr("foo(**kw)")
     97         self.check_expr("foo(key=value)")
     98         self.check_expr("foo(key=value, *args)")
     99         self.check_expr("foo(key=value, *args, **kw)")
    100         self.check_expr("foo(key=value, **kw)")
    101         self.check_expr("foo(a, b, c, *args)")
    102         self.check_expr("foo(a, b, c, *args, **kw)")
    103         self.check_expr("foo(a, b, c, **kw)")
    104         self.check_expr("foo(a, *args, keyword=23)")
    105         self.check_expr("foo + bar")
    106         self.check_expr("foo - bar")
    107         self.check_expr("foo * bar")
    108         self.check_expr("foo / bar")
    109         self.check_expr("foo // bar")
    110         self.check_expr("lambda: 0")
    111         self.check_expr("lambda x: 0")
    112         self.check_expr("lambda *y: 0")
    113         self.check_expr("lambda *y, **z: 0")
    114         self.check_expr("lambda **z: 0")
    115         self.check_expr("lambda x, y: 0")
    116         self.check_expr("lambda foo=bar: 0")
    117         self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
    118         self.check_expr("lambda foo=bar, **z: 0")
    119         self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
    120         self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
    121         self.check_expr("lambda x, *y, **z: 0")
    122         self.check_expr("lambda x: 5 if x else 2")
    123         self.check_expr("(x for x in range(10))")
    124         self.check_expr("foo(x for x in range(10))")
    125 
    126     def test_print(self):
    127         self.check_suite("print")
    128         self.check_suite("print 1")
    129         self.check_suite("print 1,")
    130         self.check_suite("print >>fp")
    131         self.check_suite("print >>fp, 1")
    132         self.check_suite("print >>fp, 1,")
    133 
    134     def test_simple_expression(self):
    135         # expr_stmt
    136         self.check_suite("a")
    137 
    138     def test_simple_assignments(self):
    139         self.check_suite("a = b")
    140         self.check_suite("a = b = c = d = e")
    141 
    142     def test_simple_augmented_assignments(self):
    143         self.check_suite("a += b")
    144         self.check_suite("a -= b")
    145         self.check_suite("a *= b")
    146         self.check_suite("a /= b")
    147         self.check_suite("a //= b")
    148         self.check_suite("a %= b")
    149         self.check_suite("a &= b")
    150         self.check_suite("a |= b")
    151         self.check_suite("a ^= b")
    152         self.check_suite("a <<= b")
    153         self.check_suite("a >>= b")
    154         self.check_suite("a **= b")
    155 
    156     def test_function_defs(self):
    157         self.check_suite("def f(): pass")
    158         self.check_suite("def f(*args): pass")
    159         self.check_suite("def f(*args, **kw): pass")
    160         self.check_suite("def f(**kw): pass")
    161         self.check_suite("def f(foo=bar): pass")
    162         self.check_suite("def f(foo=bar, *args): pass")
    163         self.check_suite("def f(foo=bar, *args, **kw): pass")
    164         self.check_suite("def f(foo=bar, **kw): pass")
    165 
    166         self.check_suite("def f(a, b): pass")
    167         self.check_suite("def f(a, b, *args): pass")
    168         self.check_suite("def f(a, b, *args, **kw): pass")
    169         self.check_suite("def f(a, b, **kw): pass")
    170         self.check_suite("def f(a, b, foo=bar): pass")
    171         self.check_suite("def f(a, b, foo=bar, *args): pass")
    172         self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
    173         self.check_suite("def f(a, b, foo=bar, **kw): pass")
    174 
    175         self.check_suite("@staticmethod\n"
    176                          "def f(): pass")
    177         self.check_suite("@staticmethod\n"
    178                          "@funcattrs(x, y)\n"
    179                          "def f(): pass")
    180         self.check_suite("@funcattrs()\n"
    181                          "def f(): pass")
    182 
    183     def test_class_defs(self):
    184         self.check_suite("class foo():pass")
    185         self.check_suite("@class_decorator\n"
    186                          "class foo():pass")
    187         self.check_suite("@class_decorator(arg)\n"
    188                          "class foo():pass")
    189         self.check_suite("@decorator1\n"
    190                          "@decorator2\n"
    191                          "class foo():pass")
    192 
    193 
    194     def test_import_from_statement(self):
    195         self.check_suite("from sys.path import *")
    196         self.check_suite("from sys.path import dirname")
    197         self.check_suite("from sys.path import (dirname)")
    198         self.check_suite("from sys.path import (dirname,)")
    199         self.check_suite("from sys.path import dirname as my_dirname")
    200         self.check_suite("from sys.path import (dirname as my_dirname)")
    201         self.check_suite("from sys.path import (dirname as my_dirname,)")
    202         self.check_suite("from sys.path import dirname, basename")
    203         self.check_suite("from sys.path import (dirname, basename)")
    204         self.check_suite("from sys.path import (dirname, basename,)")
    205         self.check_suite(
    206             "from sys.path import dirname as my_dirname, basename")
    207         self.check_suite(
    208             "from sys.path import (dirname as my_dirname, basename)")
    209         self.check_suite(
    210             "from sys.path import (dirname as my_dirname, basename,)")
    211         self.check_suite(
    212             "from sys.path import dirname, basename as my_basename")
    213         self.check_suite(
    214             "from sys.path import (dirname, basename as my_basename)")
    215         self.check_suite(
    216             "from sys.path import (dirname, basename as my_basename,)")
    217         self.check_suite("from .bogus import x")
    218 
    219     def test_basic_import_statement(self):
    220         self.check_suite("import sys")
    221         self.check_suite("import sys as system")
    222         self.check_suite("import sys, math")
    223         self.check_suite("import sys as system, math")
    224         self.check_suite("import sys, math as my_math")
    225 
    226     def test_relative_imports(self):
    227         self.check_suite("from . import name")
    228         self.check_suite("from .. import name")
    229         self.check_suite("from .pkg import name")
    230         self.check_suite("from ..pkg import name")
    231 
    232     def test_pep263(self):
    233         self.check_suite("# -*- coding: iso-8859-1 -*-\n"
    234                          "pass\n")
    235 
    236     def test_assert(self):
    237         self.check_suite("assert alo < ahi and blo < bhi\n")
    238 
    239     def test_with(self):
    240         self.check_suite("with open('x'): pass\n")
    241         self.check_suite("with open('x') as f: pass\n")
    242         self.check_suite("with open('x') as f, open('y') as g: pass\n")
    243 
    244     def test_try_stmt(self):
    245         self.check_suite("try: pass\nexcept: pass\n")
    246         self.check_suite("try: pass\nfinally: pass\n")
    247         self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
    248         self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
    249                          "finally: pass\n")
    250         self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
    251         self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
    252                          "finally: pass\n")
    253 
    254     def test_except_clause(self):
    255         self.check_suite("try: pass\nexcept: pass\n")
    256         self.check_suite("try: pass\nexcept A: pass\n")
    257         self.check_suite("try: pass\nexcept A, e: pass\n")
    258         self.check_suite("try: pass\nexcept A as e: pass\n")
    259 
    260     def test_position(self):
    261         # An absolutely minimal test of position information.  Better
    262         # tests would be a big project.
    263         code = "def f(x):\n    return x + 1"
    264         st1 = parser.suite(code)
    265         st2 = st1.totuple(line_info=1, col_info=1)
    266 
    267         def walk(tree):
    268             node_type = tree[0]
    269             next = tree[1]
    270             if isinstance(next, tuple):
    271                 for elt in tree[1:]:
    272                     for x in walk(elt):
    273                         yield x
    274             else:
    275                 yield tree
    276 
    277         terminals = list(walk(st2))
    278         self.assertEqual([
    279             (1, 'def', 1, 0),
    280             (1, 'f', 1, 4),
    281             (7, '(', 1, 5),
    282             (1, 'x', 1, 6),
    283             (8, ')', 1, 7),
    284             (11, ':', 1, 8),
    285             (4, '', 1, 9),
    286             (5, '', 2, -1),
    287             (1, 'return', 2, 4),
    288             (1, 'x', 2, 11),
    289             (14, '+', 2, 13),
    290             (2, '1', 2, 15),
    291             (4, '', 2, 16),
    292             (6, '', 2, -1),
    293             (4, '', 2, -1),
    294             (0, '', 2, -1)],
    295                          terminals)
    296 
    297 
    298 #
    299 #  Second, we take *invalid* trees and make sure we get ParserError
    300 #  rejections for them.
    301 #
    302 
    303 class IllegalSyntaxTestCase(unittest.TestCase):
    304 
    305     def check_bad_tree(self, tree, label):
    306         try:
    307             parser.sequence2st(tree)
    308         except parser.ParserError:
    309             pass
    310         else:
    311             self.fail("did not detect invalid tree for %r" % label)
    312 
    313     def test_junk(self):
    314         # not even remotely valid:
    315         self.check_bad_tree((1, 2, 3), "<junk>")
    316 
    317     def test_illegal_yield_1(self):
    318         # Illegal yield statement: def f(): return 1; yield 1
    319         tree = \
    320         (257,
    321          (264,
    322           (285,
    323            (259,
    324             (1, 'def'),
    325             (1, 'f'),
    326             (260, (7, '('), (8, ')')),
    327             (11, ':'),
    328             (291,
    329              (4, ''),
    330              (5, ''),
    331              (264,
    332               (265,
    333                (266,
    334                 (272,
    335                  (275,
    336                   (1, 'return'),
    337                   (313,
    338                    (292,
    339                     (293,
    340                      (294,
    341                       (295,
    342                        (297,
    343                         (298,
    344                          (299,
    345                           (300,
    346                            (301,
    347                             (302, (303, (304, (305, (2, '1')))))))))))))))))),
    348                (264,
    349                 (265,
    350                  (266,
    351                   (272,
    352                    (276,
    353                     (1, 'yield'),
    354                     (313,
    355                      (292,
    356                       (293,
    357                        (294,
    358                         (295,
    359                          (297,
    360                           (298,
    361                            (299,
    362                             (300,
    363                              (301,
    364                               (302,
    365                                (303, (304, (305, (2, '1')))))))))))))))))),
    366                  (4, ''))),
    367                (6, ''))))),
    368            (4, ''),
    369            (0, ''))))
    370         self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
    371 
    372     def test_illegal_yield_2(self):
    373         # Illegal return in generator: def f(): return 1; yield 1
    374         tree = \
    375         (257,
    376          (264,
    377           (265,
    378            (266,
    379             (278,
    380              (1, 'from'),
    381              (281, (1, '__future__')),
    382              (1, 'import'),
    383              (279, (1, 'generators')))),
    384            (4, ''))),
    385          (264,
    386           (285,
    387            (259,
    388             (1, 'def'),
    389             (1, 'f'),
    390             (260, (7, '('), (8, ')')),
    391             (11, ':'),
    392             (291,
    393              (4, ''),
    394              (5, ''),
    395              (264,
    396               (265,
    397                (266,
    398                 (272,
    399                  (275,
    400                   (1, 'return'),
    401                   (313,
    402                    (292,
    403                     (293,
    404                      (294,
    405                       (295,
    406                        (297,
    407                         (298,
    408                          (299,
    409                           (300,
    410                            (301,
    411                             (302, (303, (304, (305, (2, '1')))))))))))))))))),
    412                (264,
    413                 (265,
    414                  (266,
    415                   (272,
    416                    (276,
    417                     (1, 'yield'),
    418                     (313,
    419                      (292,
    420                       (293,
    421                        (294,
    422                         (295,
    423                          (297,
    424                           (298,
    425                            (299,
    426                             (300,
    427                              (301,
    428                               (302,
    429                                (303, (304, (305, (2, '1')))))))))))))))))),
    430                  (4, ''))),
    431                (6, ''))))),
    432            (4, ''),
    433            (0, ''))))
    434         self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
    435 
    436     def test_print_chevron_comma(self):
    437         # Illegal input: print >>fp,
    438         tree = \
    439         (257,
    440          (264,
    441           (265,
    442            (266,
    443             (268,
    444              (1, 'print'),
    445              (35, '>>'),
    446              (290,
    447               (291,
    448                (292,
    449                 (293,
    450                  (295,
    451                   (296,
    452                    (297,
    453                     (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
    454              (12, ','))),
    455            (4, ''))),
    456          (0, ''))
    457         self.check_bad_tree(tree, "print >>fp,")
    458 
    459     def test_a_comma_comma_c(self):
    460         # Illegal input: a,,c
    461         tree = \
    462         (258,
    463          (311,
    464           (290,
    465            (291,
    466             (292,
    467              (293,
    468               (295,
    469                (296,
    470                 (297,
    471                  (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
    472           (12, ','),
    473           (12, ','),
    474           (290,
    475            (291,
    476             (292,
    477              (293,
    478               (295,
    479                (296,
    480                 (297,
    481                  (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
    482          (4, ''),
    483          (0, ''))
    484         self.check_bad_tree(tree, "a,,c")
    485 
    486     def test_illegal_operator(self):
    487         # Illegal input: a $= b
    488         tree = \
    489         (257,
    490          (264,
    491           (265,
    492            (266,
    493             (267,
    494              (312,
    495               (291,
    496                (292,
    497                 (293,
    498                  (294,
    499                   (296,
    500                    (297,
    501                     (298,
    502                      (299,
    503                       (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
    504              (268, (37, '$=')),
    505              (312,
    506               (291,
    507                (292,
    508                 (293,
    509                  (294,
    510                   (296,
    511                    (297,
    512                     (298,
    513                      (299,
    514                       (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
    515            (4, ''))),
    516          (0, ''))
    517         self.check_bad_tree(tree, "a $= b")
    518 
    519     def test_malformed_global(self):
    520         #doesn't have global keyword in ast
    521         tree = (257,
    522                 (264,
    523                  (265,
    524                   (266,
    525                    (282, (1, 'foo'))), (4, ''))),
    526                 (4, ''),
    527                 (0, ''))
    528         self.check_bad_tree(tree, "malformed global ast")
    529 
    530     def test_missing_import_source(self):
    531         # from import a
    532         tree = \
    533             (257,
    534              (267,
    535               (268,
    536                (269,
    537                 (281,
    538                  (283, (1, 'from'), (1, 'import'),
    539                   (286, (284, (1, 'fred')))))),
    540                (4, ''))),
    541              (4, ''), (0, ''))
    542         self.check_bad_tree(tree, "from import a")
    543 
    544 
    545 class CompileTestCase(unittest.TestCase):
    546 
    547     # These tests are very minimal. :-(
    548 
    549     def test_compile_expr(self):
    550         st = parser.expr('2 + 3')
    551         code = parser.compilest(st)
    552         self.assertEqual(eval(code), 5)
    553 
    554     def test_compile_suite(self):
    555         st = parser.suite('x = 2; y = x + 3')
    556         code = parser.compilest(st)
    557         globs = {}
    558         exec code in globs
    559         self.assertEqual(globs['y'], 5)
    560 
    561     def test_compile_error(self):
    562         st = parser.suite('1 = 3 + 4')
    563         self.assertRaises(SyntaxError, parser.compilest, st)
    564 
    565     def test_compile_badunicode(self):
    566         st = parser.suite('a = u"\U12345678"')
    567         self.assertRaises(SyntaxError, parser.compilest, st)
    568         st = parser.suite('a = u"\u1"')
    569         self.assertRaises(SyntaxError, parser.compilest, st)
    570 
    571     def test_issue_9011(self):
    572         # Issue 9011: compilation of an unary minus expression changed
    573         # the meaning of the ST, so that a second compilation produced
    574         # incorrect results.
    575         st = parser.expr('-3')
    576         code1 = parser.compilest(st)
    577         self.assertEqual(eval(code1), -3)
    578         code2 = parser.compilest(st)
    579         self.assertEqual(eval(code2), -3)
    580 
    581 
    582 class ParserStackLimitTestCase(unittest.TestCase):
    583     """try to push the parser to/over its limits.
    584     see http://bugs.python.org/issue1881 for a discussion
    585     """
    586     def _nested_expression(self, level):
    587         return "["*level+"]"*level
    588 
    589     def test_deeply_nested_list(self):
    590         e = self._nested_expression(99)
    591         st = parser.expr(e)
    592         st.compile()
    593 
    594     def test_trigger_memory_error(self):
    595         e = self._nested_expression(100)
    596         rc, out, err = assert_python_failure('-c', e)
    597         # parsing the expression will result in an error message
    598         # followed by a MemoryError (see #11963)
    599         self.assertIn(b's_push: parser stack overflow', err)
    600         self.assertIn(b'MemoryError', err)
    601 
    602 class STObjectTestCase(unittest.TestCase):
    603     """Test operations on ST objects themselves"""
    604 
    605     check_sizeof = support.check_sizeof
    606 
    607     @support.cpython_only
    608     def test_sizeof(self):
    609         def XXXROUNDUP(n):
    610             if n <= 1:
    611                 return n
    612             if n <= 128:
    613                 return (n + 3) & ~3
    614             return 1 << (n - 1).bit_length()
    615 
    616         basesize = support.calcobjsize('Pii')
    617         nodesize = struct.calcsize('hP3iP0h')
    618         def sizeofchildren(node):
    619             if node is None:
    620                 return 0
    621             res = 0
    622             hasstr = len(node) > 1 and isinstance(node[-1], str)
    623             if hasstr:
    624                 res += len(node[-1]) + 1
    625             children = node[1:-1] if hasstr else node[1:]
    626             if children:
    627                 res += XXXROUNDUP(len(children)) * nodesize
    628                 for child in children:
    629                     res += sizeofchildren(child)
    630             return res
    631 
    632         def check_st_sizeof(st):
    633             self.check_sizeof(st, basesize + nodesize +
    634                                   sizeofchildren(st.totuple()))
    635 
    636         check_st_sizeof(parser.expr('2 + 3'))
    637         check_st_sizeof(parser.expr('2 + 3 + 4'))
    638         check_st_sizeof(parser.suite('x = 2 + 3'))
    639         check_st_sizeof(parser.suite(''))
    640         check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-'))
    641         check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']'))
    642 
    643 
    644     # XXX tests for pickling and unpickling of ST objects should go here
    645 
    646 def test_main():
    647     support.run_unittest(
    648         RoundtripLegalSyntaxTestCase,
    649         IllegalSyntaxTestCase,
    650         CompileTestCase,
    651         ParserStackLimitTestCase,
    652         STObjectTestCase,
    653     )
    654 
    655 
    656 if __name__ == "__main__":
    657     test_main()
    658