1 """Test suite for 2to3's parser and grammar files. 2 3 This is the place to add tests for changes to 2to3's grammar, such as those 4 merging the grammars for Python 2 and 3. In addition to specific tests for 5 parts of the grammar we've changed, we also make sure we can parse the 6 test_grammar.py files from both Python 2 and Python 3. 7 """ 8 9 # Testing imports 10 from . import support 11 from .support import driver, driver_no_print_statement 12 13 # Python imports 14 import difflib 15 import importlib 16 import operator 17 import os 18 import pickle 19 import shutil 20 import subprocess 21 import sys 22 import tempfile 23 import unittest 24 25 # Local imports 26 from lib2to3.pgen2 import driver as pgen2_driver 27 from lib2to3.pgen2 import tokenize 28 from ..pgen2.parse import ParseError 29 from lib2to3.pygram import python_symbols as syms 30 31 32 class TestDriver(support.TestCase): 33 34 def test_formfeed(self): 35 s = """print 1\n\x0Cprint 2\n""" 36 t = driver.parse_string(s) 37 self.assertEqual(t.children[0].children[0].type, syms.print_stmt) 38 self.assertEqual(t.children[1].children[0].type, syms.print_stmt) 39 40 41 class TestPgen2Caching(support.TestCase): 42 def test_load_grammar_from_txt_file(self): 43 pgen2_driver.load_grammar(support.grammar_path, save=False, force=True) 44 45 def test_load_grammar_from_pickle(self): 46 # Make a copy of the grammar file in a temp directory we are 47 # guaranteed to be able to write to. 48 tmpdir = tempfile.mkdtemp() 49 try: 50 grammar_copy = os.path.join( 51 tmpdir, os.path.basename(support.grammar_path)) 52 shutil.copy(support.grammar_path, grammar_copy) 53 pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) 54 55 pgen2_driver.load_grammar(grammar_copy, save=True, force=True) 56 self.assertTrue(os.path.exists(pickle_name)) 57 58 os.unlink(grammar_copy) # Only the pickle remains... 59 pgen2_driver.load_grammar(grammar_copy, save=False, force=False) 60 finally: 61 shutil.rmtree(tmpdir) 62 63 @unittest.skipIf(sys.executable is None, 'sys.executable required') 64 def test_load_grammar_from_subprocess(self): 65 tmpdir = tempfile.mkdtemp() 66 tmpsubdir = os.path.join(tmpdir, 'subdir') 67 try: 68 os.mkdir(tmpsubdir) 69 grammar_base = os.path.basename(support.grammar_path) 70 grammar_copy = os.path.join(tmpdir, grammar_base) 71 grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) 72 shutil.copy(support.grammar_path, grammar_copy) 73 shutil.copy(support.grammar_path, grammar_sub_copy) 74 pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) 75 pickle_sub_name = pgen2_driver._generate_pickle_name( 76 grammar_sub_copy) 77 self.assertNotEqual(pickle_name, pickle_sub_name) 78 79 # Generate a pickle file from this process. 80 pgen2_driver.load_grammar(grammar_copy, save=True, force=True) 81 self.assertTrue(os.path.exists(pickle_name)) 82 83 # Generate a new pickle file in a subprocess with a most likely 84 # different hash randomization seed. 85 sub_env = dict(os.environ) 86 sub_env['PYTHONHASHSEED'] = 'random' 87 subprocess.check_call( 88 [sys.executable, '-c', """ 89 from lib2to3.pgen2 import driver as pgen2_driver 90 pgen2_driver.load_grammar(%r, save=True, force=True) 91 """ % (grammar_sub_copy,)], 92 env=sub_env) 93 self.assertTrue(os.path.exists(pickle_sub_name)) 94 95 with open(pickle_name, 'rb') as pickle_f_1, \ 96 open(pickle_sub_name, 'rb') as pickle_f_2: 97 self.assertEqual( 98 pickle_f_1.read(), pickle_f_2.read(), 99 msg='Grammar caches generated using different hash seeds' 100 ' were not identical.') 101 finally: 102 shutil.rmtree(tmpdir) 103 104 def test_load_packaged_grammar(self): 105 modname = __name__ + '.load_test' 106 class MyLoader: 107 def get_data(self, where): 108 return pickle.dumps({'elephant': 19}) 109 class MyModule: 110 __file__ = 'parsertestmodule' 111 __spec__ = importlib.util.spec_from_loader(modname, MyLoader()) 112 sys.modules[modname] = MyModule() 113 self.addCleanup(operator.delitem, sys.modules, modname) 114 g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt') 115 self.assertEqual(g.elephant, 19) 116 117 118 class GrammarTest(support.TestCase): 119 def validate(self, code): 120 support.parse_string(code) 121 122 def invalid_syntax(self, code): 123 try: 124 self.validate(code) 125 except ParseError: 126 pass 127 else: 128 raise AssertionError("Syntax shouldn't have been valid") 129 130 131 class TestMatrixMultiplication(GrammarTest): 132 def test_matrix_multiplication_operator(self): 133 self.validate("a @ b") 134 self.validate("a @= b") 135 136 137 class TestYieldFrom(GrammarTest): 138 def test_yield_from(self): 139 self.validate("yield from x") 140 self.validate("(yield from x) + y") 141 self.invalid_syntax("yield from") 142 143 144 class TestAsyncAwait(GrammarTest): 145 def test_await_expr(self): 146 self.validate("""async def foo(): 147 await x 148 """) 149 150 self.validate("""async def foo(): 151 [i async for i in b] 152 """) 153 154 self.validate("""async def foo(): 155 {i for i in b 156 async for i in a if await i 157 for b in i} 158 """) 159 160 self.validate("""async def foo(): 161 [await i for i in b if await c] 162 """) 163 164 self.validate("""async def foo(): 165 [ i for i in b if c] 166 """) 167 168 self.validate("""async def foo(): 169 170 def foo(): pass 171 172 def foo(): pass 173 174 await x 175 """) 176 177 self.validate("""async def foo(): return await a""") 178 179 self.validate("""def foo(): 180 def foo(): pass 181 async def foo(): await x 182 """) 183 184 self.invalid_syntax("await x") 185 self.invalid_syntax("""def foo(): 186 await x""") 187 188 self.invalid_syntax("""def foo(): 189 def foo(): pass 190 async def foo(): pass 191 await x 192 """) 193 194 def test_async_var(self): 195 self.validate("""async = 1""") 196 self.validate("""await = 1""") 197 self.validate("""def async(): pass""") 198 199 def test_async_with(self): 200 self.validate("""async def foo(): 201 async for a in b: pass""") 202 203 self.invalid_syntax("""def foo(): 204 async for a in b: pass""") 205 206 def test_async_for(self): 207 self.validate("""async def foo(): 208 async with a: pass""") 209 210 self.invalid_syntax("""def foo(): 211 async with a: pass""") 212 213 214 class TestRaiseChanges(GrammarTest): 215 def test_2x_style_1(self): 216 self.validate("raise") 217 218 def test_2x_style_2(self): 219 self.validate("raise E, V") 220 221 def test_2x_style_3(self): 222 self.validate("raise E, V, T") 223 224 def test_2x_style_invalid_1(self): 225 self.invalid_syntax("raise E, V, T, Z") 226 227 def test_3x_style(self): 228 self.validate("raise E1 from E2") 229 230 def test_3x_style_invalid_1(self): 231 self.invalid_syntax("raise E, V from E1") 232 233 def test_3x_style_invalid_2(self): 234 self.invalid_syntax("raise E from E1, E2") 235 236 def test_3x_style_invalid_3(self): 237 self.invalid_syntax("raise from E1, E2") 238 239 def test_3x_style_invalid_4(self): 240 self.invalid_syntax("raise E from") 241 242 243 # Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 244 # and Lib/test/text_parser.py test_list_displays, test_set_displays, 245 # test_dict_displays, test_argument_unpacking, ... changes. 246 class TestUnpackingGeneralizations(GrammarTest): 247 def test_mid_positional_star(self): 248 self.validate("""func(1, *(2, 3), 4)""") 249 250 def test_double_star_dict_literal(self): 251 self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") 252 253 def test_double_star_dict_literal_after_keywords(self): 254 self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") 255 256 def test_list_display(self): 257 self.validate("""[*{2}, 3, *[4]]""") 258 259 def test_set_display(self): 260 self.validate("""{*{2}, 3, *[4]}""") 261 262 def test_dict_display_1(self): 263 self.validate("""{**{}}""") 264 265 def test_dict_display_2(self): 266 self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") 267 268 def test_argument_unpacking_1(self): 269 self.validate("""f(a, *b, *c, d)""") 270 271 def test_argument_unpacking_2(self): 272 self.validate("""f(**a, **b)""") 273 274 def test_argument_unpacking_3(self): 275 self.validate("""f(2, *a, *b, **b, **c, **d)""") 276 277 def test_trailing_commas_1(self): 278 self.validate("def f(a, b): call(a, b)") 279 self.validate("def f(a, b,): call(a, b,)") 280 281 def test_trailing_commas_2(self): 282 self.validate("def f(a, *b): call(a, *b)") 283 self.validate("def f(a, *b,): call(a, *b,)") 284 285 def test_trailing_commas_3(self): 286 self.validate("def f(a, b=1): call(a, b=1)") 287 self.validate("def f(a, b=1,): call(a, b=1,)") 288 289 def test_trailing_commas_4(self): 290 self.validate("def f(a, **b): call(a, **b)") 291 self.validate("def f(a, **b,): call(a, **b,)") 292 293 def test_trailing_commas_5(self): 294 self.validate("def f(*a, b=1): call(*a, b=1)") 295 self.validate("def f(*a, b=1,): call(*a, b=1,)") 296 297 def test_trailing_commas_6(self): 298 self.validate("def f(*a, **b): call(*a, **b)") 299 self.validate("def f(*a, **b,): call(*a, **b,)") 300 301 def test_trailing_commas_7(self): 302 self.validate("def f(*, b=1): call(*b)") 303 self.validate("def f(*, b=1,): call(*b,)") 304 305 def test_trailing_commas_8(self): 306 self.validate("def f(a=1, b=2): call(a=1, b=2)") 307 self.validate("def f(a=1, b=2,): call(a=1, b=2,)") 308 309 def test_trailing_commas_9(self): 310 self.validate("def f(a=1, **b): call(a=1, **b)") 311 self.validate("def f(a=1, **b,): call(a=1, **b,)") 312 313 def test_trailing_commas_lambda_1(self): 314 self.validate("f = lambda a, b: call(a, b)") 315 self.validate("f = lambda a, b,: call(a, b,)") 316 317 def test_trailing_commas_lambda_2(self): 318 self.validate("f = lambda a, *b: call(a, *b)") 319 self.validate("f = lambda a, *b,: call(a, *b,)") 320 321 def test_trailing_commas_lambda_3(self): 322 self.validate("f = lambda a, b=1: call(a, b=1)") 323 self.validate("f = lambda a, b=1,: call(a, b=1,)") 324 325 def test_trailing_commas_lambda_4(self): 326 self.validate("f = lambda a, **b: call(a, **b)") 327 self.validate("f = lambda a, **b,: call(a, **b,)") 328 329 def test_trailing_commas_lambda_5(self): 330 self.validate("f = lambda *a, b=1: call(*a, b=1)") 331 self.validate("f = lambda *a, b=1,: call(*a, b=1,)") 332 333 def test_trailing_commas_lambda_6(self): 334 self.validate("f = lambda *a, **b: call(*a, **b)") 335 self.validate("f = lambda *a, **b,: call(*a, **b,)") 336 337 def test_trailing_commas_lambda_7(self): 338 self.validate("f = lambda *, b=1: call(*b)") 339 self.validate("f = lambda *, b=1,: call(*b,)") 340 341 def test_trailing_commas_lambda_8(self): 342 self.validate("f = lambda a=1, b=2: call(a=1, b=2)") 343 self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") 344 345 def test_trailing_commas_lambda_9(self): 346 self.validate("f = lambda a=1, **b: call(a=1, **b)") 347 self.validate("f = lambda a=1, **b,: call(a=1, **b,)") 348 349 350 # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef 351 class TestFunctionAnnotations(GrammarTest): 352 def test_1(self): 353 self.validate("""def f(x) -> list: pass""") 354 355 def test_2(self): 356 self.validate("""def f(x:int): pass""") 357 358 def test_3(self): 359 self.validate("""def f(*x:str): pass""") 360 361 def test_4(self): 362 self.validate("""def f(**x:float): pass""") 363 364 def test_5(self): 365 self.validate("""def f(x, y:1+2): pass""") 366 367 def test_6(self): 368 self.validate("""def f(a, (b:1, c:2, d)): pass""") 369 370 def test_7(self): 371 self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""") 372 373 def test_8(self): 374 s = """def f(a, (b:1, c:2, d), e:3=4, f=5, 375 *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" 376 self.validate(s) 377 378 def test_9(self): 379 s = """def f( 380 a: str, 381 b: int, 382 *, 383 c: bool = False, 384 **kwargs, 385 ) -> None: 386 call(c=c, **kwargs,)""" 387 self.validate(s) 388 389 def test_10(self): 390 s = """def f( 391 a: str, 392 ) -> None: 393 call(a,)""" 394 self.validate(s) 395 396 def test_11(self): 397 s = """def f( 398 a: str = '', 399 ) -> None: 400 call(a=a,)""" 401 self.validate(s) 402 403 def test_12(self): 404 s = """def f( 405 *args: str, 406 ) -> None: 407 call(*args,)""" 408 self.validate(s) 409 410 def test_13(self): 411 self.validate("def f(a: str, b: int) -> None: call(a, b)") 412 self.validate("def f(a: str, b: int,) -> None: call(a, b,)") 413 414 def test_14(self): 415 self.validate("def f(a: str, *b: int) -> None: call(a, *b)") 416 self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") 417 418 def test_15(self): 419 self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") 420 self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") 421 422 def test_16(self): 423 self.validate("def f(a: str, **b: int) -> None: call(a, **b)") 424 self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") 425 426 def test_17(self): 427 self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") 428 self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") 429 430 def test_18(self): 431 self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") 432 self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") 433 434 def test_19(self): 435 self.validate("def f(*, b: int=1) -> None: call(*b)") 436 self.validate("def f(*, b: int=1,) -> None: call(*b,)") 437 438 def test_20(self): 439 self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") 440 self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") 441 442 def test_21(self): 443 self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") 444 self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") 445 446 447 # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot 448 class TestVarAnnotations(GrammarTest): 449 def test_1(self): 450 self.validate("var1: int = 5") 451 452 def test_2(self): 453 self.validate("var2: [int, str]") 454 455 def test_3(self): 456 self.validate("def f():\n" 457 " st: str = 'Hello'\n" 458 " a.b: int = (1, 2)\n" 459 " return st\n") 460 461 def test_4(self): 462 self.validate("def fbad():\n" 463 " x: int\n" 464 " print(x)\n") 465 466 def test_5(self): 467 self.validate("class C:\n" 468 " x: int\n" 469 " s: str = 'attr'\n" 470 " z = 2\n" 471 " def __init__(self, x):\n" 472 " self.x: int = x\n") 473 474 def test_6(self): 475 self.validate("lst: List[int] = []") 476 477 478 class TestExcept(GrammarTest): 479 def test_new(self): 480 s = """ 481 try: 482 x 483 except E as N: 484 y""" 485 self.validate(s) 486 487 def test_old(self): 488 s = """ 489 try: 490 x 491 except E, N: 492 y""" 493 self.validate(s) 494 495 496 class TestStringLiterals(GrammarTest): 497 prefixes = ("'", '"', 498 "r'", 'r"', "R'", 'R"', 499 "u'", 'u"', "U'", 'U"', 500 "b'", 'b"', "B'", 'B"', 501 "f'", 'f"', "F'", 'F"', 502 "ur'", 'ur"', "Ur'", 'Ur"', 503 "uR'", 'uR"', "UR'", 'UR"', 504 "br'", 'br"', "Br'", 'Br"', 505 "bR'", 'bR"', "BR'", 'BR"', 506 "rb'", 'rb"', "Rb'", 'Rb"', 507 "rB'", 'rB"', "RB'", 'RB"',) 508 509 def test_lit(self): 510 for pre in self.prefixes: 511 single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) 512 self.validate(single) 513 triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) 514 self.validate(triple) 515 516 517 # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms 518 class TestSetLiteral(GrammarTest): 519 def test_1(self): 520 self.validate("""x = {'one'}""") 521 522 def test_2(self): 523 self.validate("""x = {'one', 1,}""") 524 525 def test_3(self): 526 self.validate("""x = {'one', 'two', 'three'}""") 527 528 def test_4(self): 529 self.validate("""x = {2, 3, 4,}""") 530 531 532 # Adapted from Python 3's Lib/test/test_unicode_identifiers.py and 533 # Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers 534 class TestIdentfier(GrammarTest): 535 def test_non_ascii_identifiers(self): 536 self.validate("rter = 'places'\ngrn = 'green'") 537 self.validate(" = a = = 1") 538 self.validate(" = a = = 1") 539 self.validate(" = a_ = 1") 540 541 542 class TestNumericLiterals(GrammarTest): 543 def test_new_octal_notation(self): 544 self.validate("""0o7777777777777""") 545 self.invalid_syntax("""0o7324528887""") 546 547 def test_new_binary_notation(self): 548 self.validate("""0b101010""") 549 self.invalid_syntax("""0b0101021""") 550 551 552 class TestClassDef(GrammarTest): 553 def test_new_syntax(self): 554 self.validate("class B(t=7): pass") 555 self.validate("class B(t, *args): pass") 556 self.validate("class B(t, **kwargs): pass") 557 self.validate("class B(t, *args, **kwargs): pass") 558 self.validate("class B(t, y=9, *args, **kwargs,): pass") 559 560 561 class TestParserIdempotency(support.TestCase): 562 563 """A cut-down version of pytree_idempotency.py.""" 564 565 def test_all_project_files(self): 566 for filepath in support.all_project_files(): 567 with open(filepath, "rb") as fp: 568 encoding = tokenize.detect_encoding(fp.readline)[0] 569 self.assertIsNotNone(encoding, 570 "can't detect encoding for %s" % filepath) 571 with open(filepath, "r", encoding=encoding) as fp: 572 source = fp.read() 573 try: 574 tree = driver.parse_string(source) 575 except ParseError: 576 try: 577 tree = driver_no_print_statement.parse_string(source) 578 except ParseError as err: 579 self.fail('ParseError on file %s (%s)' % (filepath, err)) 580 new = str(tree) 581 if new != source: 582 print(diff_texts(source, new, filepath)) 583 self.fail("Idempotency failed: %s" % filepath) 584 585 def test_extended_unpacking(self): 586 driver.parse_string("a, *b, c = x\n") 587 driver.parse_string("[*a, b] = x\n") 588 driver.parse_string("(z, *y, w) = m\n") 589 driver.parse_string("for *z, m in d: pass\n") 590 591 592 class TestLiterals(GrammarTest): 593 594 def validate(self, s): 595 driver.parse_string(support.dedent(s) + "\n\n") 596 597 def test_multiline_bytes_literals(self): 598 s = """ 599 md5test(b"\xaa" * 80, 600 (b"Test Using Larger Than Block-Size Key " 601 b"and Larger Than One Block-Size Data"), 602 "6f630fad67cda0ee1fb1f562db3aa53e") 603 """ 604 self.validate(s) 605 606 def test_multiline_bytes_tripquote_literals(self): 607 s = ''' 608 b""" 609 <?xml version="1.0" encoding="UTF-8"?> 610 <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN"> 611 """ 612 ''' 613 self.validate(s) 614 615 def test_multiline_str_literals(self): 616 s = """ 617 md5test("\xaa" * 80, 618 ("Test Using Larger Than Block-Size Key " 619 "and Larger Than One Block-Size Data"), 620 "6f630fad67cda0ee1fb1f562db3aa53e") 621 """ 622 self.validate(s) 623 624 625 def diff_texts(a, b, filename): 626 a = a.splitlines() 627 b = b.splitlines() 628 return difflib.unified_diff(a, b, filename, filename, 629 "(original)", "(reserialized)", 630 lineterm="") 631 632 633 if __name__ == '__main__': 634 unittest.main() 635
