1 """Test suite for 2to3's parser and grammar files. 2 3 This is the place to add tests for changes to 2to3's grammar, such as those 4 merging the grammars for Python 2 and 3. In addition to specific tests for 5 parts of the grammar we've changed, we also make sure we can parse the 6 test_grammar.py files from both Python 2 and Python 3. 7 """ 8 9 # Testing imports 10 from . import support 11 from .support import driver, test_dir 12 13 # Python imports 14 import os 15 import shutil 16 import subprocess 17 import sys 18 import tempfile 19 import unittest 20 21 # Local imports 22 from lib2to3.pgen2 import driver as pgen2_driver 23 from lib2to3.pgen2 import tokenize 24 from ..pgen2.parse import ParseError 25 from lib2to3.pygram import python_symbols as syms 26 27 28 class TestDriver(support.TestCase): 29 30 def test_formfeed(self): 31 s = """print 1\n\x0Cprint 2\n""" 32 t = driver.parse_string(s) 33 self.assertEqual(t.children[0].children[0].type, syms.print_stmt) 34 self.assertEqual(t.children[1].children[0].type, syms.print_stmt) 35 36 37 class TestPgen2Caching(support.TestCase): 38 def test_load_grammar_from_txt_file(self): 39 pgen2_driver.load_grammar(support.grammar_path, save=False, force=True) 40 41 def test_load_grammar_from_pickle(self): 42 # Make a copy of the grammar file in a temp directory we are 43 # guaranteed to be able to write to. 44 tmpdir = tempfile.mkdtemp() 45 try: 46 grammar_copy = os.path.join( 47 tmpdir, os.path.basename(support.grammar_path)) 48 shutil.copy(support.grammar_path, grammar_copy) 49 pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) 50 51 pgen2_driver.load_grammar(grammar_copy, save=True, force=True) 52 self.assertTrue(os.path.exists(pickle_name)) 53 54 os.unlink(grammar_copy) # Only the pickle remains... 55 pgen2_driver.load_grammar(grammar_copy, save=False, force=False) 56 finally: 57 shutil.rmtree(tmpdir) 58 59 @unittest.skipIf(sys.executable is None, 'sys.executable required') 60 def test_load_grammar_from_subprocess(self): 61 tmpdir = tempfile.mkdtemp() 62 tmpsubdir = os.path.join(tmpdir, 'subdir') 63 try: 64 os.mkdir(tmpsubdir) 65 grammar_base = os.path.basename(support.grammar_path) 66 grammar_copy = os.path.join(tmpdir, grammar_base) 67 grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) 68 shutil.copy(support.grammar_path, grammar_copy) 69 shutil.copy(support.grammar_path, grammar_sub_copy) 70 pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) 71 pickle_sub_name = pgen2_driver._generate_pickle_name( 72 grammar_sub_copy) 73 self.assertNotEqual(pickle_name, pickle_sub_name) 74 75 # Generate a pickle file from this process. 76 pgen2_driver.load_grammar(grammar_copy, save=True, force=True) 77 self.assertTrue(os.path.exists(pickle_name)) 78 79 # Generate a new pickle file in a subprocess with a most likely 80 # different hash randomization seed. 81 sub_env = dict(os.environ) 82 sub_env['PYTHONHASHSEED'] = 'random' 83 subprocess.check_call( 84 [sys.executable, '-c', """ 85 from lib2to3.pgen2 import driver as pgen2_driver 86 pgen2_driver.load_grammar(%r, save=True, force=True) 87 """ % (grammar_sub_copy,)], 88 env=sub_env) 89 self.assertTrue(os.path.exists(pickle_sub_name)) 90 91 with open(pickle_name, 'rb') as pickle_f_1, \ 92 open(pickle_sub_name, 'rb') as pickle_f_2: 93 self.assertEqual( 94 pickle_f_1.read(), pickle_f_2.read(), 95 msg='Grammar caches generated using different hash seeds' 96 ' were not identical.') 97 finally: 98 shutil.rmtree(tmpdir) 99 100 101 102 class GrammarTest(support.TestCase): 103 def validate(self, code): 104 support.parse_string(code) 105 106 def invalid_syntax(self, code): 107 try: 108 self.validate(code) 109 except ParseError: 110 pass 111 else: 112 raise AssertionError("Syntax shouldn't have been valid") 113 114 115 class TestMatrixMultiplication(GrammarTest): 116 def test_matrix_multiplication_operator(self): 117 self.validate("a @ b") 118 self.validate("a @= b") 119 120 121 class TestYieldFrom(GrammarTest): 122 def test_matrix_multiplication_operator(self): 123 self.validate("yield from x") 124 self.validate("(yield from x) + y") 125 self.invalid_syntax("yield from") 126 127 128 class TestRaiseChanges(GrammarTest): 129 def test_2x_style_1(self): 130 self.validate("raise") 131 132 def test_2x_style_2(self): 133 self.validate("raise E, V") 134 135 def test_2x_style_3(self): 136 self.validate("raise E, V, T") 137 138 def test_2x_style_invalid_1(self): 139 self.invalid_syntax("raise E, V, T, Z") 140 141 def test_3x_style(self): 142 self.validate("raise E1 from E2") 143 144 def test_3x_style_invalid_1(self): 145 self.invalid_syntax("raise E, V from E1") 146 147 def test_3x_style_invalid_2(self): 148 self.invalid_syntax("raise E from E1, E2") 149 150 def test_3x_style_invalid_3(self): 151 self.invalid_syntax("raise from E1, E2") 152 153 def test_3x_style_invalid_4(self): 154 self.invalid_syntax("raise E from") 155 156 157 # Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 158 # and Lib/test/text_parser.py test_list_displays, test_set_displays, 159 # test_dict_displays, test_argument_unpacking, ... changes. 160 class TestUnpackingGeneralizations(GrammarTest): 161 def test_mid_positional_star(self): 162 self.validate("""func(1, *(2, 3), 4)""") 163 164 def test_double_star_dict_literal(self): 165 self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") 166 167 def test_double_star_dict_literal_after_keywords(self): 168 self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") 169 170 def test_list_display(self): 171 self.validate("""[*{2}, 3, *[4]]""") 172 173 def test_set_display(self): 174 self.validate("""{*{2}, 3, *[4]}""") 175 176 def test_dict_display_1(self): 177 self.validate("""{**{}}""") 178 179 def test_dict_display_2(self): 180 self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") 181 182 def test_argument_unpacking_1(self): 183 self.validate("""f(a, *b, *c, d)""") 184 185 def test_argument_unpacking_2(self): 186 self.validate("""f(**a, **b)""") 187 188 def test_argument_unpacking_3(self): 189 self.validate("""f(2, *a, *b, **b, **c, **d)""") 190 191 192 # Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef 193 class TestFunctionAnnotations(GrammarTest): 194 def test_1(self): 195 self.validate("""def f(x) -> list: pass""") 196 197 def test_2(self): 198 self.validate("""def f(x:int): pass""") 199 200 def test_3(self): 201 self.validate("""def f(*x:str): pass""") 202 203 def test_4(self): 204 self.validate("""def f(**x:float): pass""") 205 206 def test_5(self): 207 self.validate("""def f(x, y:1+2): pass""") 208 209 def test_6(self): 210 self.validate("""def f(a, (b:1, c:2, d)): pass""") 211 212 def test_7(self): 213 self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""") 214 215 def test_8(self): 216 s = """def f(a, (b:1, c:2, d), e:3=4, f=5, 217 *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" 218 self.validate(s) 219 220 221 class TestExcept(GrammarTest): 222 def test_new(self): 223 s = """ 224 try: 225 x 226 except E as N: 227 y""" 228 self.validate(s) 229 230 def test_old(self): 231 s = """ 232 try: 233 x 234 except E, N: 235 y""" 236 self.validate(s) 237 238 239 # Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms 240 class TestSetLiteral(GrammarTest): 241 def test_1(self): 242 self.validate("""x = {'one'}""") 243 244 def test_2(self): 245 self.validate("""x = {'one', 1,}""") 246 247 def test_3(self): 248 self.validate("""x = {'one', 'two', 'three'}""") 249 250 def test_4(self): 251 self.validate("""x = {2, 3, 4,}""") 252 253 254 class TestNumericLiterals(GrammarTest): 255 def test_new_octal_notation(self): 256 self.validate("""0o7777777777777""") 257 self.invalid_syntax("""0o7324528887""") 258 259 def test_new_binary_notation(self): 260 self.validate("""0b101010""") 261 self.invalid_syntax("""0b0101021""") 262 263 264 class TestClassDef(GrammarTest): 265 def test_new_syntax(self): 266 self.validate("class B(t=7): pass") 267 self.validate("class B(t, *args): pass") 268 self.validate("class B(t, **kwargs): pass") 269 self.validate("class B(t, *args, **kwargs): pass") 270 self.validate("class B(t, y=9, *args, **kwargs): pass") 271 272 273 class TestParserIdempotency(support.TestCase): 274 275 """A cut-down version of pytree_idempotency.py.""" 276 277 def test_all_project_files(self): 278 if sys.platform.startswith("win"): 279 # XXX something with newlines goes wrong on Windows. 280 return 281 for filepath in support.all_project_files(): 282 with open(filepath, "rb") as fp: 283 encoding = tokenize.detect_encoding(fp.readline)[0] 284 self.assertIsNotNone(encoding, 285 "can't detect encoding for %s" % filepath) 286 with open(filepath, "r") as fp: 287 source = fp.read() 288 source = source.decode(encoding) 289 tree = driver.parse_string(source) 290 new = unicode(tree) 291 if diff(filepath, new, encoding): 292 self.fail("Idempotency failed: %s" % filepath) 293 294 def test_extended_unpacking(self): 295 driver.parse_string("a, *b, c = x\n") 296 driver.parse_string("[*a, b] = x\n") 297 driver.parse_string("(z, *y, w) = m\n") 298 driver.parse_string("for *z, m in d: pass\n") 299 300 class TestLiterals(GrammarTest): 301 302 def validate(self, s): 303 driver.parse_string(support.dedent(s) + "\n\n") 304 305 def test_multiline_bytes_literals(self): 306 s = """ 307 md5test(b"\xaa" * 80, 308 (b"Test Using Larger Than Block-Size Key " 309 b"and Larger Than One Block-Size Data"), 310 "6f630fad67cda0ee1fb1f562db3aa53e") 311 """ 312 self.validate(s) 313 314 def test_multiline_bytes_tripquote_literals(self): 315 s = ''' 316 b""" 317 <?xml version="1.0" encoding="UTF-8"?> 318 <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN"> 319 """ 320 ''' 321 self.validate(s) 322 323 def test_multiline_str_literals(self): 324 s = """ 325 md5test("\xaa" * 80, 326 ("Test Using Larger Than Block-Size Key " 327 "and Larger Than One Block-Size Data"), 328 "6f630fad67cda0ee1fb1f562db3aa53e") 329 """ 330 self.validate(s) 331 332 333 def diff(fn, result, encoding): 334 f = open("@", "w") 335 try: 336 f.write(result.encode(encoding)) 337 finally: 338 f.close() 339 try: 340 fn = fn.replace('"', '\\"') 341 return os.system('diff -u "%s" @' % fn) 342 finally: 343 os.remove("@") 344
