1 from .. import abc 2 from .. import util 3 4 importlib = util.import_importlib('importlib') 5 importlib_abc = util.import_importlib('importlib.abc') 6 machinery = util.import_importlib('importlib.machinery') 7 importlib_util = util.import_importlib('importlib.util') 8 9 import errno 10 import marshal 11 import os 12 import py_compile 13 import shutil 14 import stat 15 import sys 16 import types 17 import unittest 18 import warnings 19 20 from test.support import make_legacy_pyc, unload 21 22 23 class SimpleTest(abc.LoaderTests): 24 25 """Should have no issue importing a source module [basic]. And if there is 26 a syntax error, it should raise a SyntaxError [syntax error]. 27 28 """ 29 30 def setUp(self): 31 self.name = 'spam' 32 self.filepath = os.path.join('ham', self.name + '.py') 33 self.loader = self.machinery.SourceFileLoader(self.name, self.filepath) 34 35 def test_load_module_API(self): 36 class Tester(self.abc.FileLoader): 37 def get_source(self, _): return 'attr = 42' 38 def is_package(self, _): return False 39 40 loader = Tester('blah', 'blah.py') 41 self.addCleanup(unload, 'blah') 42 with warnings.catch_warnings(): 43 warnings.simplefilter('ignore', DeprecationWarning) 44 module = loader.load_module() # Should not raise an exception. 45 46 def test_get_filename_API(self): 47 # If fullname is not set then assume self.path is desired. 48 class Tester(self.abc.FileLoader): 49 def get_code(self, _): pass 50 def get_source(self, _): pass 51 def is_package(self, _): pass 52 def module_repr(self, _): pass 53 54 path = 'some_path' 55 name = 'some_name' 56 loader = Tester(name, path) 57 self.assertEqual(path, loader.get_filename(name)) 58 self.assertEqual(path, loader.get_filename()) 59 self.assertEqual(path, loader.get_filename(None)) 60 with self.assertRaises(ImportError): 61 loader.get_filename(name + 'XXX') 62 63 def test_equality(self): 64 other = self.machinery.SourceFileLoader(self.name, self.filepath) 65 self.assertEqual(self.loader, other) 66 67 def test_inequality(self): 68 other = self.machinery.SourceFileLoader('_' + self.name, self.filepath) 69 self.assertNotEqual(self.loader, other) 70 71 # [basic] 72 def test_module(self): 73 with util.create_modules('_temp') as mapping: 74 loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) 75 with warnings.catch_warnings(): 76 warnings.simplefilter('ignore', DeprecationWarning) 77 module = loader.load_module('_temp') 78 self.assertIn('_temp', sys.modules) 79 check = {'__name__': '_temp', '__file__': mapping['_temp'], 80 '__package__': ''} 81 for attr, value in check.items(): 82 self.assertEqual(getattr(module, attr), value) 83 84 def test_package(self): 85 with util.create_modules('_pkg.__init__') as mapping: 86 loader = self.machinery.SourceFileLoader('_pkg', 87 mapping['_pkg.__init__']) 88 with warnings.catch_warnings(): 89 warnings.simplefilter('ignore', DeprecationWarning) 90 module = loader.load_module('_pkg') 91 self.assertIn('_pkg', sys.modules) 92 check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], 93 '__path__': [os.path.dirname(mapping['_pkg.__init__'])], 94 '__package__': '_pkg'} 95 for attr, value in check.items(): 96 self.assertEqual(getattr(module, attr), value) 97 98 99 def test_lacking_parent(self): 100 with util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: 101 loader = self.machinery.SourceFileLoader('_pkg.mod', 102 mapping['_pkg.mod']) 103 with warnings.catch_warnings(): 104 warnings.simplefilter('ignore', DeprecationWarning) 105 module = loader.load_module('_pkg.mod') 106 self.assertIn('_pkg.mod', sys.modules) 107 check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], 108 '__package__': '_pkg'} 109 for attr, value in check.items(): 110 self.assertEqual(getattr(module, attr), value) 111 112 def fake_mtime(self, fxn): 113 """Fake mtime to always be higher than expected.""" 114 return lambda name: fxn(name) + 1 115 116 def test_module_reuse(self): 117 with util.create_modules('_temp') as mapping: 118 loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) 119 with warnings.catch_warnings(): 120 warnings.simplefilter('ignore', DeprecationWarning) 121 module = loader.load_module('_temp') 122 module_id = id(module) 123 module_dict_id = id(module.__dict__) 124 with open(mapping['_temp'], 'w') as file: 125 file.write("testing_var = 42\n") 126 with warnings.catch_warnings(): 127 warnings.simplefilter('ignore', DeprecationWarning) 128 module = loader.load_module('_temp') 129 self.assertIn('testing_var', module.__dict__, 130 "'testing_var' not in " 131 "{0}".format(list(module.__dict__.keys()))) 132 self.assertEqual(module, sys.modules['_temp']) 133 self.assertEqual(id(module), module_id) 134 self.assertEqual(id(module.__dict__), module_dict_id) 135 136 def test_state_after_failure(self): 137 # A failed reload should leave the original module intact. 138 attributes = ('__file__', '__path__', '__package__') 139 value = '<test>' 140 name = '_temp' 141 with util.create_modules(name) as mapping: 142 orig_module = types.ModuleType(name) 143 for attr in attributes: 144 setattr(orig_module, attr, value) 145 with open(mapping[name], 'w') as file: 146 file.write('+++ bad syntax +++') 147 loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) 148 with self.assertRaises(SyntaxError): 149 loader.exec_module(orig_module) 150 for attr in attributes: 151 self.assertEqual(getattr(orig_module, attr), value) 152 with self.assertRaises(SyntaxError): 153 with warnings.catch_warnings(): 154 warnings.simplefilter('ignore', DeprecationWarning) 155 loader.load_module(name) 156 for attr in attributes: 157 self.assertEqual(getattr(orig_module, attr), value) 158 159 # [syntax error] 160 def test_bad_syntax(self): 161 with util.create_modules('_temp') as mapping: 162 with open(mapping['_temp'], 'w') as file: 163 file.write('=') 164 loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) 165 with self.assertRaises(SyntaxError): 166 with warnings.catch_warnings(): 167 warnings.simplefilter('ignore', DeprecationWarning) 168 loader.load_module('_temp') 169 self.assertNotIn('_temp', sys.modules) 170 171 def test_file_from_empty_string_dir(self): 172 # Loading a module found from an empty string entry on sys.path should 173 # not only work, but keep all attributes relative. 174 file_path = '_temp.py' 175 with open(file_path, 'w') as file: 176 file.write("# test file for importlib") 177 try: 178 with util.uncache('_temp'): 179 loader = self.machinery.SourceFileLoader('_temp', file_path) 180 with warnings.catch_warnings(): 181 warnings.simplefilter('ignore', DeprecationWarning) 182 mod = loader.load_module('_temp') 183 self.assertEqual(file_path, mod.__file__) 184 self.assertEqual(self.util.cache_from_source(file_path), 185 mod.__cached__) 186 finally: 187 os.unlink(file_path) 188 pycache = os.path.dirname(self.util.cache_from_source(file_path)) 189 if os.path.exists(pycache): 190 shutil.rmtree(pycache) 191 192 @util.writes_bytecode_files 193 def test_timestamp_overflow(self): 194 # When a modification timestamp is larger than 2**32, it should be 195 # truncated rather than raise an OverflowError. 196 with util.create_modules('_temp') as mapping: 197 source = mapping['_temp'] 198 compiled = self.util.cache_from_source(source) 199 with open(source, 'w') as f: 200 f.write("x = 5") 201 try: 202 os.utime(source, (2 ** 33 - 5, 2 ** 33 - 5)) 203 except OverflowError: 204 self.skipTest("cannot set modification time to large integer") 205 except OSError as e: 206 if e.errno != getattr(errno, 'EOVERFLOW', None): 207 raise 208 self.skipTest("cannot set modification time to large integer ({})".format(e)) 209 loader = self.machinery.SourceFileLoader('_temp', mapping['_temp']) 210 # PEP 451 211 module = types.ModuleType('_temp') 212 module.__spec__ = self.util.spec_from_loader('_temp', loader) 213 loader.exec_module(module) 214 self.assertEqual(module.x, 5) 215 self.assertTrue(os.path.exists(compiled)) 216 os.unlink(compiled) 217 # PEP 302 218 with warnings.catch_warnings(): 219 warnings.simplefilter('ignore', DeprecationWarning) 220 mod = loader.load_module('_temp') 221 # Sanity checks. 222 self.assertEqual(mod.__cached__, compiled) 223 self.assertEqual(mod.x, 5) 224 # The pyc file was created. 225 self.assertTrue(os.path.exists(compiled)) 226 227 def test_unloadable(self): 228 loader = self.machinery.SourceFileLoader('good name', {}) 229 module = types.ModuleType('bad name') 230 module.__spec__ = self.machinery.ModuleSpec('bad name', loader) 231 with self.assertRaises(ImportError): 232 loader.exec_module(module) 233 with self.assertRaises(ImportError): 234 with warnings.catch_warnings(): 235 warnings.simplefilter('ignore', DeprecationWarning) 236 loader.load_module('bad name') 237 238 239 (Frozen_SimpleTest, 240 Source_SimpleTest 241 ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, 242 abc=importlib_abc, util=importlib_util) 243 244 245 class BadBytecodeTest: 246 247 def import_(self, file, module_name): 248 raise NotImplementedError 249 250 def manipulate_bytecode(self, name, mapping, manipulator, *, 251 del_source=False): 252 """Manipulate the bytecode of a module by passing it into a callable 253 that returns what to use as the new bytecode.""" 254 try: 255 del sys.modules['_temp'] 256 except KeyError: 257 pass 258 py_compile.compile(mapping[name]) 259 if not del_source: 260 bytecode_path = self.util.cache_from_source(mapping[name]) 261 else: 262 os.unlink(mapping[name]) 263 bytecode_path = make_legacy_pyc(mapping[name]) 264 if manipulator: 265 with open(bytecode_path, 'rb') as file: 266 bc = file.read() 267 new_bc = manipulator(bc) 268 with open(bytecode_path, 'wb') as file: 269 if new_bc is not None: 270 file.write(new_bc) 271 return bytecode_path 272 273 def _test_empty_file(self, test, *, del_source=False): 274 with util.create_modules('_temp') as mapping: 275 bc_path = self.manipulate_bytecode('_temp', mapping, 276 lambda bc: b'', 277 del_source=del_source) 278 test('_temp', mapping, bc_path) 279 280 @util.writes_bytecode_files 281 def _test_partial_magic(self, test, *, del_source=False): 282 # When their are less than 4 bytes to a .pyc, regenerate it if 283 # possible, else raise ImportError. 284 with util.create_modules('_temp') as mapping: 285 bc_path = self.manipulate_bytecode('_temp', mapping, 286 lambda bc: bc[:3], 287 del_source=del_source) 288 test('_temp', mapping, bc_path) 289 290 def _test_magic_only(self, test, *, del_source=False): 291 with util.create_modules('_temp') as mapping: 292 bc_path = self.manipulate_bytecode('_temp', mapping, 293 lambda bc: bc[:4], 294 del_source=del_source) 295 test('_temp', mapping, bc_path) 296 297 def _test_partial_timestamp(self, test, *, del_source=False): 298 with util.create_modules('_temp') as mapping: 299 bc_path = self.manipulate_bytecode('_temp', mapping, 300 lambda bc: bc[:7], 301 del_source=del_source) 302 test('_temp', mapping, bc_path) 303 304 def _test_partial_size(self, test, *, del_source=False): 305 with util.create_modules('_temp') as mapping: 306 bc_path = self.manipulate_bytecode('_temp', mapping, 307 lambda bc: bc[:11], 308 del_source=del_source) 309 test('_temp', mapping, bc_path) 310 311 def _test_no_marshal(self, *, del_source=False): 312 with util.create_modules('_temp') as mapping: 313 bc_path = self.manipulate_bytecode('_temp', mapping, 314 lambda bc: bc[:12], 315 del_source=del_source) 316 file_path = mapping['_temp'] if not del_source else bc_path 317 with self.assertRaises(EOFError): 318 self.import_(file_path, '_temp') 319 320 def _test_non_code_marshal(self, *, del_source=False): 321 with util.create_modules('_temp') as mapping: 322 bytecode_path = self.manipulate_bytecode('_temp', mapping, 323 lambda bc: bc[:12] + marshal.dumps(b'abcd'), 324 del_source=del_source) 325 file_path = mapping['_temp'] if not del_source else bytecode_path 326 with self.assertRaises(ImportError) as cm: 327 self.import_(file_path, '_temp') 328 self.assertEqual(cm.exception.name, '_temp') 329 self.assertEqual(cm.exception.path, bytecode_path) 330 331 def _test_bad_marshal(self, *, del_source=False): 332 with util.create_modules('_temp') as mapping: 333 bytecode_path = self.manipulate_bytecode('_temp', mapping, 334 lambda bc: bc[:12] + b'<test>', 335 del_source=del_source) 336 file_path = mapping['_temp'] if not del_source else bytecode_path 337 with self.assertRaises(EOFError): 338 self.import_(file_path, '_temp') 339 340 def _test_bad_magic(self, test, *, del_source=False): 341 with util.create_modules('_temp') as mapping: 342 bc_path = self.manipulate_bytecode('_temp', mapping, 343 lambda bc: b'\x00\x00\x00\x00' + bc[4:]) 344 test('_temp', mapping, bc_path) 345 346 347 class BadBytecodeTestPEP451(BadBytecodeTest): 348 349 def import_(self, file, module_name): 350 loader = self.loader(module_name, file) 351 module = types.ModuleType(module_name) 352 module.__spec__ = self.util.spec_from_loader(module_name, loader) 353 loader.exec_module(module) 354 355 356 class BadBytecodeTestPEP302(BadBytecodeTest): 357 358 def import_(self, file, module_name): 359 loader = self.loader(module_name, file) 360 with warnings.catch_warnings(): 361 warnings.simplefilter('ignore', DeprecationWarning) 362 module = loader.load_module(module_name) 363 self.assertIn(module_name, sys.modules) 364 365 366 class SourceLoaderBadBytecodeTest: 367 368 @classmethod 369 def setUpClass(cls): 370 cls.loader = cls.machinery.SourceFileLoader 371 372 @util.writes_bytecode_files 373 def test_empty_file(self): 374 # When a .pyc is empty, regenerate it if possible, else raise 375 # ImportError. 376 def test(name, mapping, bytecode_path): 377 self.import_(mapping[name], name) 378 with open(bytecode_path, 'rb') as file: 379 self.assertGreater(len(file.read()), 12) 380 381 self._test_empty_file(test) 382 383 def test_partial_magic(self): 384 def test(name, mapping, bytecode_path): 385 self.import_(mapping[name], name) 386 with open(bytecode_path, 'rb') as file: 387 self.assertGreater(len(file.read()), 12) 388 389 self._test_partial_magic(test) 390 391 @util.writes_bytecode_files 392 def test_magic_only(self): 393 # When there is only the magic number, regenerate the .pyc if possible, 394 # else raise EOFError. 395 def test(name, mapping, bytecode_path): 396 self.import_(mapping[name], name) 397 with open(bytecode_path, 'rb') as file: 398 self.assertGreater(len(file.read()), 12) 399 400 self._test_magic_only(test) 401 402 @util.writes_bytecode_files 403 def test_bad_magic(self): 404 # When the magic number is different, the bytecode should be 405 # regenerated. 406 def test(name, mapping, bytecode_path): 407 self.import_(mapping[name], name) 408 with open(bytecode_path, 'rb') as bytecode_file: 409 self.assertEqual(bytecode_file.read(4), 410 self.util.MAGIC_NUMBER) 411 412 self._test_bad_magic(test) 413 414 @util.writes_bytecode_files 415 def test_partial_timestamp(self): 416 # When the timestamp is partial, regenerate the .pyc, else 417 # raise EOFError. 418 def test(name, mapping, bc_path): 419 self.import_(mapping[name], name) 420 with open(bc_path, 'rb') as file: 421 self.assertGreater(len(file.read()), 12) 422 423 self._test_partial_timestamp(test) 424 425 @util.writes_bytecode_files 426 def test_partial_size(self): 427 # When the size is partial, regenerate the .pyc, else 428 # raise EOFError. 429 def test(name, mapping, bc_path): 430 self.import_(mapping[name], name) 431 with open(bc_path, 'rb') as file: 432 self.assertGreater(len(file.read()), 12) 433 434 self._test_partial_size(test) 435 436 @util.writes_bytecode_files 437 def test_no_marshal(self): 438 # When there is only the magic number and timestamp, raise EOFError. 439 self._test_no_marshal() 440 441 @util.writes_bytecode_files 442 def test_non_code_marshal(self): 443 self._test_non_code_marshal() 444 # XXX ImportError when sourceless 445 446 # [bad marshal] 447 @util.writes_bytecode_files 448 def test_bad_marshal(self): 449 # Bad marshal data should raise a ValueError. 450 self._test_bad_marshal() 451 452 # [bad timestamp] 453 @util.writes_bytecode_files 454 def test_old_timestamp(self): 455 # When the timestamp is older than the source, bytecode should be 456 # regenerated. 457 zeros = b'\x00\x00\x00\x00' 458 with util.create_modules('_temp') as mapping: 459 py_compile.compile(mapping['_temp']) 460 bytecode_path = self.util.cache_from_source(mapping['_temp']) 461 with open(bytecode_path, 'r+b') as bytecode_file: 462 bytecode_file.seek(4) 463 bytecode_file.write(zeros) 464 self.import_(mapping['_temp'], '_temp') 465 source_mtime = os.path.getmtime(mapping['_temp']) 466 source_timestamp = self.importlib._w_long(source_mtime) 467 with open(bytecode_path, 'rb') as bytecode_file: 468 bytecode_file.seek(4) 469 self.assertEqual(bytecode_file.read(4), source_timestamp) 470 471 # [bytecode read-only] 472 @util.writes_bytecode_files 473 def test_read_only_bytecode(self): 474 # When bytecode is read-only but should be rewritten, fail silently. 475 with util.create_modules('_temp') as mapping: 476 # Create bytecode that will need to be re-created. 477 py_compile.compile(mapping['_temp']) 478 bytecode_path = self.util.cache_from_source(mapping['_temp']) 479 with open(bytecode_path, 'r+b') as bytecode_file: 480 bytecode_file.seek(0) 481 bytecode_file.write(b'\x00\x00\x00\x00') 482 # Make the bytecode read-only. 483 os.chmod(bytecode_path, 484 stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) 485 try: 486 # Should not raise OSError! 487 self.import_(mapping['_temp'], '_temp') 488 finally: 489 # Make writable for eventual clean-up. 490 os.chmod(bytecode_path, stat.S_IWUSR) 491 492 493 class SourceLoaderBadBytecodeTestPEP451( 494 SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): 495 pass 496 497 498 (Frozen_SourceBadBytecodePEP451, 499 Source_SourceBadBytecodePEP451 500 ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, 501 machinery=machinery, abc=importlib_abc, 502 util=importlib_util) 503 504 505 class SourceLoaderBadBytecodeTestPEP302( 506 SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): 507 pass 508 509 510 (Frozen_SourceBadBytecodePEP302, 511 Source_SourceBadBytecodePEP302 512 ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, 513 machinery=machinery, abc=importlib_abc, 514 util=importlib_util) 515 516 517 class SourcelessLoaderBadBytecodeTest: 518 519 @classmethod 520 def setUpClass(cls): 521 cls.loader = cls.machinery.SourcelessFileLoader 522 523 def test_empty_file(self): 524 def test(name, mapping, bytecode_path): 525 with self.assertRaises(ImportError) as cm: 526 self.import_(bytecode_path, name) 527 self.assertEqual(cm.exception.name, name) 528 self.assertEqual(cm.exception.path, bytecode_path) 529 530 self._test_empty_file(test, del_source=True) 531 532 def test_partial_magic(self): 533 def test(name, mapping, bytecode_path): 534 with self.assertRaises(ImportError) as cm: 535 self.import_(bytecode_path, name) 536 self.assertEqual(cm.exception.name, name) 537 self.assertEqual(cm.exception.path, bytecode_path) 538 self._test_partial_magic(test, del_source=True) 539 540 def test_magic_only(self): 541 def test(name, mapping, bytecode_path): 542 with self.assertRaises(EOFError): 543 self.import_(bytecode_path, name) 544 545 self._test_magic_only(test, del_source=True) 546 547 def test_bad_magic(self): 548 def test(name, mapping, bytecode_path): 549 with self.assertRaises(ImportError) as cm: 550 self.import_(bytecode_path, name) 551 self.assertEqual(cm.exception.name, name) 552 self.assertEqual(cm.exception.path, bytecode_path) 553 554 self._test_bad_magic(test, del_source=True) 555 556 def test_partial_timestamp(self): 557 def test(name, mapping, bytecode_path): 558 with self.assertRaises(EOFError): 559 self.import_(bytecode_path, name) 560 561 self._test_partial_timestamp(test, del_source=True) 562 563 def test_partial_size(self): 564 def test(name, mapping, bytecode_path): 565 with self.assertRaises(EOFError): 566 self.import_(bytecode_path, name) 567 568 self._test_partial_size(test, del_source=True) 569 570 def test_no_marshal(self): 571 self._test_no_marshal(del_source=True) 572 573 def test_non_code_marshal(self): 574 self._test_non_code_marshal(del_source=True) 575 576 577 class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, 578 BadBytecodeTestPEP451): 579 pass 580 581 582 (Frozen_SourcelessBadBytecodePEP451, 583 Source_SourcelessBadBytecodePEP451 584 ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, 585 machinery=machinery, abc=importlib_abc, 586 util=importlib_util) 587 588 589 class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, 590 BadBytecodeTestPEP302): 591 pass 592 593 594 (Frozen_SourcelessBadBytecodePEP302, 595 Source_SourcelessBadBytecodePEP302 596 ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, 597 machinery=machinery, abc=importlib_abc, 598 util=importlib_util) 599 600 601 if __name__ == '__main__': 602 unittest.main() 603
