1 import sys 2 import compileall 3 import importlib.util 4 import test.test_importlib.util 5 import os 6 import pathlib 7 import py_compile 8 import shutil 9 import struct 10 import tempfile 11 import time 12 import unittest 13 import io 14 15 from unittest import mock, skipUnless 16 try: 17 from concurrent.futures import ProcessPoolExecutor 18 _have_multiprocessing = True 19 except ImportError: 20 _have_multiprocessing = False 21 22 from test import support 23 from test.support import script_helper 24 25 class CompileallTests(unittest.TestCase): 26 27 def setUp(self): 28 self.directory = tempfile.mkdtemp() 29 self.source_path = os.path.join(self.directory, '_test.py') 30 self.bc_path = importlib.util.cache_from_source(self.source_path) 31 with open(self.source_path, 'w') as file: 32 file.write('x = 123\n') 33 self.source_path2 = os.path.join(self.directory, '_test2.py') 34 self.bc_path2 = importlib.util.cache_from_source(self.source_path2) 35 shutil.copyfile(self.source_path, self.source_path2) 36 self.subdirectory = os.path.join(self.directory, '_subdir') 37 os.mkdir(self.subdirectory) 38 self.source_path3 = os.path.join(self.subdirectory, '_test3.py') 39 shutil.copyfile(self.source_path, self.source_path3) 40 41 def tearDown(self): 42 shutil.rmtree(self.directory) 43 44 def add_bad_source_file(self): 45 self.bad_source_path = os.path.join(self.directory, '_test_bad.py') 46 with open(self.bad_source_path, 'w') as file: 47 file.write('x (\n') 48 49 def data(self): 50 with open(self.bc_path, 'rb') as file: 51 data = file.read(8) 52 mtime = int(os.stat(self.source_path).st_mtime) 53 compare = struct.pack('<4sl', importlib.util.MAGIC_NUMBER, mtime) 54 return data, compare 55 56 @unittest.skipUnless(hasattr(os, 'stat'), 'test needs os.stat()') 57 def recreation_check(self, metadata): 58 """Check that compileall recreates bytecode when the new metadata is 59 used.""" 60 py_compile.compile(self.source_path) 61 self.assertEqual(*self.data()) 62 with open(self.bc_path, 'rb') as file: 63 bc = file.read()[len(metadata):] 64 with open(self.bc_path, 'wb') as file: 65 file.write(metadata) 66 file.write(bc) 67 self.assertNotEqual(*self.data()) 68 compileall.compile_dir(self.directory, force=False, quiet=True) 69 self.assertTrue(*self.data()) 70 71 def test_mtime(self): 72 # Test a change in mtime leads to a new .pyc. 73 self.recreation_check(struct.pack('<4sl', importlib.util.MAGIC_NUMBER, 74 1)) 75 76 def test_magic_number(self): 77 # Test a change in mtime leads to a new .pyc. 78 self.recreation_check(b'\0\0\0\0') 79 80 def test_compile_files(self): 81 # Test compiling a single file, and complete directory 82 for fn in (self.bc_path, self.bc_path2): 83 try: 84 os.unlink(fn) 85 except: 86 pass 87 self.assertTrue(compileall.compile_file(self.source_path, 88 force=False, quiet=True)) 89 self.assertTrue(os.path.isfile(self.bc_path) and 90 not os.path.isfile(self.bc_path2)) 91 os.unlink(self.bc_path) 92 self.assertTrue(compileall.compile_dir(self.directory, force=False, 93 quiet=True)) 94 self.assertTrue(os.path.isfile(self.bc_path) and 95 os.path.isfile(self.bc_path2)) 96 os.unlink(self.bc_path) 97 os.unlink(self.bc_path2) 98 # Test against bad files 99 self.add_bad_source_file() 100 self.assertFalse(compileall.compile_file(self.bad_source_path, 101 force=False, quiet=2)) 102 self.assertFalse(compileall.compile_dir(self.directory, 103 force=False, quiet=2)) 104 105 def test_compile_file_pathlike(self): 106 self.assertFalse(os.path.isfile(self.bc_path)) 107 # we should also test the output 108 with support.captured_stdout() as stdout: 109 self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path))) 110 self.assertRegex(stdout.getvalue(), r'Compiling ([^WindowsPath|PosixPath].*)') 111 self.assertTrue(os.path.isfile(self.bc_path)) 112 113 def test_compile_file_pathlike_ddir(self): 114 self.assertFalse(os.path.isfile(self.bc_path)) 115 self.assertTrue(compileall.compile_file(pathlib.Path(self.source_path), 116 ddir=pathlib.Path('ddir_path'), 117 quiet=2)) 118 self.assertTrue(os.path.isfile(self.bc_path)) 119 120 def test_compile_path(self): 121 with test.test_importlib.util.import_state(path=[self.directory]): 122 self.assertTrue(compileall.compile_path(quiet=2)) 123 124 with test.test_importlib.util.import_state(path=[self.directory]): 125 self.add_bad_source_file() 126 self.assertFalse(compileall.compile_path(skip_curdir=False, 127 force=True, quiet=2)) 128 129 def test_no_pycache_in_non_package(self): 130 # Bug 8563 reported that __pycache__ directories got created by 131 # compile_file() for non-.py files. 132 data_dir = os.path.join(self.directory, 'data') 133 data_file = os.path.join(data_dir, 'file') 134 os.mkdir(data_dir) 135 # touch data/file 136 with open(data_file, 'w'): 137 pass 138 compileall.compile_file(data_file) 139 self.assertFalse(os.path.exists(os.path.join(data_dir, '__pycache__'))) 140 141 def test_optimize(self): 142 # make sure compiling with different optimization settings than the 143 # interpreter's creates the correct file names 144 optimize, opt = (1, 1) if __debug__ else (0, '') 145 compileall.compile_dir(self.directory, quiet=True, optimize=optimize) 146 cached = importlib.util.cache_from_source(self.source_path, 147 optimization=opt) 148 self.assertTrue(os.path.isfile(cached)) 149 cached2 = importlib.util.cache_from_source(self.source_path2, 150 optimization=opt) 151 self.assertTrue(os.path.isfile(cached2)) 152 cached3 = importlib.util.cache_from_source(self.source_path3, 153 optimization=opt) 154 self.assertTrue(os.path.isfile(cached3)) 155 156 def test_compile_dir_pathlike(self): 157 self.assertFalse(os.path.isfile(self.bc_path)) 158 with support.captured_stdout() as stdout: 159 compileall.compile_dir(pathlib.Path(self.directory)) 160 line = stdout.getvalue().splitlines()[0] 161 self.assertRegex(line, r'Listing ([^WindowsPath|PosixPath].*)') 162 self.assertTrue(os.path.isfile(self.bc_path)) 163 164 @mock.patch('compileall.ProcessPoolExecutor') 165 def test_compile_pool_called(self, pool_mock): 166 compileall.compile_dir(self.directory, quiet=True, workers=5) 167 self.assertTrue(pool_mock.called) 168 169 def test_compile_workers_non_positive(self): 170 with self.assertRaisesRegex(ValueError, 171 "workers must be greater or equal to 0"): 172 compileall.compile_dir(self.directory, workers=-1) 173 174 @mock.patch('compileall.ProcessPoolExecutor') 175 def test_compile_workers_cpu_count(self, pool_mock): 176 compileall.compile_dir(self.directory, quiet=True, workers=0) 177 self.assertEqual(pool_mock.call_args[1]['max_workers'], None) 178 179 @mock.patch('compileall.ProcessPoolExecutor') 180 @mock.patch('compileall.compile_file') 181 def test_compile_one_worker(self, compile_file_mock, pool_mock): 182 compileall.compile_dir(self.directory, quiet=True) 183 self.assertFalse(pool_mock.called) 184 self.assertTrue(compile_file_mock.called) 185 186 @mock.patch('compileall.ProcessPoolExecutor', new=None) 187 @mock.patch('compileall.compile_file') 188 def test_compile_missing_multiprocessing(self, compile_file_mock): 189 compileall.compile_dir(self.directory, quiet=True, workers=5) 190 self.assertTrue(compile_file_mock.called) 191 192 class EncodingTest(unittest.TestCase): 193 """Issue 6716: compileall should escape source code when printing errors 194 to stdout.""" 195 196 def setUp(self): 197 self.directory = tempfile.mkdtemp() 198 self.source_path = os.path.join(self.directory, '_test.py') 199 with open(self.source_path, 'w', encoding='utf-8') as file: 200 file.write('# -*- coding: utf-8 -*-\n') 201 file.write('print u"\u20ac"\n') 202 203 def tearDown(self): 204 shutil.rmtree(self.directory) 205 206 def test_error(self): 207 try: 208 orig_stdout = sys.stdout 209 sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii') 210 compileall.compile_dir(self.directory) 211 finally: 212 sys.stdout = orig_stdout 213 214 215 class CommandLineTests(unittest.TestCase): 216 """Test compileall's CLI.""" 217 218 @classmethod 219 def setUpClass(cls): 220 for path in filter(os.path.isdir, sys.path): 221 directory_created = False 222 directory = pathlib.Path(path) / '__pycache__' 223 path = directory / 'test.try' 224 try: 225 if not directory.is_dir(): 226 directory.mkdir() 227 directory_created = True 228 with path.open('w') as file: 229 file.write('# for test_compileall') 230 except OSError: 231 sys_path_writable = False 232 break 233 finally: 234 support.unlink(str(path)) 235 if directory_created: 236 directory.rmdir() 237 else: 238 sys_path_writable = True 239 cls._sys_path_writable = sys_path_writable 240 241 def _skip_if_sys_path_not_writable(self): 242 if not self._sys_path_writable: 243 raise unittest.SkipTest('not all entries on sys.path are writable') 244 245 def _get_run_args(self, args): 246 return [*support.optim_args_from_interpreter_flags(), 247 '-S', '-m', 'compileall', 248 *args] 249 250 def assertRunOK(self, *args, **env_vars): 251 rc, out, err = script_helper.assert_python_ok( 252 *self._get_run_args(args), **env_vars) 253 self.assertEqual(b'', err) 254 return out 255 256 def assertRunNotOK(self, *args, **env_vars): 257 rc, out, err = script_helper.assert_python_failure( 258 *self._get_run_args(args), **env_vars) 259 return rc, out, err 260 261 def assertCompiled(self, fn): 262 path = importlib.util.cache_from_source(fn) 263 self.assertTrue(os.path.exists(path)) 264 265 def assertNotCompiled(self, fn): 266 path = importlib.util.cache_from_source(fn) 267 self.assertFalse(os.path.exists(path)) 268 269 def setUp(self): 270 self.directory = tempfile.mkdtemp() 271 self.addCleanup(support.rmtree, self.directory) 272 self.pkgdir = os.path.join(self.directory, 'foo') 273 os.mkdir(self.pkgdir) 274 self.pkgdir_cachedir = os.path.join(self.pkgdir, '__pycache__') 275 # Create the __init__.py and a package module. 276 self.initfn = script_helper.make_script(self.pkgdir, '__init__', '') 277 self.barfn = script_helper.make_script(self.pkgdir, 'bar', '') 278 279 def test_no_args_compiles_path(self): 280 # Note that -l is implied for the no args case. 281 self._skip_if_sys_path_not_writable() 282 bazfn = script_helper.make_script(self.directory, 'baz', '') 283 self.assertRunOK(PYTHONPATH=self.directory) 284 self.assertCompiled(bazfn) 285 self.assertNotCompiled(self.initfn) 286 self.assertNotCompiled(self.barfn) 287 288 def test_no_args_respects_force_flag(self): 289 self._skip_if_sys_path_not_writable() 290 bazfn = script_helper.make_script(self.directory, 'baz', '') 291 self.assertRunOK(PYTHONPATH=self.directory) 292 pycpath = importlib.util.cache_from_source(bazfn) 293 # Set atime/mtime backward to avoid file timestamp resolution issues 294 os.utime(pycpath, (time.time()-60,)*2) 295 mtime = os.stat(pycpath).st_mtime 296 # Without force, no recompilation 297 self.assertRunOK(PYTHONPATH=self.directory) 298 mtime2 = os.stat(pycpath).st_mtime 299 self.assertEqual(mtime, mtime2) 300 # Now force it. 301 self.assertRunOK('-f', PYTHONPATH=self.directory) 302 mtime2 = os.stat(pycpath).st_mtime 303 self.assertNotEqual(mtime, mtime2) 304 305 def test_no_args_respects_quiet_flag(self): 306 self._skip_if_sys_path_not_writable() 307 script_helper.make_script(self.directory, 'baz', '') 308 noisy = self.assertRunOK(PYTHONPATH=self.directory) 309 self.assertIn(b'Listing ', noisy) 310 quiet = self.assertRunOK('-q', PYTHONPATH=self.directory) 311 self.assertNotIn(b'Listing ', quiet) 312 313 # Ensure that the default behavior of compileall's CLI is to create 314 # PEP 3147/PEP 488 pyc files. 315 for name, ext, switch in [ 316 ('normal', 'pyc', []), 317 ('optimize', 'opt-1.pyc', ['-O']), 318 ('doubleoptimize', 'opt-2.pyc', ['-OO']), 319 ]: 320 def f(self, ext=ext, switch=switch): 321 script_helper.assert_python_ok(*(switch + 322 ['-m', 'compileall', '-q', self.pkgdir])) 323 # Verify the __pycache__ directory contents. 324 self.assertTrue(os.path.exists(self.pkgdir_cachedir)) 325 expected = sorted(base.format(sys.implementation.cache_tag, ext) 326 for base in ('__init__.{}.{}', 'bar.{}.{}')) 327 self.assertEqual(sorted(os.listdir(self.pkgdir_cachedir)), expected) 328 # Make sure there are no .pyc files in the source directory. 329 self.assertFalse([fn for fn in os.listdir(self.pkgdir) 330 if fn.endswith(ext)]) 331 locals()['test_pep3147_paths_' + name] = f 332 333 def test_legacy_paths(self): 334 # Ensure that with the proper switch, compileall leaves legacy 335 # pyc files, and no __pycache__ directory. 336 self.assertRunOK('-b', '-q', self.pkgdir) 337 # Verify the __pycache__ directory contents. 338 self.assertFalse(os.path.exists(self.pkgdir_cachedir)) 339 expected = sorted(['__init__.py', '__init__.pyc', 'bar.py', 340 'bar.pyc']) 341 self.assertEqual(sorted(os.listdir(self.pkgdir)), expected) 342 343 def test_multiple_runs(self): 344 # Bug 8527 reported that multiple calls produced empty 345 # __pycache__/__pycache__ directories. 346 self.assertRunOK('-q', self.pkgdir) 347 # Verify the __pycache__ directory contents. 348 self.assertTrue(os.path.exists(self.pkgdir_cachedir)) 349 cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__') 350 self.assertFalse(os.path.exists(cachecachedir)) 351 # Call compileall again. 352 self.assertRunOK('-q', self.pkgdir) 353 self.assertTrue(os.path.exists(self.pkgdir_cachedir)) 354 self.assertFalse(os.path.exists(cachecachedir)) 355 356 def test_force(self): 357 self.assertRunOK('-q', self.pkgdir) 358 pycpath = importlib.util.cache_from_source(self.barfn) 359 # set atime/mtime backward to avoid file timestamp resolution issues 360 os.utime(pycpath, (time.time()-60,)*2) 361 mtime = os.stat(pycpath).st_mtime 362 # without force, no recompilation 363 self.assertRunOK('-q', self.pkgdir) 364 mtime2 = os.stat(pycpath).st_mtime 365 self.assertEqual(mtime, mtime2) 366 # now force it. 367 self.assertRunOK('-q', '-f', self.pkgdir) 368 mtime2 = os.stat(pycpath).st_mtime 369 self.assertNotEqual(mtime, mtime2) 370 371 def test_recursion_control(self): 372 subpackage = os.path.join(self.pkgdir, 'spam') 373 os.mkdir(subpackage) 374 subinitfn = script_helper.make_script(subpackage, '__init__', '') 375 hamfn = script_helper.make_script(subpackage, 'ham', '') 376 self.assertRunOK('-q', '-l', self.pkgdir) 377 self.assertNotCompiled(subinitfn) 378 self.assertFalse(os.path.exists(os.path.join(subpackage, '__pycache__'))) 379 self.assertRunOK('-q', self.pkgdir) 380 self.assertCompiled(subinitfn) 381 self.assertCompiled(hamfn) 382 383 def test_recursion_limit(self): 384 subpackage = os.path.join(self.pkgdir, 'spam') 385 subpackage2 = os.path.join(subpackage, 'ham') 386 subpackage3 = os.path.join(subpackage2, 'eggs') 387 for pkg in (subpackage, subpackage2, subpackage3): 388 script_helper.make_pkg(pkg) 389 390 subinitfn = os.path.join(subpackage, '__init__.py') 391 hamfn = script_helper.make_script(subpackage, 'ham', '') 392 spamfn = script_helper.make_script(subpackage2, 'spam', '') 393 eggfn = script_helper.make_script(subpackage3, 'egg', '') 394 395 self.assertRunOK('-q', '-r 0', self.pkgdir) 396 self.assertNotCompiled(subinitfn) 397 self.assertFalse( 398 os.path.exists(os.path.join(subpackage, '__pycache__'))) 399 400 self.assertRunOK('-q', '-r 1', self.pkgdir) 401 self.assertCompiled(subinitfn) 402 self.assertCompiled(hamfn) 403 self.assertNotCompiled(spamfn) 404 405 self.assertRunOK('-q', '-r 2', self.pkgdir) 406 self.assertCompiled(subinitfn) 407 self.assertCompiled(hamfn) 408 self.assertCompiled(spamfn) 409 self.assertNotCompiled(eggfn) 410 411 self.assertRunOK('-q', '-r 5', self.pkgdir) 412 self.assertCompiled(subinitfn) 413 self.assertCompiled(hamfn) 414 self.assertCompiled(spamfn) 415 self.assertCompiled(eggfn) 416 417 def test_quiet(self): 418 noisy = self.assertRunOK(self.pkgdir) 419 quiet = self.assertRunOK('-q', self.pkgdir) 420 self.assertNotEqual(b'', noisy) 421 self.assertEqual(b'', quiet) 422 423 def test_silent(self): 424 script_helper.make_script(self.pkgdir, 'crunchyfrog', 'bad(syntax') 425 _, quiet, _ = self.assertRunNotOK('-q', self.pkgdir) 426 _, silent, _ = self.assertRunNotOK('-qq', self.pkgdir) 427 self.assertNotEqual(b'', quiet) 428 self.assertEqual(b'', silent) 429 430 def test_regexp(self): 431 self.assertRunOK('-q', '-x', r'ba[^\\/]*$', self.pkgdir) 432 self.assertNotCompiled(self.barfn) 433 self.assertCompiled(self.initfn) 434 435 def test_multiple_dirs(self): 436 pkgdir2 = os.path.join(self.directory, 'foo2') 437 os.mkdir(pkgdir2) 438 init2fn = script_helper.make_script(pkgdir2, '__init__', '') 439 bar2fn = script_helper.make_script(pkgdir2, 'bar2', '') 440 self.assertRunOK('-q', self.pkgdir, pkgdir2) 441 self.assertCompiled(self.initfn) 442 self.assertCompiled(self.barfn) 443 self.assertCompiled(init2fn) 444 self.assertCompiled(bar2fn) 445 446 def test_d_compile_error(self): 447 script_helper.make_script(self.pkgdir, 'crunchyfrog', 'bad(syntax') 448 rc, out, err = self.assertRunNotOK('-q', '-d', 'dinsdale', self.pkgdir) 449 self.assertRegex(out, b'File "dinsdale') 450 451 def test_d_runtime_error(self): 452 bazfn = script_helper.make_script(self.pkgdir, 'baz', 'raise Exception') 453 self.assertRunOK('-q', '-d', 'dinsdale', self.pkgdir) 454 fn = script_helper.make_script(self.pkgdir, 'bing', 'import baz') 455 pyc = importlib.util.cache_from_source(bazfn) 456 os.rename(pyc, os.path.join(self.pkgdir, 'baz.pyc')) 457 os.remove(bazfn) 458 rc, out, err = script_helper.assert_python_failure(fn, __isolated=False) 459 self.assertRegex(err, b'File "dinsdale') 460 461 def test_include_bad_file(self): 462 rc, out, err = self.assertRunNotOK( 463 '-i', os.path.join(self.directory, 'nosuchfile'), self.pkgdir) 464 self.assertRegex(out, b'rror.*nosuchfile') 465 self.assertNotRegex(err, b'Traceback') 466 self.assertFalse(os.path.exists(importlib.util.cache_from_source( 467 self.pkgdir_cachedir))) 468 469 def test_include_file_with_arg(self): 470 f1 = script_helper.make_script(self.pkgdir, 'f1', '') 471 f2 = script_helper.make_script(self.pkgdir, 'f2', '') 472 f3 = script_helper.make_script(self.pkgdir, 'f3', '') 473 f4 = script_helper.make_script(self.pkgdir, 'f4', '') 474 with open(os.path.join(self.directory, 'l1'), 'w') as l1: 475 l1.write(os.path.join(self.pkgdir, 'f1.py')+os.linesep) 476 l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep) 477 self.assertRunOK('-i', os.path.join(self.directory, 'l1'), f4) 478 self.assertCompiled(f1) 479 self.assertCompiled(f2) 480 self.assertNotCompiled(f3) 481 self.assertCompiled(f4) 482 483 def test_include_file_no_arg(self): 484 f1 = script_helper.make_script(self.pkgdir, 'f1', '') 485 f2 = script_helper.make_script(self.pkgdir, 'f2', '') 486 f3 = script_helper.make_script(self.pkgdir, 'f3', '') 487 f4 = script_helper.make_script(self.pkgdir, 'f4', '') 488 with open(os.path.join(self.directory, 'l1'), 'w') as l1: 489 l1.write(os.path.join(self.pkgdir, 'f2.py')+os.linesep) 490 self.assertRunOK('-i', os.path.join(self.directory, 'l1')) 491 self.assertNotCompiled(f1) 492 self.assertCompiled(f2) 493 self.assertNotCompiled(f3) 494 self.assertNotCompiled(f4) 495 496 def test_include_on_stdin(self): 497 f1 = script_helper.make_script(self.pkgdir, 'f1', '') 498 f2 = script_helper.make_script(self.pkgdir, 'f2', '') 499 f3 = script_helper.make_script(self.pkgdir, 'f3', '') 500 f4 = script_helper.make_script(self.pkgdir, 'f4', '') 501 p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-'])) 502 p.stdin.write((f3+os.linesep).encode('ascii')) 503 script_helper.kill_python(p) 504 self.assertNotCompiled(f1) 505 self.assertNotCompiled(f2) 506 self.assertCompiled(f3) 507 self.assertNotCompiled(f4) 508 509 def test_compiles_as_much_as_possible(self): 510 bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error') 511 rc, out, err = self.assertRunNotOK('nosuchfile', self.initfn, 512 bingfn, self.barfn) 513 self.assertRegex(out, b'rror') 514 self.assertNotCompiled(bingfn) 515 self.assertCompiled(self.initfn) 516 self.assertCompiled(self.barfn) 517 518 def test_invalid_arg_produces_message(self): 519 out = self.assertRunOK('badfilename') 520 self.assertRegex(out, b"Can't list 'badfilename'") 521 522 @skipUnless(_have_multiprocessing, "requires multiprocessing") 523 def test_workers(self): 524 bar2fn = script_helper.make_script(self.directory, 'bar2', '') 525 files = [] 526 for suffix in range(5): 527 pkgdir = os.path.join(self.directory, 'foo{}'.format(suffix)) 528 os.mkdir(pkgdir) 529 fn = script_helper.make_script(pkgdir, '__init__', '') 530 files.append(script_helper.make_script(pkgdir, 'bar2', '')) 531 532 self.assertRunOK(self.directory, '-j', '0') 533 self.assertCompiled(bar2fn) 534 for file in files: 535 self.assertCompiled(file) 536 537 @mock.patch('compileall.compile_dir') 538 def test_workers_available_cores(self, compile_dir): 539 with mock.patch("sys.argv", 540 new=[sys.executable, self.directory, "-j0"]): 541 compileall.main() 542 self.assertTrue(compile_dir.called) 543 self.assertEqual(compile_dir.call_args[-1]['workers'], None) 544 545 546 if __name__ == "__main__": 547 unittest.main() 548