From 29150d5da880ac1db15e47052330790cf1b802d2 Mon Sep 17 00:00:00 2001 From: Dan Watson Date: Thu, 15 Mar 2018 15:32:56 -0400 Subject: [PATCH] Fixed #23406 -- Allowed migrations to be loaded from .pyc files. --- django/db/migrations/loader.py | 25 +++++++------ docs/releases/2.1.txt | 5 +++ tests/migrations/test_loader.py | 34 ++++++++++++++++++ .../test_migrations_bad_pyc/0001_initial.pyc | Bin 0 -> 511 bytes .../test_migrations_bad_pyc/__init__.py | 0 5 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 tests/migrations/test_migrations_bad_pyc/0001_initial.pyc create mode 100644 tests/migrations/test_migrations_bad_pyc/__init__.py diff --git a/django/db/migrations/loader.py b/django/db/migrations/loader.py index c4d26d1186..4147cc3f09 100644 --- a/django/db/migrations/loader.py +++ b/django/db/migrations/loader.py @@ -1,4 +1,4 @@ -import os +import pkgutil import sys from importlib import import_module, reload @@ -97,17 +97,20 @@ class MigrationLoader: if was_loaded: reload(module) self.migrated_apps.add(app_config.label) - directory = os.path.dirname(module.__file__) - # Scan for .py files - migration_names = set() - for name in os.listdir(directory): - if name.endswith(".py"): - import_name = name.rsplit(".", 1)[0] - if import_name[0] not in "_.~": - migration_names.add(import_name) - # Load them + migration_names = {name for _, name, is_pkg in pkgutil.iter_modules(module.__path__) if not is_pkg} + # Load migrations for migration_name in migration_names: - migration_module = import_module("%s.%s" % (module_name, migration_name)) + migration_path = '%s.%s' % (module_name, migration_name) + try: + migration_module = import_module(migration_path) + except ImportError as e: + if 'bad magic number' in str(e): + raise ImportError( + "Couldn't import %r as it appears to be a stale " + ".pyc file." % migration_path + ) from e + else: + raise if not hasattr(migration_module, "Migration"): raise BadMigrationError( "Migration %s in app %s has no Migration class" % (migration_name, app_config.label) diff --git a/docs/releases/2.1.txt b/docs/releases/2.1.txt index 463fad0b6c..2b1d6dabdc 100644 --- a/docs/releases/2.1.txt +++ b/docs/releases/2.1.txt @@ -194,6 +194,8 @@ Migrations * Added support for serialization of ``functools.partialmethod`` objects. +* To support frozen environments, migrations may be loaded from ``.pyc`` files. + Models ~~~~~~ @@ -366,6 +368,9 @@ Miscellaneous with such passwords from requesting a password reset. Audit your code to confirm that your usage of these APIs don't rely on the old behavior. +* Since migrations are now loaded from ``.pyc`` files, you might need to delete + them if you're working in a mixed Python 2 and Python 3 environment. + .. _deprecated-features-2.1: Features deprecated in 2.1 diff --git a/tests/migrations/test_loader.py b/tests/migrations/test_loader.py index e0e7e3d258..eade230640 100644 --- a/tests/migrations/test_loader.py +++ b/tests/migrations/test_loader.py @@ -1,3 +1,6 @@ +import compileall +import os + from django.db import connection, connections from django.db.migrations.exceptions import ( AmbiguityError, InconsistentMigrationHistory, NodeNotFoundError, @@ -6,6 +9,8 @@ from django.db.migrations.loader import MigrationLoader from django.db.migrations.recorder import MigrationRecorder from django.test import TestCase, modify_settings, override_settings +from .test_base import MigrationTestBase + class RecorderTests(TestCase): """ @@ -494,3 +499,32 @@ class LoaderTests(TestCase): ('app1', '4_auto'), } self.assertEqual(plan, expected_plan) + + +class PycLoaderTests(MigrationTestBase): + + def test_valid(self): + """ + To support frozen environments, MigrationLoader loads .pyc migrations. + """ + with self.temporary_migration_module(module='migrations.test_migrations') as migration_dir: + # Compile .py files to .pyc files and delete .py files. + compileall.compile_dir(migration_dir, force=True, quiet=1, legacy=True) + for name in os.listdir(migration_dir): + if name.endswith('.py'): + os.remove(os.path.join(migration_dir, name)) + loader = MigrationLoader(connection) + self.assertIn(('migrations', '0001_initial'), loader.disk_migrations) + + def test_invalid(self): + """ + MigrationLoader reraises ImportErrors caused by "bad magic number" pyc + files with a more helpful message. + """ + with self.temporary_migration_module(module='migrations.test_migrations_bad_pyc'): + msg = ( + "Couldn't import '\w+.migrations.0001_initial' as it appears " + "to be a stale .pyc file." + ) + with self.assertRaisesRegex(ImportError, msg): + MigrationLoader(connection) diff --git a/tests/migrations/test_migrations_bad_pyc/0001_initial.pyc b/tests/migrations/test_migrations_bad_pyc/0001_initial.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07854f4aebed369ef4a294ad00cab1b2d4f32f61 GIT binary patch literal 511 zcmb`EF;2rU7=>T#w1Ntx4jdv=9VA9Th#iJ>B8wGbx44R(sCI_Va3apYEjR`TfZuKl z7r=^M-t!;-`E2vA^V#Red)?wNIKA({;BUw%dNOo?%}{5^VrJ+VEJw{Lb7+oDj#UPg z;|+KPp6VQGf@LwK4(6Lik&z2?lF^yAS6X%^=xx*aePu=!Yu4K)nBMttO7Yd-j1>`H*qtYQG3;9LM=i>-yp+b7V47|*>?%eJBZrSfQ#R#~#f01e qXEla#OE?uxA?;Xg8n^Bj>bZ(LP0Rm_C7s9h{seC01t+FCoAV!L6L6pa literal 0 HcmV?d00001 diff --git a/tests/migrations/test_migrations_bad_pyc/__init__.py b/tests/migrations/test_migrations_bad_pyc/__init__.py new file mode 100644 index 0000000000..e69de29bb2