Fixed #22487: Optional rollback emulation for migrated apps

django/conf/global_settings.py

@@ -578,6 +578,10 @@ DEFAULT_EXCEPTION_REPORTER_FILTER = 'django.views.debug.SafeExceptionReporterFil
 # The name of the class to use to run the test suite
 TEST_RUNNER = 'django.test.runner.DiscoverRunner'
 
+# Apps that don't need to be serialized at test database creation time
+# (only apps with migrations are to start with)
+TEST_NON_SERIALIZED_APPS = []
+
 ############
 # FIXTURES #
 ############

django/core/management/commands/flush.py

@@ -22,10 +22,9 @@ class Command(NoArgsCommand):
         make_option('--no-initial-data', action='store_false', dest='load_initial_data', default=True,
             help='Tells Django not to load any initial data after database synchronization.'),
     )
-    help = ('Returns the database to the state it was in immediately after '
+    help = ('Removes ALL DATA from the database, including data added during '
-           'migrate was first executed. This means that all data will be removed '
+           'migrations. Unmigrated apps will also have their initial_data '
-           'from the database, any post-migration handlers will be '
+           'fixture reloaded. Does not achieve a "fresh install" state.')
-           're-executed, and the initial_data fixture will be re-installed.')
 
     def handle_noargs(self, **options):
         database = options.get('database')
@@ -54,7 +53,7 @@ class Command(NoArgsCommand):
         if interactive:
             confirm = input("""You have requested a flush of the database.
 This will IRREVERSIBLY DESTROY all data currently in the %r database,
-and return each table to a fresh state.
+and return each table to an empty state.
 Are you sure you want to do this?
 
     Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])

django/core/management/commands/testserver.py

@@ -27,7 +27,7 @@ class Command(BaseCommand):
         addrport = options.get('addrport')
 
         # Create a test database.
-        db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive)
+        db_name = connection.creation.create_test_db(verbosity=verbosity, autoclobber=not interactive, serialize=False)
 
         # Import the fixture data into the test database.
         call_command('loaddata', *fixture_labels, **{'verbosity': verbosity})

django/db/backends/creation.py

@@ -7,6 +7,11 @@ from django.db.utils import load_backend
 from django.utils.encoding import force_bytes
 from django.utils.functional import cached_property
 from django.utils.six.moves import input
+from django.utils.six import StringIO
+from django.core.management.commands.dumpdata import sort_dependencies
+from django.db import router
+from django.apps import apps
+from django.core import serializers
 
 from .utils import truncate_name
 
@@ -332,7 +337,7 @@ class BaseDatabaseCreation(object):
             ";",
         ]
 
-    def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False):
+    def create_test_db(self, verbosity=1, autoclobber=False, keepdb=False, serialize=True):
         """
         Creates a test database, prompting the user for confirmation if the
         database already exists. Returns the name of the test database created.
@@ -364,25 +369,31 @@ class BaseDatabaseCreation(object):
         settings.DATABASES[self.connection.alias]["NAME"] = test_database_name
         self.connection.settings_dict["NAME"] = test_database_name
 
-        # Report migrate messages at one level lower than that requested.
+        # We report migrate messages at one level lower than that requested.
         # This ensures we don't get flooded with messages during testing
-        # (unless you really ask to be flooded)
+        # (unless you really ask to be flooded).
-        call_command('migrate',
+        call_command(
+            'migrate',
             verbosity=max(verbosity - 1, 0),
             interactive=False,
             database=self.connection.alias,
-            load_initial_data=False,
+            test_database=True,
-            test_database=True)
+        )
 
-        # We need to then do a flush to ensure that any data installed by
+        # We then serialize the current state of the database into a string
-        # custom SQL has been removed. The only test data should come from
+        # and store it on the connection. This slightly horrific process is so people
-        # test fixtures, or autogenerated from post_migrate triggers.
+        # who are testing on databases without transactions or who are using
-        # This has the side effect of loading initial data (which was
+        # a TransactionTestCase still get a clean database on every test run.
-        # intentionally skipped in the syncdb).
+        if serialize:
-        call_command('flush',
+            self.connection._test_serialized_contents = self.serialize_db_to_string()
+
+        # Finally, we flush the database to clean
+        call_command(
+            'flush',
             verbosity=max(verbosity - 1, 0),
             interactive=False,
-            database=self.connection.alias)
+            database=self.connection.alias
+        )
 
         call_command('createcachetable', database=self.connection.alias)
 
@@ -391,6 +402,44 @@ class BaseDatabaseCreation(object):
 
         return test_database_name
 
+    def serialize_db_to_string(self):
+        """
+        Serializes all data in the database into a JSON string.
+        Designed only for test runner usage; will not handle large
+        amounts of data.
+        """
+        # Build list of all apps to serialize
+        from django.db.migrations.loader import MigrationLoader
+        loader = MigrationLoader(self.connection)
+        app_list = []
+        for app_config in apps.get_app_configs():
+            if (
+                app_config.models_module is not None and
+                app_config.label in loader.migrated_apps and
+                app_config.name not in settings.TEST_NON_SERIALIZED_APPS
+            ):
+                app_list.append((app_config, None))
+        # Make a function to iteratively return every object
+        def get_objects():
+            for model in sort_dependencies(app_list):
+                if not model._meta.proxy and router.allow_migrate(self.connection.alias, model):
+                    queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
+                    for obj in queryset.iterator():
+                        yield obj
+        # Serialise to a string
+        out = StringIO()
+        serializers.serialize("json", get_objects(), indent=None, stream=out)
+        return out.getvalue()
+
+    def deserialize_db_from_string(self, data):
+        """
+        Reloads the database with data from a string generated by
+        the serialize_db_to_string method.
+        """
+        data = StringIO(data)
+        for obj in serializers.deserialize("json", data, using=self.connection.alias):
+            obj.save()
+
     def _get_test_db_name(self):
         """
         Internal implementation - returns the name of the test DB that will be

django/test/runner.py

@@ -298,7 +298,11 @@ def setup_databases(verbosity, interactive, keepdb=False, **kwargs):
             connection = connections[alias]
             if test_db_name is None:
                 test_db_name = connection.creation.create_test_db(
-                    verbosity, autoclobber=not interactive, keepdb=keepdb)
+                    verbosity,
+                    autoclobber=not interactive,
+                    keepdb=keepdb,
+                    serialize=connection.settings_dict.get("TEST_SERIALIZE", True),
+                )
                 destroy = True
             else:
                 connection.settings_dict['NAME'] = test_db_name

django/test/testcases.py

@@ -753,6 +753,12 @@ class TransactionTestCase(SimpleTestCase):
     # Subclasses can define fixtures which will be automatically installed.
     fixtures = None
 
+    # If transactions aren't available, Django will serialize the database
+    # contents into a fixture during setup and flush and reload them
+    # during teardown (as flush does not restore data from migrations).
+    # This can be slow; this flag allows enabling on a per-case basis.
+    serialized_rollback = False
+
     def _pre_setup(self):
         """Performs any pre-test setup. This includes:
 
@@ -808,6 +814,17 @@ class TransactionTestCase(SimpleTestCase):
             if self.reset_sequences:
                 self._reset_sequences(db_name)
 
+            # If we need to provide replica initial data from migrated apps,
+            # then do so.
+            if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
+                if self.available_apps is not None:
+                    apps.unset_available_apps()
+                connections[db_name].creation.deserialize_db_from_string(
+                    connections[db_name]._test_serialized_contents
+                )
+                if self.available_apps is not None:
+                    apps.set_available_apps(self.available_apps)
+
             if self.fixtures:
                 # We have to use this slightly awkward syntax due to the fact
                 # that we're using *args and **kwargs together.
@@ -844,12 +861,14 @@ class TransactionTestCase(SimpleTestCase):
         # Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
         # when flushing only a subset of the apps
         for db_name in self._databases_names(include_mirrors=False):
+            # Flush the database
             call_command('flush', verbosity=0, interactive=False,
                          database=db_name, skip_checks=True,
                          reset_sequences=False,
                          allow_cascade=self.available_apps is not None,
                          inhibit_post_migrate=self.available_apps is not None)
 
+
     def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
         items = six.moves.map(transform, qs)
         if not ordered:

docs/ref/migration-operations.txt

@@ -199,8 +199,9 @@ model::
         # We get the model from the versioned app registry;
         # if we directly import it, it'll be the wrong version
         Country = apps.get_model("myapp", "Country")
-        Country.objects.create(name="USA", code="us")
+        db_alias = schema_editor.connection.alias
-        Country.objects.create(name="France", code="fr")
+        Country.objects.create(name="USA", code="us", using=db_alias)
+        Country.objects.create(name="France", code="fr", using=db_alias)
 
     class Migration(migrations.Migration):
 
@@ -236,6 +237,14 @@ Oracle). This should be safe, but may cause a crash if you attempt to use
 the ``schema_editor`` provided on these backends; in this case, please
 set ``atomic=False``.
 
+.. warning::
+
+    RunPython does not magically alter the connection of the models for you;
+    any model methods you call will go to the default database unless you
+    give them the current database alias (available from
+    ``schema_editor.connection.alias``, where ``schema_editor`` is the second
+    argument to your function).
+
 SeparateDatabaseAndState
 ------------------------
 

docs/ref/settings.txt

@@ -2078,6 +2078,24 @@ Default: ``'django.test.runner.DiscoverRunner'``
 The name of the class to use for starting the test suite. See
 :ref:`other-testing-frameworks`.
 
+.. setting:: TEST_NON_SERIALIZED_APPS
+
+TEST_NON_SERIALIZED_APPS
+------------------------
+
+Default: ``[]``
+
+In order to restore the database state between tests for TransactionTestCases
+and database backends without transactions, Django will :ref:`serialize the
+contents of all apps with migrations <test-case-serialized-rollback>` when it
+starts the test run so it can then reload from that copy before tests that
+need it.
+
+This slows down the startup time of the test runner; if you have apps that
+you know don't need this feature, you can add their full names in here (e.g.
+``django.contrib.contenttypes``) to exclude them from this serialization
+process.
+
 .. setting:: THOUSAND_SEPARATOR
 
 THOUSAND_SEPARATOR

docs/releases/1.7.txt

@@ -63,6 +63,10 @@ but a few of the key features are:
 * ``initial_data`` fixtures are no longer loaded for apps with migrations; if
   you want to load initial data for an app, we suggest you do it in a migration.
 
+* Test rollback behaviour is different for apps with migrations; in particular,
+  Django will no longer emulate rollbacks on non-transactional databases or
+  inside ``TransactionTestCase`` :ref:`unless specifically asked <test-case-serialized-rollback>`.
+
 App-loading refactor
 ~~~~~~~~~~~~~~~~~~~~
 

docs/topics/testing/advanced.txt

@@ -485,7 +485,7 @@ django.db.connection.creation
 The creation module of the database backend also provides some utilities that
 can be useful during testing.
 
-.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False])
+.. function:: create_test_db([verbosity=1, autoclobber=False, keepdb=False, serialize=True])
 
     Creates a new test database and runs ``migrate`` against it.
 
@@ -507,6 +507,12 @@ can be useful during testing.
     a new database will be created, prompting the user to remove
     the existing one, if present.
 
+    ``serialize`` determines if Django serializes the database into an
+    in-memory JSON string before running tests (used to restore the database
+    state between tests if you don't have transactions). You can set this to
+    False to significantly speed up creation time if you know you don't need
+    data persistance outside of test fixtures.
+
     Returns the name of the test database that it created.
 
     ``create_test_db()`` has the side effect of modifying the value of

docs/topics/testing/overview.txt

@@ -234,6 +234,33 @@ the Django test runner reorders tests in the following way:
     database by a given :class:`~django.test.TransactionTestCase` test, they
     must be updated to be able to run independently.
 
+.. _test-case-serialized-rollback:
+
+Rollback emulation
+------------------
+
+Any initial data loaded in migrations will only be available in ``TestCase``
+tests and not in ``TransactionTestCase`` tests, and additionally only on
+backends where transactions are supported (the most important exception being
+MyISAM).
+
+Django can re-load that data for you on a per-testcase basis by
+setting the ``serialized_rollback`` option to ``True`` in the body of the
+``TestCase`` or ``TransactionTestCase``, but note that this will slow down
+that test suite by approximately 3x.
+
+Third-party apps or those developing against MyISAM will need to set this;
+in general, however, you should be developing your own projects against a
+transactional database and be using ``TestCase`` for most tests, and thus
+not need this setting.
+
+The initial serialization is usually very quick, but if you wish to exclude
+some apps from this process (and speed up test runs slightly), you may add
+those apps to :setting:`TEST_NON_SERIALIZED_APPS`.
+
+Apps without migrations are not affected; ``initial_data`` fixtures are
+reloaded as usual.
+
 Other test conditions
 ---------------------
 
@@ -249,6 +276,7 @@ used. This behavior `may change`_ in the future.
 
 .. _may change: https://code.djangoproject.com/ticket/11505
 
+
 Understanding the test output
 -----------------------------
 

docs/topics/testing/tools.txt

@@ -600,9 +600,17 @@ to test the effects of commit and rollback:
   guarantees that the rollback at the end of the test restores the database to
   its initial state.
 
-  When running on a database that does not support rollback (e.g. MySQL with the
+.. warning::
-  MyISAM storage engine), ``TestCase`` falls back to initializing the database
+
-  by truncating tables and reloading initial data.
+  ``TestCase`` running on a database that does not support rollback (e.g. MySQL with the
+  MyISAM storage engine), and all instances of ``TransactionTestCase``, will
+  roll back at the end of the test by deleting all data from the test database
+  and reloading initial data for apps without migrations.
+
+  Apps with migrations :ref:`will not see their data reloaded <test-case-serialized-rollback>`;
+  if you need this functionality (for example, third-party apps should enable
+  this) you can set ``serialized_rollback = True`` inside the
+  ``TestCase`` body.
 
 .. warning::
 

tests/migration_test_data_persistence/migrations/0001_initial.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+def add_book(apps, schema_editor):
+    apps.get_model("migration_test_data_persistence", "Book").objects.using(
+        schema_editor.connection.alias,
+    ).create(
+        title="I Love Django",
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Book',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', primary_key=True, serialize=False, auto_created=True)),
+                ('title', models.CharField(max_length=100)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+        migrations.RunPython(
+            add_book,
+        ),
+    ]

tests/migration_test_data_persistence/models.py

@@ -0,0 +1,5 @@
+from django.db import models
+
+
+class Book(models.Model):
+    title = models.CharField(max_length=100)

tests/migration_test_data_persistence/tests.py

@@ -0,0 +1,33 @@
+from django.test import TransactionTestCase
+from .models import Book
+
+
+class MigrationDataPersistenceTestCase(TransactionTestCase):
+    """
+    Tests that data loaded in migrations is available if we set
+    serialized_rollback = True.
+    """
+
+    available_apps = ["migration_test_data_persistence"]
+    serialized_rollback = True
+
+    def test_persistence(self):
+        self.assertEqual(
+            Book.objects.count(),
+            1,
+        )
+
+
+class MigrationDataNoPersistenceTestCase(TransactionTestCase):
+    """
+    Tests the failure case
+    """
+
+    available_apps = ["migration_test_data_persistence"]
+    serialized_rollback = False
+
+    def test_no_persistence(self):
+        self.assertEqual(
+            Book.objects.count(),
+            0,
+        )