Fixed #25251 -- Made data migrations available in TransactionTestCase when using --keepdb.

Data loaded in migrations were restored at the beginning of each
TransactionTestCase and all the tables are truncated at the end of
these test cases. If there was a TransactionTestCase at the end of
the test suite, the migrated data weren't restored in the database
(especially unexpected when using --keepdb). Now data is restored
at the end of each TransactionTestCase.

django/test/runner.py

@@ -11,7 +11,7 @@ from io import StringIO
 
 from django.core.management import call_command
 from django.db import connections
-from django.test import SimpleTestCase, TestCase
+from django.test import SimpleTestCase, TestCase, TransactionTestCase
 from django.test.utils import (
     setup_databases as _setup_databases, setup_test_environment,
     teardown_databases as _teardown_databases, teardown_test_environment,
@@ -399,7 +399,7 @@ class DiscoverRunner:
     parallel_test_suite = ParallelTestSuite
     test_runner = unittest.TextTestRunner
     test_loader = unittest.defaultTestLoader
-    reorder_by = (TestCase, SimpleTestCase)
+    reorder_by = (TestCase, TransactionTestCase, SimpleTestCase)
 
     def __init__(self, pattern=None, top_level=None, verbosity=1,
                  interactive=True, failfast=False, keepdb=False,
@@ -637,6 +637,22 @@ def is_discoverable(label):
     return os.path.isdir(os.path.abspath(label))
 
 
+def reorder_postprocess(reordered_suite):
+    """
+    To make TransactionTestCases initialize their data properly, they must know
+    if the next TransactionTestCase needs initial data migrations serialized in
+    the connection. Initialize _next_serialized_rollback attribute depending on
+    the serialized_rollback option present in the next test class in the suite.
+    If the next test has no serialized_rollback attribute, it means there
+    aren't any more TransactionTestCases.
+    """
+    for previous_test, next_test in zip(reordered_suite._tests[:-1], reordered_suite._tests[1:]):
+        next_serialized_rollback = getattr(next_test, 'serialized_rollback', None)
+        if next_serialized_rollback is not None:
+            previous_test._next_serialized_rollback = next_serialized_rollback
+    return reordered_suite
+
+
 def reorder_suite(suite, classes, reverse=False):
     """
     Reorder a test suite by test type.
@@ -656,7 +672,7 @@ def reorder_suite(suite, classes, reverse=False):
     reordered_suite = suite_class()
     for i in range(class_count + 1):
         reordered_suite.addTests(bins[i])
-    return reordered_suite
+    return reorder_postprocess(reordered_suite)
 
 
 def partition_suite_by_type(suite, classes, bins, reverse=False):

django/test/testcases.py

@@ -827,6 +827,15 @@ class TransactionTestCase(SimpleTestCase):
     # This can be slow; this flag allows enabling on a per-case basis.
     serialized_rollback = False
 
+    # This attribute is strongly linked to serialized_rollback parameter and
+    # allows the data restoration after the database flush, at the end of the
+    # test, if the next test needs the initial data. This attribute is updated
+    # by the test runner when the test suite is built. Being initialized to
+    # True is crucial: the last TransactionTestCase, which doesn't have any
+    # test classes with the serialized_rollback attribute, will always have
+    # this value set to True.
+    _next_serialized_rollback = True
+
     # Since tests will be wrapped in a transaction, or serialized if they
     # are not available, we allow queries to be run.
     allow_database_queries = True
@@ -897,17 +906,6 @@ class TransactionTestCase(SimpleTestCase):
             if self.reset_sequences:
                 self._reset_sequences(db_name)
 
-            # If we need to provide replica initial data from migrated apps,
-            # then do so.
-            if self.serialized_rollback and hasattr(connections[db_name], "_test_serialized_contents"):
-                if self.available_apps is not None:
-                    apps.unset_available_apps()
-                connections[db_name].creation.deserialize_db_from_string(
-                    connections[db_name]._test_serialized_contents
-                )
-                if self.available_apps is not None:
-                    apps.set_available_apps(self.available_apps)
-
             if self.fixtures:
                 # We have to use this slightly awkward syntax due to the fact
                 # that we're using *args and **kwargs together.
@@ -961,6 +959,15 @@ class TransactionTestCase(SimpleTestCase):
                          database=db_name, reset_sequences=False,
                          allow_cascade=self.available_apps is not None,
                          inhibit_post_migrate=inhibit_post_migrate)
+            # Provide replica initial data from migrated apps, if needed.
+            if self._next_serialized_rollback and hasattr(connections[db_name], '_test_serialized_contents'):
+                if self.available_apps is not None:
+                    apps.unset_available_apps()
+                connections[db_name].creation.deserialize_db_from_string(
+                    connections[db_name]._test_serialized_contents
+                )
+                if self.available_apps is not None:
+                    apps.set_available_apps(self.available_apps)
 
     def assertQuerysetEqual(self, qs, values, transform=repr, ordered=True, msg=None):
         items = map(transform, qs)

docs/ref/settings.txt

@@ -777,6 +777,11 @@ the database state between tests if you don't have transactions). You can set
 this to ``False`` to speed up creation time if you don't have any test classes
 with :ref:`serialized_rollback=True <test-case-serialized-rollback>`.
 
+Don't set this to ``False`` if you want to use :option:`test --keepdb`
+and your test suite contains :class:`~django.test.TransactionTestCase` or
+doesn't support transactions, as this in-memory JSON string is used to restore
+the initial data migrations in these situations.
+
 .. setting:: TEST_TEMPLATE
 
 ``TEMPLATE``

docs/releases/2.2.txt

@@ -298,6 +298,17 @@ Database backend API
 
 * Support for GDAL 1.9 and 1.10 is dropped.
 
+``TransactionTestCase`` serialized data loading
+-----------------------------------------------
+
+Initial data migrations are now loaded in
+:class:`~django.test.TransactionTestCase` at the end of the test, after the
+database flush. In older versions, this data was loaded at the beginning of the
+test, but this prevents the :option:`test --keepdb` option from working
+properly (the database was empty at the end of the whole test suite). This
+change shouldn't have an impact on your tests unless you've customized
+:class:`~django.test.TransactionTestCase`'s internals.
+
 Miscellaneous
 -------------
 

tests/test_discovery_sample3/tests_transaction_test_case_mixed.py

@@ -0,0 +1,37 @@
+from unittest import TestCase
+
+from django.test import TestCase as DjangoTestCase, TransactionTestCase
+
+
+class TestVanillaUnittest(TestCase):
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestDjangoTestCase(DjangoTestCase):
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestTransactionTestCase1(TransactionTestCase):
+    available_apps = ['test_discovery_sample3']
+    serialized_rollback = False
+
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestTransactionTestCase2(TransactionTestCase):
+    available_apps = ['test_discovery_sample3']
+    serialized_rollback = True
+
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestTransactionTestCase3(TransactionTestCase):
+    available_apps = ['test_discovery_sample3']
+    serialized_rollback = False
+
+    def test_sample(self):
+        self.assertEqual(1, 1)

tests/test_discovery_sample3/tests_transaction_test_case_ordering.py

@@ -0,0 +1,28 @@
+from unittest import TestCase
+
+from django.test import (
+    SimpleTestCase, TestCase as DjangoTestCase, TransactionTestCase,
+)
+
+
+class TestDjangoTestCase(DjangoTestCase):
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestVanillaUnittest(TestCase):
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestZimpleTestCase(SimpleTestCase):
+    # Z gets this test to appear after Vanilla in the default suite.
+    def test_sample(self):
+        self.assertEqual(1, 1)
+
+
+class TestTransactionTestCase(TransactionTestCase):
+    available_apps = ['test_discovery_sample3']
+
+    def test_sample(self):
+        self.assertEqual(1, 1)

tests/test_runner/test_discover_runner.py

@@ -223,3 +223,31 @@ class DiscoverRunnerTest(TestCase):
         with captured_stdout() as stdout:
             runner.build_suite(['test_runner_apps.tagged.tests'])
             self.assertIn('Excluding test tag(s): bar, foo.\n', stdout.getvalue())
+
+    def test_transaction_test_case_before_simple_test_case(self):
+        runner = DiscoverRunner()
+        suite = runner.build_suite(['test_discovery_sample3.tests_transaction_test_case_ordering'])
+        suite = tuple(suite)
+        # TransactionTestCase is second after TestCase.
+        self.assertIn('TestTransactionTestCase', suite[1].id())
+
+    def test_transaction_test_case_next_serialized_rollback_option(self):
+        runner = DiscoverRunner()
+        suite = runner.build_suite(['test_discovery_sample3.tests_transaction_test_case_mixed'])
+        django_test_case, first_transaction_test_case, middle_transaction_test_case, \
+            last_transaction_test_case, vanilla_test_case = suite
+        # TransactionTestCase1._next_serialized_rollback is
+        # TransactionTestCase2.serialize_rollback.
+        self.assertEqual(
+            first_transaction_test_case._next_serialized_rollback,
+            middle_transaction_test_case.serialized_rollback
+        )
+        # TransactionTestCase2._next_serialized_rollback is
+        # TransactionTestCase3.serialize_rollback.
+        self.assertEqual(
+            middle_transaction_test_case._next_serialized_rollback,
+            last_transaction_test_case.serialized_rollback
+        )
+        # The last TransactionTestCase of the suite has
+        # _next_serialized_rollback to = True.
+        self.assertIs(last_transaction_test_case._next_serialized_rollback, True)

tests/test_utils/test_transactiontestcase.py

@@ -1,10 +1,44 @@
+import json
 from unittest import mock
 
+from django.apps import apps
 from django.db import connections
 from django.test import TestCase, TransactionTestCase, override_settings
 
+from .models import Car
 
-class TestSerializedRollbackInhibitsPostMigrate(TransactionTestCase):
+
+class TestSerializedContentMockMixin:
+    """
+    Use this mixin on each test involving TransactionTestCase and
+    serialized_rollback = True option to avoid test dependencies. It mocks what
+    would be serialized after initial data migrations and restores it at the
+    end of the test.
+    """
+    initial_data_migration = '[]'
+    _connections_test_serialized_content = {}
+
+    def _pre_setup(self):
+        for db_name in self._databases_names(include_mirrors=False):
+            self._connections_test_serialized_content[db_name] = connections[db_name]._test_serialized_contents
+            connections[db_name]._test_serialized_contents = self.initial_data_migration
+        super()._pre_setup()
+
+    def _post_teardown(self):
+        super()._post_teardown()
+        for db_name in self._databases_names(include_mirrors=False):
+            connections[db_name]._test_serialized_contents = self._connections_test_serialized_content[db_name]
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        # Clean up any data that has been created by the class.
+        for data in json.loads(cls.initial_data_migration):
+            model = apps.get_model(*data['model'].split('.'))
+            model.objects.filter(pk=data['pk']).delete()
+
+
+class TestSerializedRollbackInhibitsPostMigrate(TestSerializedContentMockMixin, TransactionTestCase):
     """
     TransactionTestCase._fixture_teardown() inhibits the post_migrate signal
     for test classes with serialized_rollback=True.
@@ -44,3 +78,39 @@ class TransactionTestCaseMultiDbTests(TestCase):
         """
         for alias in connections:
             self.assertEqual(len(connections[alias].queries_log), 0, 'Failed for alias %s' % alias)
+
+
+class TestDataRestoredOnTearDownIfSerializedRollback(TestSerializedContentMockMixin, TransactionTestCase):
+    """
+    Initial data is recreated in TransactionTestCase._fixture_teardown()
+    after the database is flushed so it's available in next test.
+    """
+    available_apps = ['test_utils']
+    _next_serialized_rollback = True
+    initial_data_migration = '[{"model": "test_utils.car", "pk": 666, "fields": {"name": "K 2000"}}]'
+
+    def _post_teardown(self):
+        super()._post_teardown()
+        # Won't be True if running the tests with --reverse.
+        if self._next_serialized_rollback:
+            self.assertTrue(Car.objects.exists())
+
+    def test(self):
+        pass  # Should be the only one in this class.
+
+
+class TestDataNotRestoredOnTearDownIfNotSerializedRollback(TestSerializedContentMockMixin, TransactionTestCase):
+    """
+    Initial data isn't recreated in TransactionTestCase._fixture_teardown()
+    if _next_serialized_rollback is False.
+    """
+    available_apps = ['test_utils']
+    _next_serialized_rollback = False
+    initial_data_migration = '[{"model": "test_utils.car", "pk": 666, "fields": {"name": "K 2000"}}]'
+
+    def _post_teardown(self):
+        super()._post_teardown()
+        self.assertFalse(Car.objects.exists())
+
+    def test(self):
+        pass  # Should be the only one in this class.