From 3089018e951dcca568574c0e0ebf9d8aab112389 Mon Sep 17 00:00:00 2001 From: Simon Charette Date: Sun, 14 Feb 2021 22:42:47 -0500 Subject: [PATCH] Fixed #32446 -- Deprecated SERIALIZE test database setting. Whether or not the state of a test database should be serialized can be inferred from the set of databases allowed to be access from discovered TestCase/TransactionTestCase enabling the serialized_rollback feature which makes this setting unnecessary. This should make a significant test suite bootstraping time difference on large projects that didn't explicitly disable test database serialization. --- django/test/runner.py | 19 +++++++++--- django/test/utils.py | 35 +++++++++++++++++++++-- docs/internals/deprecation.txt | 2 ++ docs/ref/settings.txt | 6 ++++ docs/releases/4.0.txt | 10 +++++-- docs/topics/testing/advanced.txt | 11 ++++++- tests/test_runner/test_discover_runner.py | 20 ++++++++----- tests/test_runner/tests.py | 15 ++++++++-- tests/test_runner_apps/databases/tests.py | 5 ++++ 9 files changed, 103 insertions(+), 20 deletions(-) diff --git a/django/test/runner.py b/django/test/runner.py index e14c6381eb..ff646b7692 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -682,14 +682,18 @@ class DiscoverRunner: return len(result.failures) + len(result.errors) def _get_databases(self, suite): - databases = set() + databases = {} for test in suite: if isinstance(test, unittest.TestCase): test_databases = getattr(test, 'databases', None) if test_databases == '__all__': - return set(connections) + test_databases = connections if test_databases: - databases.update(test_databases) + serialized_rollback = getattr(test, 'serialized_rollback', False) + databases.update( + (alias, serialized_rollback or databases.get(alias, False)) + for alias in test_databases + ) else: databases.update(self._get_databases(test)) return databases @@ -717,8 +721,15 @@ class DiscoverRunner: self.setup_test_environment() suite = self.build_suite(test_labels, extra_tests) databases = self.get_databases(suite) + serialized_aliases = set( + alias + for alias, serialize in databases.items() if serialize + ) with self.time_keeper.timed('Total database setup'): - old_config = self.setup_databases(aliases=databases) + old_config = self.setup_databases( + aliases=databases, + serialized_aliases=serialized_aliases, + ) run_failed = False try: self.run_checks(databases) diff --git a/django/test/utils.py b/django/test/utils.py index b91532e597..c019d77354 100644 --- a/django/test/utils.py +++ b/django/test/utils.py @@ -25,6 +25,7 @@ from django.db.models.options import Options from django.template import Template from django.test.signals import setting_changed, template_rendered from django.urls import get_script_prefix, set_script_prefix +from django.utils.deprecation import RemovedInDjango50Warning from django.utils.translation import deactivate try: @@ -156,8 +157,18 @@ def teardown_test_environment(): del mail.outbox -def setup_databases(verbosity, interactive, *, time_keeper=None, keepdb=False, debug_sql=False, parallel=0, - aliases=None, **kwargs): +def setup_databases( + verbosity, + interactive, + *, + time_keeper=None, + keepdb=False, + debug_sql=False, + parallel=0, + aliases=None, + serialized_aliases=None, + **kwargs, +): """Create the test databases.""" if time_keeper is None: time_keeper = NullTimeKeeper() @@ -176,11 +187,29 @@ def setup_databases(verbosity, interactive, *, time_keeper=None, keepdb=False, d if first_alias is None: first_alias = alias with time_keeper.timed(" Creating '%s'" % alias): + # RemovedInDjango50Warning: when the deprecation ends, + # replace with: + # serialize_alias = serialized_aliases is None or alias in serialized_aliases + try: + serialize_alias = connection.settings_dict['TEST']['SERIALIZE'] + except KeyError: + serialize_alias = ( + serialized_aliases is None or + alias in serialized_aliases + ) + else: + warnings.warn( + 'The SERIALIZE test database setting is ' + 'deprecated as it can be inferred from the ' + 'TestCase/TransactionTestCase.databases that ' + 'enable the serialized_rollback feature.', + category=RemovedInDjango50Warning, + ) connection.creation.create_test_db( verbosity=verbosity, autoclobber=not interactive, keepdb=keepdb, - serialize=connection.settings_dict['TEST'].get('SERIALIZE', True), + serialize=serialize_alias, ) if parallel > 1: for index in range(parallel): diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 4db5d685d7..861242dcfd 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -15,6 +15,8 @@ about each item can often be found in the release notes of two versions prior. See the :ref:`Django 4.0 release notes ` for more details on these changes. +* The ``SERIALIZE`` test setting will be removed. + .. _deprecation-removed-in-4.1: 4.1 diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 3f18af978d..18eb941ce8 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -821,6 +821,12 @@ the database state between tests if you don't have transactions). You can set this to ``False`` to speed up creation time if you don't have any test classes with :ref:`serialized_rollback=True `. +.. deprecated:: 4.0 + + This setting is deprecated as it can be inferred from the + :attr:`~django.test.TestCase.databases` with the + :ref:`serialized_rollback ` option enabled. + .. setting:: TEST_TEMPLATE ``TEMPLATE`` diff --git a/docs/releases/4.0.txt b/docs/releases/4.0.txt index 502aabd4d1..50afdb56f2 100644 --- a/docs/releases/4.0.txt +++ b/docs/releases/4.0.txt @@ -248,7 +248,11 @@ Templates Tests ~~~~~ -* ... +* The new ``serialized_aliases`` argument of + :func:`django.test.utils.setup_databases` determines which + :setting:`DATABASES` aliases test databases should have their state + serialized to allow usage of the + :ref:`serialized_rollback ` feature. URLs ~~~~ @@ -313,7 +317,9 @@ Features deprecated in 4.0 Miscellaneous ------------- -* ... +* ``SERIALIZE`` test setting is deprecated as it can be inferred from the + :attr:`~django.test.TestCase.databases` with the + :ref:`serialized_rollback ` option enabled. Features removed in 4.0 ======================= diff --git a/docs/topics/testing/advanced.txt b/docs/topics/testing/advanced.txt index b20812c745..97072eea83 100644 --- a/docs/topics/testing/advanced.txt +++ b/docs/topics/testing/advanced.txt @@ -718,7 +718,7 @@ utility methods in the ``django.test.utils`` module. Performs global post-test teardown, such as removing instrumentation from the template system and restoring normal email services. -.. function:: setup_databases(verbosity, interactive, *, time_keeper=None, keepdb=False, debug_sql=False, parallel=0, aliases=None, **kwargs) +.. function:: setup_databases(verbosity, interactive, *, time_keeper=None, keepdb=False, debug_sql=False, parallel=0, aliases=None, serialized_aliases=None, **kwargs) Creates the test databases. @@ -730,11 +730,20 @@ utility methods in the ``django.test.utils`` module. databases should be setup for. If it's not provided, it defaults to all of :setting:`DATABASES` aliases. + The ``serialized_aliases`` argument determines what subset of ``aliases`` + test databases should have their state serialized to allow usage of the + :ref:`serialized_rollback ` feature. If + it's not provided, it defaults to ``aliases``. + .. versionchanged:: 3.2 The ``time_keeper`` kwarg was added, and all kwargs were made keyword-only. + .. versionchanged:: 4.0 + + The ``serialized_aliases`` kwarg was added. + .. function:: teardown_databases(old_config, parallel=0, keepdb=False) Destroys the test databases, restoring pre-test conditions. diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py index ee95f9da72..1b5177ccf7 100644 --- a/tests/test_runner/test_discover_runner.py +++ b/tests/test_runner/test_discover_runner.py @@ -354,7 +354,7 @@ class DiscoverRunnerGetDatabasesTests(SimpleTestCase): def assertSkippedDatabases(self, test_labels, expected_databases): databases, output = self.get_databases(test_labels) self.assertEqual(databases, expected_databases) - skipped_databases = set(connections) - expected_databases + skipped_databases = set(connections) - set(expected_databases) if skipped_databases: self.assertIn(self.skip_msg + ', '.join(sorted(skipped_databases)), output) else: @@ -362,31 +362,37 @@ class DiscoverRunnerGetDatabasesTests(SimpleTestCase): def test_mixed(self): databases, output = self.get_databases(['test_runner_apps.databases.tests']) - self.assertEqual(databases, set(connections)) + self.assertEqual(databases, {'default': True, 'other': False}) self.assertNotIn(self.skip_msg, output) def test_all(self): databases, output = self.get_databases(['test_runner_apps.databases.tests.AllDatabasesTests']) - self.assertEqual(databases, set(connections)) + self.assertEqual(databases, {alias: False for alias in connections}) self.assertNotIn(self.skip_msg, output) def test_default_and_other(self): self.assertSkippedDatabases([ 'test_runner_apps.databases.tests.DefaultDatabaseTests', 'test_runner_apps.databases.tests.OtherDatabaseTests', - ], {'default', 'other'}) + ], {'default': False, 'other': False}) def test_default_only(self): self.assertSkippedDatabases([ 'test_runner_apps.databases.tests.DefaultDatabaseTests', - ], {'default'}) + ], {'default': False}) def test_other_only(self): self.assertSkippedDatabases([ 'test_runner_apps.databases.tests.OtherDatabaseTests' - ], {'other'}) + ], {'other': False}) def test_no_databases_required(self): self.assertSkippedDatabases([ 'test_runner_apps.databases.tests.NoDatabaseTests' - ], set()) + ], {}) + + def test_serialize(self): + databases, _ = self.get_databases([ + 'test_runner_apps.databases.tests.DefaultDatabaseSerializedTests' + ]) + self.assertEqual(databases, {'default': True}) diff --git a/tests/test_runner/tests.py b/tests/test_runner/tests.py index 87d432a2ab..5dc3126581 100644 --- a/tests/test_runner/tests.py +++ b/tests/test_runner/tests.py @@ -11,12 +11,15 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.core.management.base import SystemCheckError -from django.test import TransactionTestCase, skipUnlessDBFeature +from django.test import ( + SimpleTestCase, TransactionTestCase, skipUnlessDBFeature, +) from django.test.runner import DiscoverRunner from django.test.testcases import connections_support_transactions from django.test.utils import ( captured_stderr, dependency_ordered, get_unique_databases_and_mirrors, ) +from django.utils.deprecation import RemovedInDjango50Warning from .models import B, Person, Through @@ -315,7 +318,7 @@ class AliasedDefaultTestSetupTest(unittest.TestCase): runner_instance.teardown_databases(old_config) -class SetupDatabasesTests(unittest.TestCase): +class SetupDatabasesTests(SimpleTestCase): def setUp(self): self.runner_instance = DiscoverRunner(verbosity=0) @@ -398,9 +401,15 @@ class SetupDatabasesTests(unittest.TestCase): 'TEST': {'SERIALIZE': False}, }, }) + msg = ( + 'The SERIALIZE test database setting is deprecated as it can be ' + 'inferred from the TestCase/TransactionTestCase.databases that ' + 'enable the serialized_rollback feature.' + ) with mock.patch('django.db.backends.dummy.base.DatabaseWrapper.creation_class') as mocked_db_creation: with mock.patch('django.test.utils.connections', new=tested_connections): - self.runner_instance.setup_databases() + with self.assertWarnsMessage(RemovedInDjango50Warning, msg): + self.runner_instance.setup_databases() mocked_db_creation.return_value.create_test_db.assert_called_once_with( verbosity=0, autoclobber=False, serialize=False, keepdb=False ) diff --git a/tests/test_runner_apps/databases/tests.py b/tests/test_runner_apps/databases/tests.py index 4be260e689..dbb3b46e0b 100644 --- a/tests/test_runner_apps/databases/tests.py +++ b/tests/test_runner_apps/databases/tests.py @@ -10,6 +10,11 @@ class DefaultDatabaseTests(NoDatabaseTests): databases = {'default'} +class DefaultDatabaseSerializedTests(NoDatabaseTests): + databases = {'default'} + serialized_rollback = True + + class OtherDatabaseTests(NoDatabaseTests): databases = {'other'}