Fixed #26840 -- Added test.utils.setup/teardown_databases().

django/test/runner.py

@@ -1,4 +1,3 @@
-import collections
 import ctypes
 import itertools
 import logging
@@ -7,13 +6,17 @@ import os
 import pickle
 import textwrap
 import unittest
+import warnings
 from importlib import import_module
 
-from django.core.exceptions import ImproperlyConfigured
+from django.db import connections
-from django.db import DEFAULT_DB_ALIAS, connections
 from django.test import SimpleTestCase, TestCase
-from django.test.utils import setup_test_environment, teardown_test_environment
+from django.test.utils import (
+    setup_databases as _setup_databases, setup_test_environment,
+    teardown_databases as _teardown_databases, teardown_test_environment,
+)
 from django.utils.datastructures import OrderedSet
+from django.utils.deprecation import RemovedInDjango21Warning
 from django.utils.six import StringIO
 
 try:
@@ -498,7 +501,7 @@ class DiscoverRunner(object):
         return suite
 
     def setup_databases(self, **kwargs):
-        return setup_databases(
+        return _setup_databases(
             self.verbosity, self.interactive, self.keepdb, self.debug_sql,
             self.parallel, **kwargs
         )
@@ -522,16 +525,12 @@ class DiscoverRunner(object):
         """
         Destroys all the non-mirror databases.
         """
-        for connection, old_name, destroy in old_config:
+        _teardown_databases(
-            if destroy:
+            old_config,
-                if self.parallel > 1:
+            verbosity=self.verbosity,
-                    for index in range(self.parallel):
+            parallel=self.parallel,
-                        connection.creation.destroy_test_db(
+            keepdb=self.keepdb,
-                            number=index + 1,
+        )
-                            verbosity=self.verbosity,
-                            keepdb=self.keepdb,
-                        )
-                connection.creation.destroy_test_db(old_name, self.verbosity, self.keepdb)
 
     def teardown_test_environment(self, **kwargs):
         unittest.removeHandler()
@@ -577,48 +576,6 @@ def is_discoverable(label):
     return os.path.isdir(os.path.abspath(label))
 
 
-def dependency_ordered(test_databases, dependencies):
-    """
-    Reorder test_databases into an order that honors the dependencies
-    described in TEST[DEPENDENCIES].
-    """
-    ordered_test_databases = []
-    resolved_databases = set()
-
-    # Maps db signature to dependencies of all it's aliases
-    dependencies_map = {}
-
-    # sanity check - no DB can depend on its own alias
-    for sig, (_, aliases) in test_databases:
-        all_deps = set()
-        for alias in aliases:
-            all_deps.update(dependencies.get(alias, []))
-        if not all_deps.isdisjoint(aliases):
-            raise ImproperlyConfigured(
-                "Circular dependency: databases %r depend on each other, "
-                "but are aliases." % aliases)
-        dependencies_map[sig] = all_deps
-
-    while test_databases:
-        changed = False
-        deferred = []
-
-        # Try to find a DB that has all it's dependencies met
-        for signature, (db_name, aliases) in test_databases:
-            if dependencies_map[signature].issubset(resolved_databases):
-                resolved_databases.update(aliases)
-                ordered_test_databases.append((signature, (db_name, aliases)))
-                changed = True
-            else:
-                deferred.append((signature, (db_name, aliases)))
-
-        if not changed:
-            raise ImproperlyConfigured(
-                "Circular dependency in TEST[DEPENDENCIES]")
-        test_databases = deferred
-    return ordered_test_databases
-
-
 def reorder_suite(suite, classes, reverse=False):
     """
     Reorders a test suite by test type.
@@ -682,96 +639,14 @@ def partition_suite_by_case(suite):
     return groups
 
 
-def get_unique_databases_and_mirrors():
+def setup_databases(*args, **kwargs):
-    """
+    warnings.warn(
-    Figure out which databases actually need to be created.
+        '`django.test.runner.setup_databases()` has moved to '
-
+        '`django.test.utils.setup_databases()`.',
-    Deduplicate entries in DATABASES that correspond the same database or are
+        RemovedInDjango21Warning,
-    configured as test mirrors.
+        stacklevel=2,
-
+    )
-    Return two values:
+    return _setup_databases(*args, **kwargs)
-    - test_databases: ordered mapping of signatures to (name, list of aliases)
-                      where all aliases share the same underlying database.
-    - mirrored_aliases: mapping of mirror aliases to original aliases.
-    """
-    mirrored_aliases = {}
-    test_databases = {}
-    dependencies = {}
-    default_sig = connections[DEFAULT_DB_ALIAS].creation.test_db_signature()
-
-    for alias in connections:
-        connection = connections[alias]
-        test_settings = connection.settings_dict['TEST']
-
-        if test_settings['MIRROR']:
-            # If the database is marked as a test mirror, save the alias.
-            mirrored_aliases[alias] = test_settings['MIRROR']
-        else:
-            # Store a tuple with DB parameters that uniquely identify it.
-            # If we have two aliases with the same values for that tuple,
-            # we only need to create the test database once.
-            item = test_databases.setdefault(
-                connection.creation.test_db_signature(),
-                (connection.settings_dict['NAME'], set())
-            )
-            item[1].add(alias)
-
-            if 'DEPENDENCIES' in test_settings:
-                dependencies[alias] = test_settings['DEPENDENCIES']
-            else:
-                if alias != DEFAULT_DB_ALIAS and connection.creation.test_db_signature() != default_sig:
-                    dependencies[alias] = test_settings.get('DEPENDENCIES', [DEFAULT_DB_ALIAS])
-
-    test_databases = dependency_ordered(test_databases.items(), dependencies)
-    test_databases = collections.OrderedDict(test_databases)
-    return test_databases, mirrored_aliases
-
-
-def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, parallel=0, **kwargs):
-    """
-    Creates the test databases.
-    """
-    test_databases, mirrored_aliases = get_unique_databases_and_mirrors()
-
-    old_names = []
-
-    for signature, (db_name, aliases) in test_databases.items():
-        first_alias = None
-        for alias in aliases:
-            connection = connections[alias]
-            old_names.append((connection, db_name, first_alias is None))
-
-            # Actually create the database for the first connection
-            if first_alias is None:
-                first_alias = alias
-                connection.creation.create_test_db(
-                    verbosity=verbosity,
-                    autoclobber=not interactive,
-                    keepdb=keepdb,
-                    serialize=connection.settings_dict.get("TEST", {}).get("SERIALIZE", True),
-                )
-                if parallel > 1:
-                    for index in range(parallel):
-                        connection.creation.clone_test_db(
-                            number=index + 1,
-                            verbosity=verbosity,
-                            keepdb=keepdb,
-                        )
-            # Configure all other connections as mirrors of the first one
-            else:
-                connections[alias].creation.set_as_test_mirror(
-                    connections[first_alias].settings_dict)
-
-    # Configure the test mirrors.
-    for alias, mirror_alias in mirrored_aliases.items():
-        connections[alias].creation.set_as_test_mirror(
-            connections[mirror_alias].settings_dict)
-
-    if debug_sql:
-        for alias in connections:
-            connections[alias].force_debug_cursor = True
-
-    return old_names
 
 
 def filter_tests_by_tags(suite, tags, exclude_tags):

django/test/utils.py

@@ -1,3 +1,4 @@
+import collections
 import logging
 import re
 import sys
@@ -12,8 +13,9 @@ from django.apps import apps
 from django.apps.registry import Apps
 from django.conf import UserSettingsHolder, settings
 from django.core import mail
+from django.core.exceptions import ImproperlyConfigured
 from django.core.signals import request_started
-from django.db import reset_queries
+from django.db import DEFAULT_DB_ALIAS, connections, reset_queries
 from django.db.models.options import Options
 from django.template import Template
 from django.test.signals import setting_changed, template_rendered
@@ -155,6 +157,155 @@ def teardown_test_environment():
     del mail.outbox
 
 
+def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, parallel=0, **kwargs):
+    """
+    Create the test databases.
+    """
+    test_databases, mirrored_aliases = get_unique_databases_and_mirrors()
+
+    old_names = []
+
+    for signature, (db_name, aliases) in test_databases.items():
+        first_alias = None
+        for alias in aliases:
+            connection = connections[alias]
+            old_names.append((connection, db_name, first_alias is None))
+
+            # Actually create the database for the first connection
+            if first_alias is None:
+                first_alias = alias
+                connection.creation.create_test_db(
+                    verbosity=verbosity,
+                    autoclobber=not interactive,
+                    keepdb=keepdb,
+                    serialize=connection.settings_dict.get('TEST', {}).get('SERIALIZE', True),
+                )
+                if parallel > 1:
+                    for index in range(parallel):
+                        connection.creation.clone_test_db(
+                            number=index + 1,
+                            verbosity=verbosity,
+                            keepdb=keepdb,
+                        )
+            # Configure all other connections as mirrors of the first one
+            else:
+                connections[alias].creation.set_as_test_mirror(connections[first_alias].settings_dict)
+
+    # Configure the test mirrors.
+    for alias, mirror_alias in mirrored_aliases.items():
+        connections[alias].creation.set_as_test_mirror(
+            connections[mirror_alias].settings_dict)
+
+    if debug_sql:
+        for alias in connections:
+            connections[alias].force_debug_cursor = True
+
+    return old_names
+
+
+def dependency_ordered(test_databases, dependencies):
+    """
+    Reorder test_databases into an order that honors the dependencies
+    described in TEST[DEPENDENCIES].
+    """
+    ordered_test_databases = []
+    resolved_databases = set()
+
+    # Maps db signature to dependencies of all its aliases
+    dependencies_map = {}
+
+    # Check that no database depends on its own alias
+    for sig, (_, aliases) in test_databases:
+        all_deps = set()
+        for alias in aliases:
+            all_deps.update(dependencies.get(alias, []))
+        if not all_deps.isdisjoint(aliases):
+            raise ImproperlyConfigured(
+                "Circular dependency: databases %r depend on each other, "
+                "but are aliases." % aliases
+            )
+        dependencies_map[sig] = all_deps
+
+    while test_databases:
+        changed = False
+        deferred = []
+
+        # Try to find a DB that has all its dependencies met
+        for signature, (db_name, aliases) in test_databases:
+            if dependencies_map[signature].issubset(resolved_databases):
+                resolved_databases.update(aliases)
+                ordered_test_databases.append((signature, (db_name, aliases)))
+                changed = True
+            else:
+                deferred.append((signature, (db_name, aliases)))
+
+        if not changed:
+            raise ImproperlyConfigured("Circular dependency in TEST[DEPENDENCIES]")
+        test_databases = deferred
+    return ordered_test_databases
+
+
+def get_unique_databases_and_mirrors():
+    """
+    Figure out which databases actually need to be created.
+
+    Deduplicate entries in DATABASES that correspond the same database or are
+    configured as test mirrors.
+
+    Return two values:
+    - test_databases: ordered mapping of signatures to (name, list of aliases)
+                      where all aliases share the same underlying database.
+    - mirrored_aliases: mapping of mirror aliases to original aliases.
+    """
+    mirrored_aliases = {}
+    test_databases = {}
+    dependencies = {}
+    default_sig = connections[DEFAULT_DB_ALIAS].creation.test_db_signature()
+
+    for alias in connections:
+        connection = connections[alias]
+        test_settings = connection.settings_dict['TEST']
+
+        if test_settings['MIRROR']:
+            # If the database is marked as a test mirror, save the alias.
+            mirrored_aliases[alias] = test_settings['MIRROR']
+        else:
+            # Store a tuple with DB parameters that uniquely identify it.
+            # If we have two aliases with the same values for that tuple,
+            # we only need to create the test database once.
+            item = test_databases.setdefault(
+                connection.creation.test_db_signature(),
+                (connection.settings_dict['NAME'], set())
+            )
+            item[1].add(alias)
+
+            if 'DEPENDENCIES' in test_settings:
+                dependencies[alias] = test_settings['DEPENDENCIES']
+            else:
+                if alias != DEFAULT_DB_ALIAS and connection.creation.test_db_signature() != default_sig:
+                    dependencies[alias] = test_settings.get('DEPENDENCIES', [DEFAULT_DB_ALIAS])
+
+    test_databases = dependency_ordered(test_databases.items(), dependencies)
+    test_databases = collections.OrderedDict(test_databases)
+    return test_databases, mirrored_aliases
+
+
+def teardown_databases(old_config, verbosity, parallel=0, keepdb=False):
+    """
+    Destroy all the non-mirror databases.
+    """
+    for connection, old_name, destroy in old_config:
+        if destroy:
+            if parallel > 1:
+                for index in range(parallel):
+                    connection.creation.destroy_test_db(
+                        number=index + 1,
+                        verbosity=verbosity,
+                        keepdb=keepdb,
+                    )
+            connection.creation.destroy_test_db(old_name, verbosity, keepdb)
+
+
 def get_runner(settings, test_runner_class=None):
     if not test_runner_class:
         test_runner_class = settings.TEST_RUNNER

docs/internals/deprecation.txt

@@ -23,6 +23,8 @@ details on these changes.
 * The ``extra_context`` parameter of ``contrib.auth.views.logout_then_login()``
   will be removed.
 
+* ``django.test.runner.setup_databases()`` will be removed.
+
 .. _deprecation-removed-in-2.0:
 
 2.0

docs/releases/1.11.txt

@@ -275,6 +275,10 @@ Tests
 * Added the :option:`test --debug-mode` option to help troubleshoot test
   failures by setting the :setting:`DEBUG` setting to ``True``.
 
+* The new :func:`django.test.utils.setup_databases` (moved from
+  ``django.test.runner``) and :func:`~django.test.utils.teardown_databases`
+  functions make it easier to build custom test runners.
+
 URLs
 ~~~~
 
@@ -425,3 +429,6 @@ Miscellaneous
   :class:`~django.contrib.auth.views.PasswordResetDoneView`,
   :class:`~django.contrib.auth.views.PasswordResetConfirmView`, and
   :class:`~django.contrib.auth.views.PasswordResetCompleteView`.
+
+* ``django.test.runner.setup_databases()`` is moved to
+  :func:`django.test.utils.setup_databases`. The old location is deprecated.

docs/topics/testing/advanced.txt

@@ -563,11 +563,8 @@ Methods
 
 .. method:: DiscoverRunner.setup_databases(**kwargs)
 
-    Creates the test databases.
+    Creates the test databases by calling
-
+    :func:`~django.test.utils.setup_databases`.
-    Returns a data structure that provides enough detail to undo the changes
-    that have been made. This data will be provided to the ``teardown_databases()``
-    function at the conclusion of testing.
 
 .. method:: DiscoverRunner.run_suite(suite, **kwargs)
 
@@ -584,11 +581,8 @@ Methods
 
 .. method:: DiscoverRunner.teardown_databases(old_config, **kwargs)
 
-    Destroys the test databases, restoring pre-test conditions.
+    Destroys the test databases, restoring pre-test conditions by calling
-
+    :func:`~django.test.utils.teardown_databases`.
-    ``old_config`` is a data structure defining the changes in the
-    database configuration that need to be reversed. It is the return
-    value of the ``setup_databases()`` method.
 
 .. method:: DiscoverRunner.teardown_test_environment(**kwargs)
 
@@ -629,6 +623,26 @@ utility methods in the ``django.test.utils`` module.
     Performs global post-test teardown, such as removing instrumentation from
     the template system and restoring normal email services.
 
+.. function:: setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, parallel=0, **kwargs)
+
+    .. versionadded:: 1.11
+
+    Creates the test databases.
+
+    Returns a data structure that provides enough detail to undo the changes
+    that have been made. This data will be provided to the
+    :func:`teardown_databases` function at the conclusion of testing.
+
+.. function:: teardown_databases(old_config, parallel=0, keepdb=False)
+
+    .. versionadded:: 1.11
+
+    Destroys the test databases, restoring pre-test conditions.
+
+    ``old_config`` is a data structure defining the changes in the database
+    configuration that need to be reversed. It's the return value of the
+    :meth:`setup_databases` method.
+
 ``django.db.connection.creation``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

tests/test_runner/tests.py

@@ -14,8 +14,9 @@ from django.core.management import call_command
 from django.test import (
     TestCase, TransactionTestCase, mock, skipUnlessDBFeature, testcases,
 )
-from django.test.runner import DiscoverRunner, dependency_ordered
+from django.test.runner import DiscoverRunner
 from django.test.testcases import connections_support_transactions
+from django.test.utils import dependency_ordered
 
 from .models import Person
 
@@ -238,7 +239,7 @@ class DummyBackendTest(unittest.TestCase):
         Test that setup_databases() doesn't fail with dummy database backend.
         """
         tested_connections = db.ConnectionHandler({})
-        with mock.patch('django.test.runner.connections', new=tested_connections):
+        with mock.patch('django.test.utils.connections', new=tested_connections):
             runner_instance = DiscoverRunner(verbosity=0)
             old_config = runner_instance.setup_databases()
             runner_instance.teardown_databases(old_config)
@@ -257,7 +258,7 @@ class AliasedDefaultTestSetupTest(unittest.TestCase):
                 'NAME': 'dummy'
             }
         })
-        with mock.patch('django.test.runner.connections', new=tested_connections):
+        with mock.patch('django.test.utils.connections', new=tested_connections):
             runner_instance = DiscoverRunner(verbosity=0)
             old_config = runner_instance.setup_databases()
             runner_instance.teardown_databases(old_config)
@@ -281,7 +282,7 @@ class SetupDatabasesTests(unittest.TestCase):
         })
 
         with mock.patch('django.db.backends.dummy.base.DatabaseCreation') as mocked_db_creation:
-            with mock.patch('django.test.runner.connections', new=tested_connections):
+            with mock.patch('django.test.utils.connections', new=tested_connections):
                 old_config = self.runner_instance.setup_databases()
                 self.runner_instance.teardown_databases(old_config)
         mocked_db_creation.return_value.destroy_test_db.assert_called_once_with('dbname', 0, False)
@@ -306,7 +307,7 @@ class SetupDatabasesTests(unittest.TestCase):
             },
         })
         with mock.patch('django.db.backends.dummy.base.DatabaseCreation') as mocked_db_creation:
-            with mock.patch('django.test.runner.connections', new=tested_connections):
+            with mock.patch('django.test.utils.connections', new=tested_connections):
                 self.runner_instance.setup_databases()
         mocked_db_creation.return_value.create_test_db.assert_called_once_with(
             verbosity=0, autoclobber=False, serialize=True, keepdb=False
@@ -320,7 +321,7 @@ class SetupDatabasesTests(unittest.TestCase):
             },
         })
         with mock.patch('django.db.backends.dummy.base.DatabaseCreation') as mocked_db_creation:
-            with mock.patch('django.test.runner.connections', new=tested_connections):
+            with mock.patch('django.test.utils.connections', new=tested_connections):
                 self.runner_instance.setup_databases()
         mocked_db_creation.return_value.create_test_db.assert_called_once_with(
             verbosity=0, autoclobber=False, serialize=False, keepdb=False