[1.2.X] Fixed #14799 -- Provided a full solution for test database creation order problems.

Backport of r14822, r14823 and r14824 from trunk.

git-svn-id: http://code.djangoproject.com/svn/django/branches/releases/1.2.X@14825 bcc190cf-cafb-0310-a4f2-bffc1f526a37

django/test/simple.py

@@ -3,11 +3,18 @@ import signal
 import unittest
 
 from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
 from django.db.models import get_app, get_apps
 from django.test import _doctest as doctest
 from django.test.utils import setup_test_environment, teardown_test_environment
 from django.test.testcases import OutputChecker, DocTestRunner, TestCase
 
+
+try:
+    all
+except NameError:
+    from django.utils.itercompat import all
+
 # The module name for tests outside models.py
 TEST_MODULE = 'tests'
 
@@ -222,6 +229,40 @@ def reorder_suite(suite, classes):
         bins[0].addTests(bins[i+1])
     return bins[0]
 
+def dependency_ordered(test_databases, dependencies):
+    """Reorder test_databases into an order that honors the dependencies
+    described in TEST_DEPENDENCIES.
+    """
+    ordered_test_databases = []
+    resolved_databases = set()
+    while test_databases:
+        changed = False
+        deferred = []
+
+        while test_databases:
+            signature, aliases = test_databases.pop()
+            dependencies_satisfied = True
+            for alias in aliases:
+                if alias in dependencies:
+                    if all(a in resolved_databases for a in dependencies[alias]):
+                        # all dependencies for this alias are satisfied
+                        dependencies.pop(alias)
+                        resolved_databases.add(alias)
+                    else:
+                        dependencies_satisfied = False
+                else:
+                    resolved_databases.add(alias)
+
+            if dependencies_satisfied:
+                ordered_test_databases.append((signature, aliases))
+                changed = True
+            else:
+                deferred.append((signature, aliases))
+
+        if not changed:
+            raise ImproperlyConfigured("Circular dependency in TEST_DEPENDENCIES")
+        test_databases = deferred
+    return ordered_test_databases
 
 class DjangoTestSuiteRunner(object):
     def __init__(self, verbosity=1, interactive=True, failfast=True, **kwargs):
@@ -260,6 +301,7 @@ class DjangoTestSuiteRunner(object):
         # and which ones are test mirrors or duplicate entries in DATABASES
         mirrored_aliases = {}
         test_databases = {}
+        dependencies = {}
         for alias in connections:
             connection = connections[alias]
             if connection.settings_dict['TEST_MIRROR']:
@@ -276,21 +318,17 @@ class DjangoTestSuiteRunner(object):
                         connection.settings_dict['ENGINE'],
                         connection.settings_dict['NAME'],
                     ), []).append(alias)
-        
+
-        # Re-order the list of databases to create, making sure the default
+                if 'TEST_DEPENDENCIES' in connection.settings_dict:
-        # database is first. Otherwise, creation order is semi-random (i.e. 
+                    dependencies[alias] = connection.settings_dict['TEST_DEPENDENCIES']
-        # dict ordering dependent).
+                else:
-        dbs_to_create = []
+                    if alias != 'default':
-        for dbinfo, aliases in test_databases.items():
+                        dependencies[alias] = connection.settings_dict.get('TEST_DEPENDENCIES', ['default'])
-            if DEFAULT_DB_ALIAS in aliases:
+
-                dbs_to_create.insert(0, (dbinfo, aliases))
+        # Second pass -- actually create the databases.
-            else:
-                dbs_to_create.append((dbinfo, aliases))
-                
-        # Final pass -- actually create the databases.
         old_names = []
         mirrors = []
-        for (host, port, engine, db_name), aliases in dbs_to_create:
+        for (host, port, engine, db_name), aliases in dependency_ordered(test_databases.items(), dependencies):
             # Actually create the database for the first connection
             connection = connections[aliases[0]]
             old_names.append((connection, db_name, True))

docs/ref/settings.txt

@@ -380,6 +380,20 @@ Only supported for the ``mysql`` backend (see the `MySQL manual`_ for details).
 
 .. _MySQL manual: MySQL_
 
+.. setting:: TEST_DEPENDENCIES
+
+TEST_DEPENDENCIES
+~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.2.4
+
+Default: ``['default']``, for all databases other than ``default``,
+which has no dependencies.
+
+The creation-order dependencies of the database. See the documentation
+on :ref:`controlling the creation order of test databases
+<topics-testing-creation-dependencies>` for details.
+
 .. setting:: TEST_MIRROR
 
 TEST_MIRROR

docs/releases/1.2.4.txt

@@ -0,0 +1,39 @@
+==========================
+Django 1.2.4 release notes
+==========================
+
+Welcome to Django 1.2.4!
+
+This is the fourth "bugfix" release in the Django 1.2 series,
+improving the stability and performance of the Django 1.2 codebase.
+
+Django 1.2.4 maintains backwards compatibility with Django
+1.2.3, but contain a number of fixes and other
+improvements. Django 1.2.4 is a recommended upgrade for any
+development or deployment currently using or targeting Django 1.2.
+
+For full details on the new features, backwards incompatibilities, and
+deprecated features in the 1.2 branch, see the :doc:`/releases/1.2`.
+
+One new feature
+===============
+
+Ordinarily, a point release would not include new features, but in the
+case of Django 1.2.4, we have made an exception to this rule.
+
+One of the bugs fixed in Django 1.2.4 involves a set of
+circumstances whereby a running a test suite on a multiple database
+configuration could cause the original source database (i.e., the
+actual production database) to be dropped, causing catastrophic loss
+of data. In order to provide a fix for this problem, it was necessary
+to introduce a new setting -- :setting:`TEST_DEPENDENCIES` -- that
+allows you to define any creation order dependencies in your database
+configuration.
+
+Most users -- even users with multiple-database configurations -- need
+not be concerned about the data loss bug, or the manual configuration of
+:setting:`TEST_DEPENDENCIES`. See the `original problem report`_
+documentation on :ref:`controlling the creation order of test
+databases <topics-testing-creation-dependencies>` for details.
+
+.. _original problem report: http://code.djangoproject.com/ticket/14415

docs/releases/index.txt

@@ -19,6 +19,7 @@ Final releases
 .. toctree::
    :maxdepth: 1
 
+   1.2.4
    1.2.2
    1.2
 

docs/topics/testing.txt

@@ -422,6 +422,53 @@ will be redirected to point at ``default``. As a result, writes to
 the same database, not because there is data replication between the
 two databases.
 
+.. _topics-testing-creation-dependencies:
+
+Controlling creation order for test databases
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.2.4
+
+By default, Django will always create the ``default`` database first.
+However, no guarantees are made on the creation order of any other
+databases in your test setup.
+
+If your database configuration requires a specific creation order, you
+can specify the dependencies that exist using the
+:setting:`TEST_DEPENDENCIES` setting. Consider the following
+(simplified) example database configuration::
+
+    DATABASES = {
+        'default': {
+             # ... db settings
+             'TEST_DEPENDENCIES': ['diamonds']
+        },
+        'diamonds': {
+            # ... db settings
+        },
+        'clubs': {
+            # ... db settings
+            'TEST_DEPENDENCIES': ['diamonds']
+        },
+        'spades': {
+            # ... db settings
+            'TEST_DEPENDENCIES': ['diamonds','hearts']
+        },
+        'hearts': {
+            # ... db settings
+            'TEST_DEPENDENCIES': ['diamonds','clubs']
+        }
+    }
+
+Under this configuration, the ``diamonds`` database will be created first,
+as it is the only database alias without dependencies. The ``default``` and
+``clubs`` alias will be created next (although the order of creation of this
+pair is not guaranteed); then ``hearts``; and finally ``spades``.
+
+If there are any circular dependencies in the
+:setting:`TEST_DEPENDENCIES` definition, an ``ImproperlyConfigured``
+exception will be raised.
+
 Other test conditions
 ---------------------
 

tests/regressiontests/test_runner/tests.py

@@ -4,6 +4,7 @@ Tests for django test runner
 import StringIO
 import unittest
 import django
+from django.core.exceptions import ImproperlyConfigured
 from django.test import simple
 
 class DjangoTestRunnerTests(unittest.TestCase):
@@ -27,3 +28,93 @@ class DjangoTestRunnerTests(unittest.TestCase):
         result = dtr.run(suite)
         self.assertEqual(1, result.testsRun)
         self.assertEqual(1, len(result.failures))
+
+class DependencyOrderingTests(unittest.TestCase):
+
+    def test_simple_dependencies(self):
+        raw = [
+            ('s1', ['alpha']),
+            ('s2', ['bravo']),
+            ('s3', ['charlie']),
+        ]
+        dependencies = {
+            'alpha': ['charlie'],
+            'bravo': ['charlie'],
+        }
+
+        ordered = simple.dependency_ordered(raw, dependencies=dependencies)
+        ordered_sigs = [sig for sig,aliases in ordered]
+
+        self.assertIn('s1', ordered_sigs)
+        self.assertIn('s2', ordered_sigs)
+        self.assertIn('s3', ordered_sigs)
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))
+
+    def test_chained_dependencies(self):
+        raw = [
+            ('s1', ['alpha']),
+            ('s2', ['bravo']),
+            ('s3', ['charlie']),
+        ]
+        dependencies = {
+            'alpha': ['bravo'],
+            'bravo': ['charlie'],
+        }
+
+        ordered = simple.dependency_ordered(raw, dependencies=dependencies)
+        ordered_sigs = [sig for sig,aliases in ordered]
+
+        self.assertIn('s1', ordered_sigs)
+        self.assertIn('s2', ordered_sigs)
+        self.assertIn('s3', ordered_sigs)
+
+        # Explicit dependencies
+        self.assertLess(ordered_sigs.index('s2'), ordered_sigs.index('s1'))
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))
+
+        # Implied dependencies
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))
+
+    def test_multiple_dependencies(self):
+        raw = [
+            ('s1', ['alpha']),
+            ('s2', ['bravo']),
+            ('s3', ['charlie']),
+            ('s4', ['delta']),
+        ]
+        dependencies = {
+            'alpha': ['bravo','delta'],
+            'bravo': ['charlie'],
+            'delta': ['charlie'],
+        }
+
+        ordered = simple.dependency_ordered(raw, dependencies=dependencies)
+        ordered_sigs = [sig for sig,aliases in ordered]
+
+        self.assertIn('s1', ordered_sigs)
+        self.assertIn('s2', ordered_sigs)
+        self.assertIn('s3', ordered_sigs)
+        self.assertIn('s4', ordered_sigs)
+
+        # Explicit dependencies
+        self.assertLess(ordered_sigs.index('s2'), ordered_sigs.index('s1'))
+        self.assertLess(ordered_sigs.index('s4'), ordered_sigs.index('s1'))
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s2'))
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s4'))
+
+        # Implicit dependencies
+        self.assertLess(ordered_sigs.index('s3'), ordered_sigs.index('s1'))
+
+    def test_circular_dependencies(self):
+        raw = [
+            ('s1', ['alpha']),
+            ('s2', ['bravo']),
+        ]
+        dependencies = {
+            'bravo': ['alpha'],
+            'alpha': ['bravo'],
+        }
+
+        self.assertRaises(ImproperlyConfigured, simple.dependency_ordered, raw, dependencies=dependencies)
+