Merge pull request #4761 from aaugustin/parallelize-tests-attempt-1

Fixed #20461 -- Allowed running tests in parallel.

django/core/management/commands/test.py

@@ -47,7 +47,7 @@ class Command(BaseCommand):
             action='store', dest='liveserver', default=None,
             help='Overrides the default address where the live server (used '
                  'with LiveServerTestCase) is expected to run from. The '
-                 'default value is localhost:8081.'),
+                 'default value is localhost:8081-8179.'),
 
         test_runner_class = get_runner(settings, self.test_runner)
         if hasattr(test_runner_class, 'option_list'):

django/db/backends/base/base.py

@@ -1,3 +1,4 @@
+import copy
 import time
 import warnings
 from collections import deque
@@ -622,3 +623,16 @@ class BaseDatabaseWrapper(object):
                 func()
         finally:
             self.run_on_commit = []
+
+    def copy(self, alias=None, allow_thread_sharing=None):
+        """
+        Return a copy of this connection.
+
+        For tests that require two connections to the same database.
+        """
+        settings_dict = copy.deepcopy(self.settings_dict)
+        if alias is None:
+            alias = self.alias
+        if allow_thread_sharing is None:
+            allow_thread_sharing = self.allow_thread_sharing
+        return type(self)(settings_dict, alias, allow_thread_sharing)

django/db/backends/base/creation.py

@@ -190,13 +190,56 @@ class BaseDatabaseCreation(object):
 
         return test_database_name
 
-    def destroy_test_db(self, old_database_name, verbosity=1, keepdb=False):
+    def clone_test_db(self, number, verbosity=1, autoclobber=False, keepdb=False):
+        """
+        Clone a test database.
+        """
+        source_database_name = self.connection.settings_dict['NAME']
+
+        if verbosity >= 1:
+            test_db_repr = ''
+            action = 'Cloning test database'
+            if verbosity >= 2:
+                test_db_repr = " ('%s')" % source_database_name
+            if keepdb:
+                action = 'Using existing clone'
+            print("%s for alias '%s'%s..." % (action, self.connection.alias, test_db_repr))
+
+        # We could skip this call if keepdb is True, but we instead
+        # give it the keepdb param. See create_test_db for details.
+        self._clone_test_db(number, verbosity, keepdb)
+
+    def get_test_db_clone_settings(self, number):
+        """
+        Return a modified connection settings dict for the n-th clone of a DB.
+        """
+        # When this function is called, the test database has been created
+        # already and its name has been copied to settings_dict['NAME'] so
+        # we don't need to call _get_test_db_name.
+        orig_settings_dict = self.connection.settings_dict
+        new_settings_dict = orig_settings_dict.copy()
+        new_settings_dict['NAME'] = '{}_{}'.format(orig_settings_dict['NAME'], number)
+        return new_settings_dict
+
+    def _clone_test_db(self, number, verbosity, keepdb=False):
+        """
+        Internal implementation - duplicate the test db tables.
+        """
+        raise NotImplementedError(
+            "The database backend doesn't support cloning databases. "
+            "Disable the option to run tests in parallel processes.")
+
+    def destroy_test_db(self, old_database_name=None, verbosity=1, keepdb=False, number=None):
         """
         Destroy a test database, prompting the user for confirmation if the
         database already exists.
         """
         self.connection.close()
-        test_database_name = self.connection.settings_dict['NAME']
+        if number is None:
+            test_database_name = self.connection.settings_dict['NAME']
+        else:
+            test_database_name = self.get_test_db_clone_settings(number)['NAME']
+
         if verbosity >= 1:
             test_db_repr = ''
             action = 'Destroying'
@@ -213,8 +256,9 @@ class BaseDatabaseCreation(object):
             self._destroy_test_db(test_database_name, verbosity)
 
         # Restore the original database name
-        settings.DATABASES[self.connection.alias]["NAME"] = old_database_name
-        self.connection.settings_dict["NAME"] = old_database_name
+        if old_database_name is not None:
+            settings.DATABASES[self.connection.alias]["NAME"] = old_database_name
+            self.connection.settings_dict["NAME"] = old_database_name
 
     def _destroy_test_db(self, test_database_name, verbosity):
         """

django/db/backends/base/features.py

@@ -212,6 +212,10 @@ class BaseDatabaseFeatures(object):
     # every expression is null?
     greatest_least_ignores_nulls = False
 
+    # Can the backend clone databases for parallel test execution?
+    # Defaults to False to allow third-party backends to opt-in.
+    can_clone_databases = False
+
     def __init__(self, connection):
         self.connection = connection
 

django/db/backends/mysql/creation.py

@@ -1,5 +1,10 @@
+import subprocess
+import sys
+
 from django.db.backends.base.creation import BaseDatabaseCreation
 
+from .client import DatabaseClient
+
 
 class DatabaseCreation(BaseDatabaseCreation):
 
@@ -11,3 +16,34 @@ class DatabaseCreation(BaseDatabaseCreation):
         if test_settings['COLLATION']:
             suffix.append('COLLATE %s' % test_settings['COLLATION'])
         return ' '.join(suffix)
+
+    def _clone_test_db(self, number, verbosity, keepdb=False):
+        qn = self.connection.ops.quote_name
+        source_database_name = self.connection.settings_dict['NAME']
+        target_database_name = self.get_test_db_clone_settings(number)['NAME']
+
+        with self._nodb_connection.cursor() as cursor:
+            try:
+                cursor.execute("CREATE DATABASE %s" % qn(target_database_name))
+            except Exception as e:
+                if keepdb:
+                    return
+                try:
+                    if verbosity >= 1:
+                        print("Destroying old test database '%s'..." % self.connection.alias)
+                    cursor.execute("DROP DATABASE %s" % qn(target_database_name))
+                    cursor.execute("CREATE DATABASE %s" % qn(target_database_name))
+                except Exception as e:
+                    sys.stderr.write("Got an error recreating the test database: %s\n" % e)
+                    sys.exit(2)
+
+        dump_cmd = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict)
+        dump_cmd[0] = 'mysqldump'
+        dump_cmd[-1] = source_database_name
+        load_cmd = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict)
+        load_cmd[-1] = target_database_name
+
+        dump_proc = subprocess.Popen(dump_cmd, stdout=subprocess.PIPE)
+        load_proc = subprocess.Popen(load_cmd, stdin=dump_proc.stdout, stdout=subprocess.PIPE)
+        dump_proc.stdout.close()    # allow dump_proc to receive a SIGPIPE if load_proc exits.
+        load_proc.communicate()

django/db/backends/mysql/features.py

@@ -31,6 +31,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     can_release_savepoints = True
     atomic_transactions = False
     supports_column_check_constraints = False
+    can_clone_databases = True
 
     @cached_property
     def _mysql_storage_engine(self):

django/db/backends/postgresql/creation.py

@@ -1,3 +1,5 @@
+import sys
+
 from django.db.backends.base.creation import BaseDatabaseCreation
 
 
@@ -11,3 +13,29 @@ class DatabaseCreation(BaseDatabaseCreation):
         if test_settings['CHARSET']:
             return "WITH ENCODING '%s'" % test_settings['CHARSET']
         return ''
+
+    def _clone_test_db(self, number, verbosity, keepdb=False):
+        # CREATE DATABASE ... WITH TEMPLATE ... requires closing connections
+        # to the template database.
+        self.connection.close()
+
+        qn = self.connection.ops.quote_name
+        source_database_name = self.connection.settings_dict['NAME']
+        target_database_name = self.get_test_db_clone_settings(number)['NAME']
+
+        with self._nodb_connection.cursor() as cursor:
+            try:
+                cursor.execute("CREATE DATABASE %s WITH TEMPLATE %s" % (
+                    qn(target_database_name), qn(source_database_name)))
+            except Exception as e:
+                if keepdb:
+                    return
+                try:
+                    if verbosity >= 1:
+                        print("Destroying old test database '%s'..." % self.connection.alias)
+                    cursor.execute("DROP DATABASE %s" % qn(target_database_name))
+                    cursor.execute("CREATE DATABASE %s WITH TEMPLATE %s" % (
+                        qn(target_database_name), qn(source_database_name)))
+                except Exception as e:
+                    sys.stderr.write("Got an error cloning the test database: %s\n" % e)
+                    sys.exit(2)

django/db/backends/postgresql/features.py

@@ -28,3 +28,4 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     has_case_insensitive_like = False
     requires_sqlparse_for_splitting = False
     greatest_least_ignores_nulls = True
+    can_clone_databases = True

django/db/backends/sqlite3/creation.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import sys
 
 from django.core.exceptions import ImproperlyConfigured
@@ -47,6 +48,39 @@ class DatabaseCreation(BaseDatabaseCreation):
                     sys.exit(1)
         return test_database_name
 
+    def get_test_db_clone_settings(self, number):
+        orig_settings_dict = self.connection.settings_dict
+        source_database_name = orig_settings_dict['NAME']
+        if self.connection.is_in_memory_db(source_database_name):
+            return orig_settings_dict
+        else:
+            new_settings_dict = orig_settings_dict.copy()
+            root, ext = os.path.splitext(orig_settings_dict['NAME'])
+            new_settings_dict['NAME'] = '{}_{}.{}'.format(root, number, ext)
+            return new_settings_dict
+
+    def _clone_test_db(self, number, verbosity, keepdb=False):
+        source_database_name = self.connection.settings_dict['NAME']
+        target_database_name = self.get_test_db_clone_settings(number)['NAME']
+        # Forking automatically makes a copy of an in-memory database.
+        if not self.connection.is_in_memory_db(source_database_name):
+            # Erase the old test database
+            if os.access(target_database_name, os.F_OK):
+                if keepdb:
+                    return
+                if verbosity >= 1:
+                    print("Destroying old test database '%s'..." % target_database_name)
+                try:
+                    os.remove(target_database_name)
+                except Exception as e:
+                    sys.stderr.write("Got an error deleting the old test database: %s\n" % e)
+                    sys.exit(2)
+            try:
+                shutil.copy(source_database_name, target_database_name)
+            except Exception as e:
+                sys.stderr.write("Got an error cloning the test database: %s\n" % e)
+                sys.exit(2)
+
     def _destroy_test_db(self, test_database_name, verbosity):
         if test_database_name and not self.connection.is_in_memory_db(test_database_name):
             # Remove the SQLite database file

django/db/backends/sqlite3/features.py

@@ -37,6 +37,7 @@ class DatabaseFeatures(BaseDatabaseFeatures):
     can_rollback_ddl = True
     supports_paramstyle_pyformat = False
     supports_sequence_reset = False
+    can_clone_databases = True
 
     @cached_property
     def uses_savepoints(self):

django/test/runner.py

@@ -1,9 +1,13 @@
 import collections
+import ctypes
+import itertools
 import logging
+import multiprocessing
 import os
+import pickle
+import textwrap
 import unittest
 from importlib import import_module
-from unittest import TestSuite, defaultTestLoader
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
@@ -13,6 +17,11 @@ from django.test.utils import setup_test_environment, teardown_test_environment
 from django.utils.datastructures import OrderedSet
 from django.utils.six import StringIO
 
+try:
+    import tblib.pickling_support
+except ImportError:
+    tblib = None
+
 
 class DebugSQLTextTestResult(unittest.TextTestResult):
     def __init__(self, stream, descriptions, verbosity):
@@ -54,19 +63,295 @@ class DebugSQLTextTestResult(unittest.TextTestResult):
             self.stream.writeln("%s" % sql_debug)
 
 
+class RemoteTestResult(object):
+    """
+    Record information about which tests have succeeded and which have failed.
+
+    The sole purpose of this class is to record events in the child processes
+    so they can be replayed in the master process. As a consequence it doesn't
+    inherit unittest.TestResult and doesn't attempt to implement all its API.
+
+    The implementation matches the unpythonic coding style of unittest2.
+    """
+
+    def __init__(self):
+        self.events = []
+        self.failfast = False
+        self.shouldStop = False
+        self.testsRun = 0
+
+    @property
+    def test_index(self):
+        return self.testsRun - 1
+
+    def check_pickleable(self, test, err):
+        # Ensure that sys.exc_info() tuples are picklable. This displays a
+        # clear multiprocessing.pool.RemoteTraceback generated in the child
+        # process instead of a multiprocessing.pool.MaybeEncodingError, making
+        # the root cause easier to figure out for users who aren't familiar
+        # with the multiprocessing module. Since we're in a forked process,
+        # our best chance to communicate with them is to print to stdout.
+        try:
+            pickle.dumps(err)
+        except Exception as exc:
+            original_exc_txt = repr(err[1])
+            original_exc_txt = textwrap.fill(original_exc_txt, 75)
+            original_exc_txt = textwrap.indent(original_exc_txt, '    ')
+            pickle_exc_txt = repr(exc)
+            pickle_exc_txt = textwrap.fill(pickle_exc_txt, 75)
+            pickle_exc_txt = textwrap.indent(pickle_exc_txt, '    ')
+            if tblib is None:
+                print("""
+
+{} failed:
+
+{}
+
+Unfortunately, tracebacks cannot be pickled, making it impossible for the
+parallel test runner to handle this exception cleanly.
+
+In order to see the traceback, you should install tblib:
+
+    pip install tblib
+""".format(test, original_exc_txt))
+            else:
+                print("""
+
+{} failed:
+
+{}
+
+Unfortunately, the exception it raised cannot be pickled, making it impossible
+for the parallel test runner to handle it cleanly.
+
+Here's the error encountered while trying to pickle the exception:
+
+{}
+
+You should re-run this test without the --parallel option to reproduce the
+failure and get a correct traceback.
+""".format(test, original_exc_txt, pickle_exc_txt))
+            raise
+
+    def stop_if_failfast(self):
+        if self.failfast:
+            self.stop()
+
+    def stop(self):
+        self.shouldStop = True
+
+    def startTestRun(self):
+        self.events.append(('startTestRun',))
+
+    def stopTestRun(self):
+        self.events.append(('stopTestRun',))
+
+    def startTest(self, test):
+        self.testsRun += 1
+        self.events.append(('startTest', self.test_index))
+
+    def stopTest(self, test):
+        self.events.append(('stopTest', self.test_index))
+
+    def addError(self, test, err):
+        self.check_pickleable(test, err)
+        self.events.append(('addError', self.test_index, err))
+        self.stop_if_failfast()
+
+    def addFailure(self, test, err):
+        self.check_pickleable(test, err)
+        self.events.append(('addFailure', self.test_index, err))
+        self.stop_if_failfast()
+
+    def addSubTest(self, test, subtest, err):
+        raise NotImplementedError("subtests aren't supported at this time")
+
+    def addSuccess(self, test):
+        self.events.append(('addSuccess', self.test_index))
+
+    def addSkip(self, test, reason):
+        self.events.append(('addSkip', self.test_index, reason))
+
+    def addExpectedFailure(self, test, err):
+        self.check_pickleable(test, err)
+        self.events.append(('addExpectedFailure', self.test_index, err))
+
+    def addUnexpectedSuccess(self, test):
+        self.events.append(('addUnexpectedSuccess', self.test_index))
+        self.stop_if_failfast()
+
+
+class RemoteTestRunner(object):
+    """
+    Run tests and record everything but don't display anything.
+
+    The implementation matches the unpythonic coding style of unittest2.
+    """
+
+    resultclass = RemoteTestResult
+
+    def __init__(self, failfast=False, resultclass=None):
+        self.failfast = failfast
+        if resultclass is not None:
+            self.resultclass = resultclass
+
+    def run(self, test):
+        result = self.resultclass()
+        unittest.registerResult(result)
+        result.failfast = self.failfast
+        test(result)
+        return result
+
+
+def default_test_processes():
+    """
+    Default number of test processes when using the --parallel option.
+    """
+    try:
+        return int(os.environ['DJANGO_TEST_PROCESSES'])
+    except KeyError:
+        return multiprocessing.cpu_count()
+
+
+_worker_id = 0
+
+
+def _init_worker(counter):
+    """
+    Switch to databases dedicated to this worker.
+
+    This helper lives at module-level because of the multiprocessing module's
+    requirements.
+    """
+
+    global _worker_id
+
+    with counter.get_lock():
+        counter.value += 1
+        _worker_id = counter.value
+
+    for alias in connections:
+        connection = connections[alias]
+        settings_dict = connection.creation.get_test_db_clone_settings(_worker_id)
+        # connection.settings_dict must be updated in place for changes to be
+        # reflected in django.db.connections. If the following line assigned
+        # connection.settings_dict = settings_dict, new threads would connect
+        # to the default database instead of the appropriate clone.
+        connection.settings_dict.update(settings_dict)
+        connection.close()
+
+
+def _run_subsuite(args):
+    """
+    Run a suite of tests with a RemoteTestRunner and return a RemoteTestResult.
+
+    This helper lives at module-level and its arguments are wrapped in a tuple
+    because of the multiprocessing module's requirements.
+    """
+    subsuite_index, subsuite, failfast = args
+    runner = RemoteTestRunner(failfast=failfast)
+    result = runner.run(subsuite)
+    return subsuite_index, result.events
+
+
+class ParallelTestSuite(unittest.TestSuite):
+    """
+    Run a series of tests in parallel in several processes.
+
+    While the unittest module's documentation implies that orchestrating the
+    execution of tests is the responsibility of the test runner, in practice,
+    it appears that TestRunner classes are more concerned with formatting and
+    displaying test results.
+
+    Since there are fewer use cases for customizing TestSuite than TestRunner,
+    implementing parallelization at the level of the TestSuite improves
+    interoperability with existing custom test runners. A single instance of a
+    test runner can still collect results from all tests without being aware
+    that they have been run in parallel.
+    """
+
+    # In case someone wants to modify these in a subclass.
+    init_worker = _init_worker
+    run_subsuite = _run_subsuite
+
+    def __init__(self, suite, processes, failfast=False):
+        self.subsuites = partition_suite_by_case(suite)
+        self.processes = processes
+        self.failfast = failfast
+        super(ParallelTestSuite, self).__init__()
+
+    def run(self, result):
+        """
+        Distribute test cases across workers.
+
+        Return an identifier of each test case with its result in order to use
+        imap_unordered to show results as soon as they're available.
+
+        To minimize pickling errors when getting results from workers:
+
+        - pass back numeric indexes in self.subsuites instead of tests
+        - make tracebacks pickleable with tblib, if available
+
+        Even with tblib, errors may still occur for dynamically created
+        exception classes such Model.DoesNotExist which cannot be unpickled.
+        """
+        if tblib is not None:
+            tblib.pickling_support.install()
+
+        counter = multiprocessing.Value(ctypes.c_int, 0)
+        pool = multiprocessing.Pool(
+            processes=self.processes,
+            initializer=self.init_worker.__func__,
+            initargs=[counter])
+        args = [
+            (index, subsuite, self.failfast)
+            for index, subsuite in enumerate(self.subsuites)
+        ]
+        test_results = pool.imap_unordered(self.run_subsuite.__func__, args)
+
+        while True:
+            if result.shouldStop:
+                pool.terminate()
+                break
+
+            try:
+                subsuite_index, events = test_results.next(timeout=0.1)
+            except multiprocessing.TimeoutError:
+                continue
+            except StopIteration:
+                pool.close()
+                break
+
+            tests = list(self.subsuites[subsuite_index])
+            for event in events:
+                event_name = event[0]
+                handler = getattr(result, event_name, None)
+                if handler is None:
+                    continue
+                test = tests[event[1]]
+                args = event[2:]
+                handler(test, *args)
+
+        pool.join()
+
+        return result
+
+
 class DiscoverRunner(object):
     """
     A Django test runner that uses unittest2 test discovery.
     """
 
-    test_suite = TestSuite
+    test_suite = unittest.TestSuite
+    parallel_test_suite = ParallelTestSuite
     test_runner = unittest.TextTestRunner
-    test_loader = defaultTestLoader
+    test_loader = unittest.defaultTestLoader
     reorder_by = (TestCase, SimpleTestCase)
 
     def __init__(self, pattern=None, top_level=None, verbosity=1,
                  interactive=True, failfast=False, keepdb=False,
-                 reverse=False, debug_sql=False, **kwargs):
+                 reverse=False, debug_sql=False, parallel=0,
+                 **kwargs):
 
         self.pattern = pattern
         self.top_level = top_level
@@ -77,6 +362,7 @@ class DiscoverRunner(object):
         self.keepdb = keepdb
         self.reverse = reverse
         self.debug_sql = debug_sql
+        self.parallel = parallel
 
     @classmethod
     def add_arguments(cls, parser):
@@ -95,6 +381,10 @@ class DiscoverRunner(object):
         parser.add_argument('-d', '--debug-sql', action='store_true', dest='debug_sql',
             default=False,
             help='Prints logged SQL queries on failure.')
+        parser.add_argument(
+            '--parallel', dest='parallel', nargs='?', default=1, type=int,
+            const=default_test_processes(),
+            help='Run tests in parallel processes.')
 
     def setup_test_environment(self, **kwargs):
         setup_test_environment()
@@ -160,12 +450,27 @@ class DiscoverRunner(object):
         for test in extra_tests:
             suite.addTest(test)
 
-        return reorder_suite(suite, self.reorder_by, self.reverse)
+        suite = reorder_suite(suite, self.reorder_by, self.reverse)
+
+        if self.parallel > 1:
+            parallel_suite = self.parallel_test_suite(suite, self.parallel, self.failfast)
+
+            # Since tests are distributed across processes on a per-TestCase
+            # basis, there's no need for more processes than TestCases.
+            parallel_units = len(parallel_suite.subsuites)
+            if self.parallel > parallel_units:
+                self.parallel = parallel_units
+
+            # If there's only one TestCase, parallelization isn't needed.
+            if self.parallel > 1:
+                suite = parallel_suite
+
+        return suite
 
     def setup_databases(self, **kwargs):
         return setup_databases(
             self.verbosity, self.interactive, self.keepdb, self.debug_sql,
-            **kwargs
+            self.parallel, **kwargs
         )
 
     def get_resultclass(self):
@@ -185,6 +490,13 @@ class DiscoverRunner(object):
         """
         for connection, old_name, destroy in old_config:
             if destroy:
+                if self.parallel > 1:
+                    for index in range(self.parallel):
+                        connection.creation.destroy_test_db(
+                            number=index + 1,
+                            verbosity=self.verbosity,
+                            keepdb=self.keepdb,
+                        )
                 connection.creation.destroy_test_db(old_name, self.verbosity, self.keepdb)
 
     def teardown_test_environment(self, **kwargs):
@@ -288,14 +600,14 @@ def reorder_suite(suite, classes, reverse=False):
     class_count = len(classes)
     suite_class = type(suite)
     bins = [OrderedSet() for i in range(class_count + 1)]
-    partition_suite(suite, classes, bins, reverse=reverse)
+    partition_suite_by_type(suite, classes, bins, reverse=reverse)
     reordered_suite = suite_class()
     for i in range(class_count + 1):
         reordered_suite.addTests(bins[i])
     return reordered_suite
 
 
-def partition_suite(suite, classes, bins, reverse=False):
+def partition_suite_by_type(suite, classes, bins, reverse=False):
     """
     Partitions a test suite by test type. Also prevents duplicated tests.
 
@@ -311,7 +623,7 @@ def partition_suite(suite, classes, bins, reverse=False):
         suite = reversed(tuple(suite))
     for test in suite:
         if isinstance(test, suite_class):
-            partition_suite(test, classes, bins, reverse=reverse)
+            partition_suite_by_type(test, classes, bins, reverse=reverse)
         else:
             for i in range(len(classes)):
                 if isinstance(test, classes[i]):
@@ -321,6 +633,21 @@ def partition_suite(suite, classes, bins, reverse=False):
                 bins[-1].add(test)
 
 
+def partition_suite_by_case(suite):
+    """
+    Partitions a test suite by test case, preserving the order of tests.
+    """
+    groups = []
+    suite_class = type(suite)
+    for test_type, test_group in itertools.groupby(suite, type):
+        if issubclass(test_type, unittest.TestCase):
+            groups.append(suite_class(test_group))
+        else:
+            for item in test_group:
+                groups.extend(partition_suite_by_case(item))
+    return groups
+
+
 def get_unique_databases():
     """
     Figure out which databases actually need to be created.
@@ -363,7 +690,7 @@ def get_unique_databases():
     return test_databases
 
 
-def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, **kwargs):
+def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, parallel=0, **kwargs):
     """
     Creates the test databases.
     """
@@ -381,11 +708,18 @@ def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, **kwa
             if first_alias is None:
                 first_alias = alias
                 connection.creation.create_test_db(
-                    verbosity,
+                    verbosity=verbosity,
                     autoclobber=not interactive,
                     keepdb=keepdb,
                     serialize=connection.settings_dict.get("TEST", {}).get("SERIALIZE", True),
                 )
+                if parallel > 1:
+                    for index in range(parallel):
+                        connection.creation.clone_test_db(
+                            number=index + 1,
+                            verbosity=verbosity,
+                            keepdb=keepdb,
+                        )
             # Configure all other connections as mirrors of the first one
             else:
                 connections[alias].creation.set_as_test_mirror(

django/test/testcases.py

@@ -20,6 +20,7 @@ from django.apps import apps
 from django.conf import settings
 from django.core import mail
 from django.core.exceptions import ImproperlyConfigured, ValidationError
+from django.core.files import locks
 from django.core.handlers.wsgi import WSGIHandler, get_path_info
 from django.core.management import call_command
 from django.core.management.color import no_style
@@ -1318,7 +1319,7 @@ class LiveServerTestCase(TransactionTestCase):
 
         # Launch the live server's thread
         specified_address = os.environ.get(
-            'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081')
+            'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081-8179')
 
         # The specified ports may be of the form '8000-8010,8080,9200-9300'
         # i.e. a comma-separated list of ports or ranges of ports, so we break
@@ -1379,3 +1380,31 @@ class LiveServerTestCase(TransactionTestCase):
     def tearDownClass(cls):
         cls._tearDownClassInternal()
         super(LiveServerTestCase, cls).tearDownClass()
+
+
+class SerializeMixin(object):
+    """
+    Mixin to enforce serialization of TestCases that share a common resource.
+
+    Define a common 'lockfile' for each set of TestCases to serialize. This
+    file must exist on the filesystem.
+
+    Place it early in the MRO in order to isolate setUpClass / tearDownClass.
+    """
+
+    lockfile = None
+
+    @classmethod
+    def setUpClass(cls):
+        if cls.lockfile is None:
+            raise ValueError(
+                "{}.lockfile isn't set. Set it to a unique value "
+                "in the base class.".format(cls.__name__))
+        cls._lockfile = open(cls.lockfile)
+        locks.lock(cls._lockfile, locks.LOCK_EX)
+        super(SerializeMixin, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        super(SerializeMixin, cls).tearDownClass()
+        cls._lockfile.close()

docs/internals/contributing/writing-code/unit-tests.txt

@@ -284,3 +284,16 @@ combine this with ``--verbosity=2``, all SQL queries will be output::
 .. versionadded:: 1.8
 
     The ``--reverse`` and ``--debug-sql`` options were added.
+
+By default tests are run in parallel with one process per core. You can adjust
+this behavior with the ``--parallel`` option::
+
+    $ ./runtests.py basic --parallel=1
+
+You can also use the ``DJANGO_TEST_PROCESSES`` environment variable for this
+purpose.
+
+.. versionadded:: 1.9
+
+    Support for running tests in parallel and the ``--parallel`` option were
+    added.

docs/ref/django-admin.txt

@@ -1227,7 +1227,11 @@ provided by the :setting:`TEST_RUNNER` setting.
 
 The ``--liveserver`` option can be used to override the default address where
 the live server (used with :class:`~django.test.LiveServerTestCase`) is
-expected to run from. The default value is ``localhost:8081``.
+expected to run from. The default value is ``localhost:8081-8179``.
+
+.. versionchanged:: 1.9
+
+    In earlier versions, the default value was ``localhost:8081``.
 
 .. django-admin-option:: --keepdb
 
@@ -1257,6 +1261,48 @@ The ``--debug-sql`` option can be used to enable :ref:`SQL logging
 <django-db-logger>` for failing tests. If :djadminopt:`--verbosity` is ``2``,
 then queries in passing tests are also output.
 
+.. django-admin-option:: --parallel
+
+.. versionadded:: 1.9
+
+The ``--parallel`` option can be used to run tests in parallel in separate
+processes. Since modern processors have multiple cores, this allows running
+tests significantly faster.
+
+By default ``--parallel`` runs one process per core according to
+:func:`multiprocessing.cpu_count()`. You can adjust the number of processes
+either by providing it as the option's value, e.g. ``--parallel=4``, or by
+setting the ``DJANGO_TEST_PROCESSES`` environment variable.
+
+Django distributes test cases — :class:`unittest.TestCase` subclasses — to
+subprocesses. If there are fewer test cases than configured processes, Django
+will reduce the number of processes accordingly.
+
+Each process gets its own database. You must ensure that different test cases
+don't access the same resources. For instance, test cases that touch the
+filesystem should create a temporary directory for their own use.
+
+This option requires the third-party ``tblib`` package to display tracebacks
+correctly:
+
+.. code-block:: console
+
+    $ pip install tblib
+
+This feature isn't available on Windows. It doesn't work with the Oracle
+database backend either.
+
+.. warning::
+
+    When test parallelization is enabled and a test fails, Django may be
+    unable to display the exception traceback. This can make debugging
+    difficult. If you encounter this problem, run the affected test without
+    parallelization to see the traceback of the failure.
+
+    This is a known limitation. It arises from the need to serialize objects
+    in order to exchange them between processes. See
+    :ref:`python:pickle-picklable` for details.
+
 testserver <fixture fixture ...>
 --------------------------------
 

docs/releases/1.9.txt

@@ -125,6 +125,21 @@ degradation.
 
 .. _YUI's A-grade: https://github.com/yui/yui3/wiki/Graded-Browser-Support
 
+Running tests in parallel
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :djadmin:`test` command now supports a :djadminopt:`--parallel` option to
+run a project's tests in multiple processes in parallel.
+
+Each process gets its own database. You must ensure that different test cases
+don't access the same resources. For instance, test cases that touch the
+filesystem should create a temporary directory for their own use.
+
+This option is enabled by default for Django's own test suite provided:
+
+- the OS supports it (all but Windows)
+- the database backend supports it (all the built-in backends but Oracle)
+
 Minor features
 ~~~~~~~~~~~~~~
 
@@ -675,6 +690,11 @@ Database backend API
   before 1.0, but hasn't been overridden by any core backend in years
   and hasn't been called anywhere in Django's code or tests.
 
+* In order to support test parallelization, you must implement the
+  ``DatabaseCreation._clone_test_db()`` method and set
+  ``DatabaseFeatures.can_clone_databases = True``. You may have to adjust
+  ``DatabaseCreation.get_test_db_clone_settings()``.
+
 Default settings that were tuples are now lists
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1041,6 +1061,9 @@ Miscellaneous
   changed. They used to be ``(old_names, mirrors)`` tuples. Now they're just
   the first item, ``old_names``.
 
+* By default :class:`~django.test.LiveServerTestCase` attempts to find an
+  available port in the 8081-8179 range instead of just trying port 8081.
+
 .. _deprecated-features-1.9:
 
 Features deprecated in 1.9

docs/topics/testing/tools.txt

@@ -812,11 +812,18 @@ This allows the use of automated test clients other than the
 client, to execute a series of functional tests inside a browser and simulate a
 real user's actions.
 
-By default the live server's address is ``'localhost:8081'`` and the full URL
-can be accessed during the tests with ``self.live_server_url``. If you'd like
-to change the default address (in the case, for example, where the 8081 port is
-already taken) then you may pass a different one to the :djadmin:`test` command
-via the :djadminopt:`--liveserver` option, for example:
+By default the live server listens on ``localhost`` and picks the first
+available port in the ``8081-8179`` range. Its full URL can be accessed with
+``self.live_server_url`` during the tests.
+
+.. versionchanged:: 1.9
+
+    In earlier versions, the live server's default address was always
+    ``'localhost:8081'``.
+
+If you'd like to select another address then you may pass a different one to
+the :djadmin:`test` command via the :djadminopt:`--liveserver` option, for
+example:
 
 .. code-block:: console
 

tests/admin_scripts/tests.py

@@ -33,23 +33,38 @@ from django.utils._os import npath, upath
 from django.utils.encoding import force_text
 from django.utils.six import PY3, StringIO
 
-test_dir = os.path.realpath(os.path.join(tempfile.gettempdir(), 'test_project'))
-if not os.path.exists(test_dir):
-    os.mkdir(test_dir)
-    open(os.path.join(test_dir, '__init__.py'), 'w').close()
-
 custom_templates_dir = os.path.join(os.path.dirname(upath(__file__)), 'custom_templates')
+
 SYSTEM_CHECK_MSG = 'System check identified no issues'
 
 
 class AdminScriptTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        super(AdminScriptTestCase, cls).setUpClass()
+        cls.test_dir = os.path.realpath(os.path.join(
+            tempfile.gettempdir(),
+            cls.__name__,
+            'test_project',
+        ))
+        if not os.path.exists(cls.test_dir):
+            os.makedirs(cls.test_dir)
+        with open(os.path.join(cls.test_dir, '__init__.py'), 'w'):
+            pass
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.test_dir)
+        super(AdminScriptTestCase, cls).tearDownClass()
+
     def write_settings(self, filename, apps=None, is_dir=False, sdict=None, extra=None):
         if is_dir:
-            settings_dir = os.path.join(test_dir, filename)
+            settings_dir = os.path.join(self.test_dir, filename)
             os.mkdir(settings_dir)
             settings_file_path = os.path.join(settings_dir, '__init__.py')
         else:
-            settings_file_path = os.path.join(test_dir, filename)
+            settings_file_path = os.path.join(self.test_dir, filename)
 
         with open(settings_file_path, 'w') as settings_file:
             settings_file.write('# -*- coding: utf-8 -*\n')
@@ -78,7 +93,7 @@ class AdminScriptTestCase(unittest.TestCase):
                     settings_file.write("%s = %s\n" % (k, v))
 
     def remove_settings(self, filename, is_dir=False):
-        full_name = os.path.join(test_dir, filename)
+        full_name = os.path.join(self.test_dir, filename)
         if is_dir:
             shutil.rmtree(full_name)
         else:
@@ -96,7 +111,7 @@ class AdminScriptTestCase(unittest.TestCase):
         except OSError:
             pass
         # Also remove a __pycache__ directory, if it exists
-        cache_name = os.path.join(test_dir, '__pycache__')
+        cache_name = os.path.join(self.test_dir, '__pycache__')
         if os.path.isdir(cache_name):
             shutil.rmtree(cache_name)
 
@@ -115,7 +130,7 @@ class AdminScriptTestCase(unittest.TestCase):
         return paths
 
     def run_test(self, script, args, settings_file=None, apps=None):
-        base_dir = os.path.dirname(test_dir)
+        base_dir = os.path.dirname(self.test_dir)
         # The base dir for Django's tests is one level up.
         tests_dir = os.path.dirname(os.path.dirname(upath(__file__)))
         # The base dir for Django is one level above the test dir. We don't use
@@ -145,7 +160,7 @@ class AdminScriptTestCase(unittest.TestCase):
         test_environ[str('PYTHONWARNINGS')] = str('')
 
         # Move to the test directory and run
-        os.chdir(test_dir)
+        os.chdir(self.test_dir)
         out, err = subprocess.Popen([sys.executable, script] + args,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                 env=test_environ, universal_newlines=True).communicate()
@@ -168,7 +183,7 @@ class AdminScriptTestCase(unittest.TestCase):
         conf_dir = os.path.dirname(upath(conf.__file__))
         template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py')
 
-        test_manage_py = os.path.join(test_dir, 'manage.py')
+        test_manage_py = os.path.join(self.test_dir, 'manage.py')
         shutil.copyfile(template_manage_py, test_manage_py)
 
         with open(test_manage_py, 'r') as fp:
@@ -590,7 +605,7 @@ class DjangoAdminSettingsDirectory(AdminScriptTestCase):
     def test_setup_environ(self):
         "directory: startapp creates the correct directory"
         args = ['startapp', 'settings_test']
-        app_path = os.path.join(test_dir, 'settings_test')
+        app_path = os.path.join(self.test_dir, 'settings_test')
         out, err = self.run_django_admin(args, 'test_project.settings')
         self.addCleanup(shutil.rmtree, app_path)
         self.assertNoOutput(err)
@@ -611,7 +626,7 @@ class DjangoAdminSettingsDirectory(AdminScriptTestCase):
         "directory: startapp creates the correct directory with a custom template"
         template_path = os.path.join(custom_templates_dir, 'app_template')
         args = ['startapp', '--template', template_path, 'custom_settings_test']
-        app_path = os.path.join(test_dir, 'custom_settings_test')
+        app_path = os.path.join(self.test_dir, 'custom_settings_test')
         out, err = self.run_django_admin(args, 'test_project.settings')
         self.addCleanup(shutil.rmtree, app_path)
         self.assertNoOutput(err)
@@ -1047,7 +1062,7 @@ class ManageSettingsWithSettingsErrors(AdminScriptTestCase):
         self.remove_settings('settings.py')
 
     def write_settings_with_import_error(self, filename):
-        settings_file_path = os.path.join(test_dir, filename)
+        settings_file_path = os.path.join(self.test_dir, filename)
         with open(settings_file_path, 'w') as settings_file:
             settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
             settings_file.write('# The next line will cause an import error:\nimport foo42bar\n')
@@ -1802,7 +1817,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
     def test_simple_project(self):
         "Make sure the startproject management command creates a project"
         args = ['startproject', 'testproject']
-        testproject_dir = os.path.join(test_dir, 'testproject')
+        testproject_dir = os.path.join(self.test_dir, 'testproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1818,7 +1833,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Make sure the startproject management command validates a project name"
         for bad_name in ('7testproject', '../testproject'):
             args = ['startproject', bad_name]
-            testproject_dir = os.path.join(test_dir, bad_name)
+            testproject_dir = os.path.join(self.test_dir, bad_name)
             self.addCleanup(shutil.rmtree, testproject_dir, True)
 
             out, err = self.run_django_admin(args)
@@ -1829,7 +1844,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
     def test_simple_project_different_directory(self):
         "Make sure the startproject management command creates a project in a specific directory"
         args = ['startproject', 'testproject', 'othertestproject']
-        testproject_dir = os.path.join(test_dir, 'othertestproject')
+        testproject_dir = os.path.join(self.test_dir, 'othertestproject')
         os.mkdir(testproject_dir)
         self.addCleanup(shutil.rmtree, testproject_dir)
 
@@ -1846,7 +1861,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Make sure the startproject management command is able to use a different project template"
         template_path = os.path.join(custom_templates_dir, 'project_template')
         args = ['startproject', '--template', template_path, 'customtestproject']
-        testproject_dir = os.path.join(test_dir, 'customtestproject')
+        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1858,7 +1873,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Ticket 17475: Template dir passed has a trailing path separator"
         template_path = os.path.join(custom_templates_dir, 'project_template' + os.sep)
         args = ['startproject', '--template', template_path, 'customtestproject']
-        testproject_dir = os.path.join(test_dir, 'customtestproject')
+        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1870,7 +1885,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Make sure the startproject management command is able to use a different project template from a tarball"
         template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
         args = ['startproject', '--template', template_path, 'tarballtestproject']
-        testproject_dir = os.path.join(test_dir, 'tarballtestproject')
+        testproject_dir = os.path.join(self.test_dir, 'tarballtestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1882,7 +1897,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Startproject can use a project template from a tarball and create it in a specified location"
         template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
         args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation']
-        testproject_dir = os.path.join(test_dir, 'altlocation')
+        testproject_dir = os.path.join(self.test_dir, 'altlocation')
         os.mkdir(testproject_dir)
         self.addCleanup(shutil.rmtree, testproject_dir)
 
@@ -1896,7 +1911,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         template_url = '%s/custom_templates/project_template.tgz' % self.live_server_url
 
         args = ['startproject', '--template', template_url, 'urltestproject']
-        testproject_dir = os.path.join(test_dir, 'urltestproject')
+        testproject_dir = os.path.join(self.test_dir, 'urltestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1909,7 +1924,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         template_url = '%s/custom_templates/project_template.tgz/' % self.live_server_url
 
         args = ['startproject', '--template', template_url, 'urltestproject']
-        testproject_dir = os.path.join(test_dir, 'urltestproject')
+        testproject_dir = os.path.join(self.test_dir, 'urltestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1921,7 +1936,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Make sure the startproject management command is able to render custom files"
         template_path = os.path.join(custom_templates_dir, 'project_template')
         args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile']
-        testproject_dir = os.path.join(test_dir, 'customtestproject')
+        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)
@@ -1939,7 +1954,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Make sure template context variables are rendered with proper values"
         template_path = os.path.join(custom_templates_dir, 'project_template')
         args = ['startproject', '--template', template_path, 'another_project', 'project_dir']
-        testproject_dir = os.path.join(test_dir, 'project_dir')
+        testproject_dir = os.path.join(self.test_dir, 'project_dir')
         os.mkdir(testproject_dir)
         self.addCleanup(shutil.rmtree, testproject_dir)
         out, err = self.run_django_admin(args)
@@ -1957,7 +1972,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         self.addCleanup(self.remove_settings, 'alternate_settings.py')
         template_path = os.path.join(custom_templates_dir, 'project_template')
         args = ['custom_startproject', '--template', template_path, 'another_project', 'project_dir', '--extra', '<&>', '--settings=alternate_settings']
-        testproject_dir = os.path.join(test_dir, 'project_dir')
+        testproject_dir = os.path.join(self.test_dir, 'project_dir')
         os.mkdir(testproject_dir)
         self.addCleanup(shutil.rmtree, testproject_dir)
         out, err = self.run_manage(args)
@@ -1974,7 +1989,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         """
         template_path = os.path.join(custom_templates_dir, 'project_template')
         args = ['startproject', '--template', template_path, 'yet_another_project', 'project_dir2']
-        testproject_dir = os.path.join(test_dir, 'project_dir2')
+        testproject_dir = os.path.join(self.test_dir, 'project_dir2')
         out, err = self.run_django_admin(args)
         self.assertNoOutput(out)
         self.assertOutput(err, "Destination directory '%s' does not exist, please create it first." % testproject_dir)
@@ -1984,7 +1999,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase):
         "Ticket 18091: Make sure the startproject management command is able to render templates with non-ASCII content"
         template_path = os.path.join(custom_templates_dir, 'project_template')
         args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject']
-        testproject_dir = os.path.join(test_dir, 'customtestproject')
+        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
         self.addCleanup(shutil.rmtree, testproject_dir, True)
 
         out, err = self.run_django_admin(args)

tests/backends/tests.py

@@ -2,7 +2,6 @@
 # Unit and doctests for specific database backends.
 from __future__ import unicode_literals
 
-import copy
 import datetime
 import re
 import threading
@@ -10,7 +9,6 @@ import unittest
 import warnings
 from decimal import Decimal, Rounded
 
-from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management.color import no_style
 from django.db import (
@@ -182,8 +180,8 @@ class PostgreSQLTests(TestCase):
                 nodb_conn = connection._nodb_connection
         del connection._nodb_connection
         self.assertIsNotNone(nodb_conn.settings_dict['NAME'])
-        self.assertEqual(nodb_conn.settings_dict['NAME'], settings.DATABASES[DEFAULT_DB_ALIAS]['NAME'])
-        # Check a RuntimeWarning nas been emitted
+        self.assertEqual(nodb_conn.settings_dict['NAME'], connection.settings_dict['NAME'])
+        # Check a RuntimeWarning has been emitted
         self.assertEqual(len(w), 1)
         self.assertEqual(w[0].message.__class__, RuntimeWarning)
 
@@ -219,9 +217,7 @@ class PostgreSQLTests(TestCase):
         PostgreSQL shouldn't roll back SET TIME ZONE, even if the first
         transaction is rolled back (#17062).
         """
-        databases = copy.deepcopy(settings.DATABASES)
-        new_connections = ConnectionHandler(databases)
-        new_connection = new_connections[DEFAULT_DB_ALIAS]
+        new_connection = connection.copy()
 
         try:
             # Ensure the database default time zone is different than
@@ -258,10 +254,9 @@ class PostgreSQLTests(TestCase):
         The connection wrapper shouldn't believe that autocommit is enabled
         after setting the time zone when AUTOCOMMIT is False (#21452).
         """
-        databases = copy.deepcopy(settings.DATABASES)
-        databases[DEFAULT_DB_ALIAS]['AUTOCOMMIT'] = False
-        new_connections = ConnectionHandler(databases)
-        new_connection = new_connections[DEFAULT_DB_ALIAS]
+        new_connection = connection.copy()
+        new_connection.settings_dict['AUTOCOMMIT'] = False
+
         try:
             # Open a database connection.
             new_connection.cursor()
@@ -285,10 +280,8 @@ class PostgreSQLTests(TestCase):
         # Check the level on the psycopg2 connection, not the Django wrapper.
         self.assertEqual(connection.connection.isolation_level, read_committed)
 
-        databases = copy.deepcopy(settings.DATABASES)
-        databases[DEFAULT_DB_ALIAS]['OPTIONS']['isolation_level'] = serializable
-        new_connections = ConnectionHandler(databases)
-        new_connection = new_connections[DEFAULT_DB_ALIAS]
+        new_connection = connection.copy()
+        new_connection.settings_dict['OPTIONS']['isolation_level'] = serializable
         try:
             # Start a transaction so the isolation level isn't reported as 0.
             new_connection.set_autocommit(False)
@@ -748,8 +741,7 @@ class BackendTestCase(TransactionTestCase):
         """
         old_queries_limit = BaseDatabaseWrapper.queries_limit
         BaseDatabaseWrapper.queries_limit = 3
-        new_connections = ConnectionHandler(settings.DATABASES)
-        new_connection = new_connections[DEFAULT_DB_ALIAS]
+        new_connection = connection.copy()
 
         # Initialize the connection and clear initialization statements.
         with new_connection.cursor():

tests/delete_regress/tests.py

@@ -2,9 +2,7 @@ from __future__ import unicode_literals
 
 import datetime
 
-from django.conf import settings
-from django.db import DEFAULT_DB_ALIAS, models, transaction
-from django.db.utils import ConnectionHandler
+from django.db import connection, models, transaction
 from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature
 
 from .models import (
@@ -24,8 +22,7 @@ class DeleteLockingTest(TransactionTestCase):
 
     def setUp(self):
         # Create a second connection to the default database
-        new_connections = ConnectionHandler(settings.DATABASES)
-        self.conn2 = new_connections[DEFAULT_DB_ALIAS]
+        self.conn2 = connection.copy()
         self.conn2.set_autocommit(False)
 
     def tearDown(self):

tests/i18n/test_extraction.py

@@ -17,6 +17,7 @@ from django.core.management.commands.makemessages import \
     Command as MakeMessagesCommand
 from django.core.management.utils import find_command
 from django.test import SimpleTestCase, mock, override_settings
+from django.test.testcases import SerializeMixin
 from django.test.utils import captured_stderr, captured_stdout
 from django.utils import six
 from django.utils._os import upath
@@ -30,7 +31,13 @@ this_directory = os.path.dirname(upath(__file__))
 
 
 @skipUnless(has_xgettext, 'xgettext is mandatory for extraction tests')
-class ExtractorTests(SimpleTestCase):
+class ExtractorTests(SerializeMixin, SimpleTestCase):
+
+    # makemessages scans the current working directory and writes in the
+    # locale subdirectory. There aren't any options to control this. As a
+    # consequence tests can't run in parallel. Since i18n tests run in less
+    # than 4 seconds, serializing them with SerializeMixin is acceptable.
+    lockfile = __file__
 
     test_dir = os.path.abspath(os.path.join(this_directory, 'commands'))
 
@@ -610,9 +617,6 @@ class KeepPotFileExtractorTests(ExtractorTests):
 
     POT_FILE = 'locale/django.pot'
 
-    def setUp(self):
-        super(KeepPotFileExtractorTests, self).setUp()
-
     def tearDown(self):
         super(KeepPotFileExtractorTests, self).tearDown()
         os.chdir(self.test_dir)
@@ -646,6 +650,7 @@ class MultipleLocaleExtractionTests(ExtractorTests):
     LOCALES = ['pt', 'de', 'ch']
 
     def tearDown(self):
+        super(MultipleLocaleExtractionTests, self).tearDown()
         os.chdir(self.test_dir)
         for locale in self.LOCALES:
             try:
@@ -677,7 +682,6 @@ class ExcludedLocaleExtractionTests(ExtractorTests):
 
     def setUp(self):
         super(ExcludedLocaleExtractionTests, self).setUp()
-
         os.chdir(self.test_dir)  # ExtractorTests.tearDown() takes care of restoring.
         shutil.copytree('canned_locale', 'locale')
         self._set_times_for_all_po_files()
@@ -719,7 +723,7 @@ class ExcludedLocaleExtractionTests(ExtractorTests):
 class CustomLayoutExtractionTests(ExtractorTests):
 
     def setUp(self):
-        self._cwd = os.getcwd()
+        super(CustomLayoutExtractionTests, self).setUp()
         self.test_dir = os.path.join(this_directory, 'project_dir')
 
     def test_no_locale_raises(self):

tests/model_fields/test_imagefield.py

@@ -8,6 +8,7 @@ from django.core.exceptions import ImproperlyConfigured
 from django.core.files import File
 from django.core.files.images import ImageFile
 from django.test import TestCase
+from django.test.testcases import SerializeMixin
 from django.utils._os import upath
 
 try:
@@ -27,11 +28,13 @@ else:
     PersonTwoImages = Person
 
 
-class ImageFieldTestMixin(object):
+class ImageFieldTestMixin(SerializeMixin):
     """
     Mixin class to provide common functionality to ImageField test classes.
     """
 
+    lockfile = __file__
+
     # Person model to use for tests.
     PersonModel = PersonWithHeightAndWidth
     # File class to use for file instances.

tests/requirements/base.txt

@@ -8,3 +8,4 @@ PyYAML
 pytz > dev
 selenium
 sqlparse
+tblib

tests/runtests.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import atexit
 import logging
 import os
 import shutil
@@ -11,8 +12,9 @@ from argparse import ArgumentParser
 import django
 from django.apps import apps
 from django.conf import settings
-from django.db import connection
+from django.db import connection, connections
 from django.test import TestCase, TransactionTestCase
+from django.test.runner import default_test_processes
 from django.test.utils import get_runner
 from django.utils import six
 from django.utils._os import upath
@@ -34,6 +36,12 @@ TMPDIR = tempfile.mkdtemp(prefix='django_')
 # so that children processes inherit it.
 tempfile.tempdir = os.environ['TMPDIR'] = TMPDIR
 
+# Removing the temporary TMPDIR. Ensure we pass in unicode so that it will
+# successfully remove temp trees containing non-ASCII filenames on Windows.
+# (We're assuming the temp dir name itself only contains ASCII characters.)
+atexit.register(shutil.rmtree, six.text_type(TMPDIR))
+
+
 SUBDIRS_TO_SKIP = [
     'data',
     'import_error_package',
@@ -93,9 +101,12 @@ def get_installed():
     return [app_config.name for app_config in apps.get_app_configs()]
 
 
-def setup(verbosity, test_labels):
+def setup(verbosity, test_labels, parallel):
     if verbosity >= 1:
-        print("Testing against Django installed in '%s'" % os.path.dirname(django.__file__))
+        msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__)
+        if parallel > 1:
+            msg += " with %d processes" % parallel
+        print(msg)
 
     # Force declaring available_apps in TransactionTestCase for faster tests.
     def no_available_apps(self):
@@ -218,22 +229,28 @@ def setup(verbosity, test_labels):
 
 
 def teardown(state):
-    try:
-        # Removing the temporary TMPDIR. Ensure we pass in unicode
-        # so that it will successfully remove temp trees containing
-        # non-ASCII filenames on Windows. (We're assuming the temp dir
-        # name itself does not contain non-ASCII characters.)
-        shutil.rmtree(six.text_type(TMPDIR))
-    except OSError:
-        print('Failed to remove temp directory: %s' % TMPDIR)
-
     # Restore the old settings.
     for key, value in state.items():
         setattr(settings, key, value)
 
 
-def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels, debug_sql):
-    state = setup(verbosity, test_labels)
+def actual_test_processes(parallel):
+    if parallel == 0:
+        # On Python 3.4+: if multiprocessing.get_start_method() != 'fork':
+        if not hasattr(os, 'fork'):
+            return 1
+        # This doesn't work before django.setup() on some databases.
+        elif all(conn.features.can_clone_databases for conn in connections.all()):
+            return default_test_processes()
+        else:
+            return 1
+    else:
+        return parallel
+
+
+def django_tests(verbosity, interactive, failfast, keepdb, reverse,
+                 test_labels, debug_sql, parallel):
+    state = setup(verbosity, test_labels, parallel)
     extra_tests = []
 
     # Run the test suite, including the extra validation tests.
@@ -248,6 +265,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels,
         keepdb=keepdb,
         reverse=reverse,
         debug_sql=debug_sql,
+        parallel=actual_test_processes(parallel),
     )
     failures = test_runner.run_tests(
         test_labels or get_installed(),
@@ -386,13 +404,18 @@ if __name__ == "__main__":
     parser.add_argument('--liveserver',
         help='Overrides the default address where the live server (used with '
              'LiveServerTestCase) is expected to run from. The default value '
-             'is localhost:8081.')
+             'is localhost:8081-8179.')
     parser.add_argument(
         '--selenium', action='store_true', dest='selenium', default=False,
-        help='Run the Selenium tests as well (if Selenium is installed)')
+        help='Run the Selenium tests as well (if Selenium is installed).')
     parser.add_argument(
         '--debug-sql', action='store_true', dest='debug_sql', default=False,
-        help='Turn on the SQL query logger within tests')
+        help='Turn on the SQL query logger within tests.')
+    parser.add_argument(
+        '--parallel', dest='parallel', nargs='?', default=0, type=int,
+        const=default_test_processes(),
+        help='Run tests in parallel processes.')
+
     options = parser.parse_args()
 
     # mock is a required dependency
@@ -429,6 +452,6 @@ if __name__ == "__main__":
         failures = django_tests(options.verbosity, options.interactive,
                                 options.failfast, options.keepdb,
                                 options.reverse, options.modules,
-                                options.debug_sql)
+                                options.debug_sql, options.parallel)
         if failures:
             sys.exit(bool(failures))

tests/select_for_update/tests.py

@@ -5,9 +5,7 @@ import time
 
 from multiple_database.routers import TestRouter
 
-from django.conf import settings
-from django.db import connection, router, transaction
-from django.db.utils import DEFAULT_DB_ALIAS, ConnectionHandler, DatabaseError
+from django.db import DatabaseError, connection, router, transaction
 from django.test import (
     TransactionTestCase, override_settings, skipIfDBFeature,
     skipUnlessDBFeature,
@@ -30,8 +28,7 @@ class SelectForUpdateTests(TransactionTestCase):
 
         # We need another database connection in transaction to test that one
         # connection issuing a SELECT ... FOR UPDATE will block.
-        new_connections = ConnectionHandler(settings.DATABASES)
-        self.new_connection = new_connections[DEFAULT_DB_ALIAS]
+        self.new_connection = connection.copy()
 
     def tearDown(self):
         try:

tests/staticfiles_tests/settings.py

@@ -6,8 +6,6 @@ from django.utils._os import upath
 
 TEST_ROOT = os.path.dirname(upath(__file__))
 
-TESTFILES_PATH = os.path.join(TEST_ROOT, 'apps', 'test', 'static', 'test')
-
 TEST_SETTINGS = {
     'DEBUG': True,
     'MEDIA_URL': '/media/',

tests/staticfiles_tests/test_management.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import codecs
 import os
 import shutil
+import tempfile
 import unittest
 
 from django.conf import settings
@@ -11,6 +12,7 @@ from django.contrib.staticfiles.management.commands import collectstatic
 from django.core.exceptions import ImproperlyConfigured
 from django.core.management import call_command
 from django.test import override_settings
+from django.test.utils import extend_sys_path
 from django.utils import six
 from django.utils._os import symlinks_supported
 from django.utils.encoding import force_text
@@ -197,31 +199,45 @@ class TestCollectionFilesOverride(CollectionTestCase):
     Test overriding duplicated files by ``collectstatic`` management command.
     Check for proper handling of apps order in installed apps even if file modification
     dates are in different order:
-        'staticfiles_tests.apps.test',
+        'staticfiles_test_app',
         'staticfiles_tests.apps.no_label',
     """
     def setUp(self):
-        self.orig_path = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'file2.txt')
+        self.temp_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.temp_dir)
+
         # get modification and access times for no_label/static/file2.txt
+        self.orig_path = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'file2.txt')
         self.orig_mtime = os.path.getmtime(self.orig_path)
         self.orig_atime = os.path.getatime(self.orig_path)
 
-        # prepare duplicate of file2.txt from no_label app
+        # prepare duplicate of file2.txt from a temporary app
         # this file will have modification time older than no_label/static/file2.txt
-        # anyway it should be taken to STATIC_ROOT because 'test' app is before
+        # anyway it should be taken to STATIC_ROOT because the temporary app is before
         # 'no_label' app in installed apps
-        self.testfile_path = os.path.join(TEST_ROOT, 'apps', 'test', 'static', 'file2.txt')
+        self.temp_app_path = os.path.join(self.temp_dir, 'staticfiles_test_app')
+        self.testfile_path = os.path.join(self.temp_app_path, 'static', 'file2.txt')
+
+        os.makedirs(self.temp_app_path)
+        with open(os.path.join(self.temp_app_path, '__init__.py'), 'w+'):
+            pass
+
+        os.makedirs(os.path.dirname(self.testfile_path))
         with open(self.testfile_path, 'w+') as f:
             f.write('duplicate of file2.txt')
+
         os.utime(self.testfile_path, (self.orig_atime - 1, self.orig_mtime - 1))
+
+        self.settings_with_test_app = self.modify_settings(
+            INSTALLED_APPS={'prepend': 'staticfiles_test_app'})
+        with extend_sys_path(self.temp_dir):
+            self.settings_with_test_app.enable()
+
         super(TestCollectionFilesOverride, self).setUp()
 
     def tearDown(self):
-        if os.path.exists(self.testfile_path):
-            os.unlink(self.testfile_path)
-        # set back original modification time
-        os.utime(self.orig_path, (self.orig_atime, self.orig_mtime))
         super(TestCollectionFilesOverride, self).tearDown()
+        self.settings_with_test_app.disable()
 
     def test_ordering_override(self):
         """
@@ -234,22 +250,11 @@ class TestCollectionFilesOverride(CollectionTestCase):
 
         self.assertFileContains('file2.txt', 'duplicate of file2.txt')
 
-        # and now change modification time of no_label/static/file2.txt
-        # test app is first in installed apps so file2.txt should remain unmodified
-        mtime = os.path.getmtime(self.testfile_path)
-        atime = os.path.getatime(self.testfile_path)
-        os.utime(self.orig_path, (mtime + 1, atime + 1))
-
-        # run collectstatic again
-        self.run_collectstatic()
-
-        self.assertFileContains('file2.txt', 'duplicate of file2.txt')
-
 
 # The collectstatic test suite already has conflicting files since both
 # project/test/file.txt and apps/test/static/test/file.txt are collected. To
 # properly test for the warning not happening unless we tell it to explicitly,
-# we only include static files from the default finders.
+# we remove the project directory and will add back a conflicting file later.
 @override_settings(STATICFILES_DIRS=[])
 class TestCollectionOverwriteWarning(CollectionTestCase):
     """
@@ -268,41 +273,37 @@ class TestCollectionOverwriteWarning(CollectionTestCase):
         """
         out = six.StringIO()
         call_command('collectstatic', interactive=False, verbosity=3, stdout=out, **kwargs)
-        out.seek(0)
-        return out.read()
+        return force_text(out.getvalue())
 
     def test_no_warning(self):
         """
         There isn't a warning if there isn't a duplicate destination.
         """
         output = self._collectstatic_output(clear=True)
-        self.assertNotIn(self.warning_string, force_text(output))
+        self.assertNotIn(self.warning_string, output)
 
     def test_warning(self):
         """
         There is a warning when there are duplicate destinations.
         """
-        # Create new file in the no_label app that also exists in the test app.
-        test_dir = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'test')
-        if not os.path.exists(test_dir):
-            os.mkdir(test_dir)
+        static_dir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, static_dir)
 
-        try:
-            duplicate_path = os.path.join(test_dir, 'file.txt')
-            with open(duplicate_path, 'w+') as f:
-                f.write('duplicate of file.txt')
+        duplicate = os.path.join(static_dir, 'test', 'file.txt')
+        os.mkdir(os.path.dirname(duplicate))
+        with open(duplicate, 'w+') as f:
+            f.write('duplicate of file.txt')
+
+        with self.settings(STATICFILES_DIRS=[static_dir]):
             output = self._collectstatic_output(clear=True)
-            self.assertIn(self.warning_string, force_text(output))
-        finally:
-            if os.path.exists(duplicate_path):
-                os.unlink(duplicate_path)
+        self.assertIn(self.warning_string, output)
 
-        if os.path.exists(test_dir):
-            os.rmdir(test_dir)
+        os.remove(duplicate)
 
         # Make sure the warning went away again.
-        output = self._collectstatic_output(clear=True)
-        self.assertNotIn(self.warning_string, force_text(output))
+        with self.settings(STATICFILES_DIRS=[static_dir]):
+            output = self._collectstatic_output(clear=True)
+        self.assertNotIn(self.warning_string, output)
 
 
 @override_settings(STATICFILES_STORAGE='staticfiles_tests.storage.DummyStorage')

tests/staticfiles_tests/test_storage.py

@@ -1,7 +1,9 @@
 from __future__ import unicode_literals
 
 import os
+import shutil
 import sys
+import tempfile
 import unittest
 
 from django.conf import settings
@@ -18,7 +20,7 @@ from django.utils.encoding import force_text
 from .cases import (
     BaseCollectionTestCase, BaseStaticFilesTestCase, StaticFilesTestCase,
 )
-from .settings import TEST_ROOT, TEST_SETTINGS, TESTFILES_PATH
+from .settings import TEST_ROOT, TEST_SETTINGS
 
 
 def hashed_file_path(test, path):
@@ -252,15 +254,25 @@ class TestCollectionManifestStorage(TestHashedFiles, BaseCollectionTestCase,
     def setUp(self):
         super(TestCollectionManifestStorage, self).setUp()
 
-        self._clear_filename = os.path.join(TESTFILES_PATH, 'cleared.txt')
+        temp_dir = tempfile.mkdtemp()
+        os.makedirs(os.path.join(temp_dir, 'test'))
+        self._clear_filename = os.path.join(temp_dir, 'test', 'cleared.txt')
         with open(self._clear_filename, 'w') as f:
             f.write('to be deleted in one test')
 
+        self.patched_settings = self.settings(
+            STATICFILES_DIRS=settings.STATICFILES_DIRS + [temp_dir])
+        self.patched_settings.enable()
+        self.addCleanup(shutil.rmtree, six.text_type(temp_dir))
+
     def tearDown(self):
-        super(TestCollectionManifestStorage, self).tearDown()
+        self.patched_settings.disable()
+
         if os.path.exists(self._clear_filename):
             os.unlink(self._clear_filename)
 
+        super(TestCollectionManifestStorage, self).tearDown()
+
     def test_manifest_exists(self):
         filename = storage.staticfiles_storage.manifest_name
         path = storage.staticfiles_storage.path(filename)

tests/test_runner/tests.py

@@ -319,7 +319,7 @@ class SetupDatabasesTests(unittest.TestCase):
             with mock.patch('django.test.runner.connections', new=tested_connections):
                 self.runner_instance.setup_databases()
         mocked_db_creation.return_value.create_test_db.assert_called_once_with(
-            0, autoclobber=False, serialize=True, keepdb=False
+            verbosity=0, autoclobber=False, serialize=True, keepdb=False
         )
 
     def test_serialized_off(self):
@@ -333,7 +333,7 @@ class SetupDatabasesTests(unittest.TestCase):
             with mock.patch('django.test.runner.connections', new=tested_connections):
                 self.runner_instance.setup_databases()
         mocked_db_creation.return_value.create_test_db.assert_called_once_with(
-            0, autoclobber=False, serialize=False, keepdb=False
+            verbosity=0, autoclobber=False, serialize=False, keepdb=False
         )
 
 

tests/timezones/tests.py

@@ -4,6 +4,7 @@ import datetime
 import re
 import sys
 import warnings
+from contextlib import contextmanager
 from unittest import SkipTest, skipIf
 from xml.dom.minidom import parseString
 
@@ -611,27 +612,41 @@ class ForcedTimeZoneDatabaseTests(TransactionTestCase):
             raise SkipTest("Database doesn't support feature(s): test_db_allows_multiple_connections")
 
         super(ForcedTimeZoneDatabaseTests, cls).setUpClass()
-        connections.databases['tz'] = connections.databases['default'].copy()
-        connections.databases['tz']['TIME_ZONE'] = 'Asia/Bangkok'
 
-    @classmethod
-    def tearDownClass(cls):
-        connections['tz'].close()
-        del connections['tz']
-        del connections.databases['tz']
-        super(ForcedTimeZoneDatabaseTests, cls).tearDownClass()
+    @contextmanager
+    def override_database_connection_timezone(self, timezone):
+        try:
+            orig_timezone = connection.settings_dict['TIME_ZONE']
+            connection.settings_dict['TIME_ZONE'] = timezone
+            # Clear cached properties, after first accessing them to ensure they exist.
+            connection.timezone
+            del connection.timezone
+            connection.timezone_name
+            del connection.timezone_name
+
+            yield
+
+        finally:
+            connection.settings_dict['TIME_ZONE'] = orig_timezone
+            # Clear cached properties, after first accessing them to ensure they exist.
+            connection.timezone
+            del connection.timezone
+            connection.timezone_name
+            del connection.timezone_name
 
     def test_read_datetime(self):
         fake_dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=UTC)
         Event.objects.create(dt=fake_dt)
 
-        event = Event.objects.using('tz').get()
-        dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
+        with self.override_database_connection_timezone('Asia/Bangkok'):
+            event = Event.objects.get()
+            dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
         self.assertEqual(event.dt, dt)
 
     def test_write_datetime(self):
         dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC)
-        Event.objects.using('tz').create(dt=dt)
+        with self.override_database_connection_timezone('Asia/Bangkok'):
+            Event.objects.create(dt=dt)
 
         event = Event.objects.get()
         fake_dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=UTC)