diff --git a/django/core/management/commands/test.py b/django/core/management/commands/test.py index 0d084e1e91..ad54994d74 100644 --- a/django/core/management/commands/test.py +++ b/django/core/management/commands/test.py @@ -47,7 +47,7 @@ class Command(BaseCommand): action='store', dest='liveserver', default=None, help='Overrides the default address where the live server (used ' 'with LiveServerTestCase) is expected to run from. The ' - 'default value is localhost:8081.'), + 'default value is localhost:8081-8179.'), test_runner_class = get_runner(settings, self.test_runner) if hasattr(test_runner_class, 'option_list'): diff --git a/django/db/backends/base/base.py b/django/db/backends/base/base.py index 96ca0ccfde..2e6a528171 100644 --- a/django/db/backends/base/base.py +++ b/django/db/backends/base/base.py @@ -1,3 +1,4 @@ +import copy import time import warnings from collections import deque @@ -622,3 +623,16 @@ class BaseDatabaseWrapper(object): func() finally: self.run_on_commit = [] + + def copy(self, alias=None, allow_thread_sharing=None): + """ + Return a copy of this connection. + + For tests that require two connections to the same database. + """ + settings_dict = copy.deepcopy(self.settings_dict) + if alias is None: + alias = self.alias + if allow_thread_sharing is None: + allow_thread_sharing = self.allow_thread_sharing + return type(self)(settings_dict, alias, allow_thread_sharing) diff --git a/django/db/backends/base/creation.py b/django/db/backends/base/creation.py index 82ccadcbe7..888dc792fe 100644 --- a/django/db/backends/base/creation.py +++ b/django/db/backends/base/creation.py @@ -190,13 +190,56 @@ class BaseDatabaseCreation(object): return test_database_name - def destroy_test_db(self, old_database_name, verbosity=1, keepdb=False): + def clone_test_db(self, number, verbosity=1, autoclobber=False, keepdb=False): + """ + Clone a test database. + """ + source_database_name = self.connection.settings_dict['NAME'] + + if verbosity >= 1: + test_db_repr = '' + action = 'Cloning test database' + if verbosity >= 2: + test_db_repr = " ('%s')" % source_database_name + if keepdb: + action = 'Using existing clone' + print("%s for alias '%s'%s..." % (action, self.connection.alias, test_db_repr)) + + # We could skip this call if keepdb is True, but we instead + # give it the keepdb param. See create_test_db for details. + self._clone_test_db(number, verbosity, keepdb) + + def get_test_db_clone_settings(self, number): + """ + Return a modified connection settings dict for the n-th clone of a DB. + """ + # When this function is called, the test database has been created + # already and its name has been copied to settings_dict['NAME'] so + # we don't need to call _get_test_db_name. + orig_settings_dict = self.connection.settings_dict + new_settings_dict = orig_settings_dict.copy() + new_settings_dict['NAME'] = '{}_{}'.format(orig_settings_dict['NAME'], number) + return new_settings_dict + + def _clone_test_db(self, number, verbosity, keepdb=False): + """ + Internal implementation - duplicate the test db tables. + """ + raise NotImplementedError( + "The database backend doesn't support cloning databases. " + "Disable the option to run tests in parallel processes.") + + def destroy_test_db(self, old_database_name=None, verbosity=1, keepdb=False, number=None): """ Destroy a test database, prompting the user for confirmation if the database already exists. """ self.connection.close() - test_database_name = self.connection.settings_dict['NAME'] + if number is None: + test_database_name = self.connection.settings_dict['NAME'] + else: + test_database_name = self.get_test_db_clone_settings(number)['NAME'] + if verbosity >= 1: test_db_repr = '' action = 'Destroying' @@ -213,8 +256,9 @@ class BaseDatabaseCreation(object): self._destroy_test_db(test_database_name, verbosity) # Restore the original database name - settings.DATABASES[self.connection.alias]["NAME"] = old_database_name - self.connection.settings_dict["NAME"] = old_database_name + if old_database_name is not None: + settings.DATABASES[self.connection.alias]["NAME"] = old_database_name + self.connection.settings_dict["NAME"] = old_database_name def _destroy_test_db(self, test_database_name, verbosity): """ diff --git a/django/db/backends/base/features.py b/django/db/backends/base/features.py index b7861865ac..095164770f 100644 --- a/django/db/backends/base/features.py +++ b/django/db/backends/base/features.py @@ -212,6 +212,10 @@ class BaseDatabaseFeatures(object): # every expression is null? greatest_least_ignores_nulls = False + # Can the backend clone databases for parallel test execution? + # Defaults to False to allow third-party backends to opt-in. + can_clone_databases = False + def __init__(self, connection): self.connection = connection diff --git a/django/db/backends/mysql/creation.py b/django/db/backends/mysql/creation.py index aba9e8b841..a992e65952 100644 --- a/django/db/backends/mysql/creation.py +++ b/django/db/backends/mysql/creation.py @@ -1,5 +1,10 @@ +import subprocess +import sys + from django.db.backends.base.creation import BaseDatabaseCreation +from .client import DatabaseClient + class DatabaseCreation(BaseDatabaseCreation): @@ -11,3 +16,34 @@ class DatabaseCreation(BaseDatabaseCreation): if test_settings['COLLATION']: suffix.append('COLLATE %s' % test_settings['COLLATION']) return ' '.join(suffix) + + def _clone_test_db(self, number, verbosity, keepdb=False): + qn = self.connection.ops.quote_name + source_database_name = self.connection.settings_dict['NAME'] + target_database_name = self.get_test_db_clone_settings(number)['NAME'] + + with self._nodb_connection.cursor() as cursor: + try: + cursor.execute("CREATE DATABASE %s" % qn(target_database_name)) + except Exception as e: + if keepdb: + return + try: + if verbosity >= 1: + print("Destroying old test database '%s'..." % self.connection.alias) + cursor.execute("DROP DATABASE %s" % qn(target_database_name)) + cursor.execute("CREATE DATABASE %s" % qn(target_database_name)) + except Exception as e: + sys.stderr.write("Got an error recreating the test database: %s\n" % e) + sys.exit(2) + + dump_cmd = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict) + dump_cmd[0] = 'mysqldump' + dump_cmd[-1] = source_database_name + load_cmd = DatabaseClient.settings_to_cmd_args(self.connection.settings_dict) + load_cmd[-1] = target_database_name + + dump_proc = subprocess.Popen(dump_cmd, stdout=subprocess.PIPE) + load_proc = subprocess.Popen(load_cmd, stdin=dump_proc.stdout, stdout=subprocess.PIPE) + dump_proc.stdout.close() # allow dump_proc to receive a SIGPIPE if load_proc exits. + load_proc.communicate() diff --git a/django/db/backends/mysql/features.py b/django/db/backends/mysql/features.py index 8f2e99a791..c8ea8f7fb6 100644 --- a/django/db/backends/mysql/features.py +++ b/django/db/backends/mysql/features.py @@ -31,6 +31,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_release_savepoints = True atomic_transactions = False supports_column_check_constraints = False + can_clone_databases = True @cached_property def _mysql_storage_engine(self): diff --git a/django/db/backends/postgresql/creation.py b/django/db/backends/postgresql/creation.py index 13eea0d3b7..dcb0430f1b 100644 --- a/django/db/backends/postgresql/creation.py +++ b/django/db/backends/postgresql/creation.py @@ -1,3 +1,5 @@ +import sys + from django.db.backends.base.creation import BaseDatabaseCreation @@ -11,3 +13,29 @@ class DatabaseCreation(BaseDatabaseCreation): if test_settings['CHARSET']: return "WITH ENCODING '%s'" % test_settings['CHARSET'] return '' + + def _clone_test_db(self, number, verbosity, keepdb=False): + # CREATE DATABASE ... WITH TEMPLATE ... requires closing connections + # to the template database. + self.connection.close() + + qn = self.connection.ops.quote_name + source_database_name = self.connection.settings_dict['NAME'] + target_database_name = self.get_test_db_clone_settings(number)['NAME'] + + with self._nodb_connection.cursor() as cursor: + try: + cursor.execute("CREATE DATABASE %s WITH TEMPLATE %s" % ( + qn(target_database_name), qn(source_database_name))) + except Exception as e: + if keepdb: + return + try: + if verbosity >= 1: + print("Destroying old test database '%s'..." % self.connection.alias) + cursor.execute("DROP DATABASE %s" % qn(target_database_name)) + cursor.execute("CREATE DATABASE %s WITH TEMPLATE %s" % ( + qn(target_database_name), qn(source_database_name))) + except Exception as e: + sys.stderr.write("Got an error cloning the test database: %s\n" % e) + sys.exit(2) diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index a07700e601..9130a4521e 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -28,3 +28,4 @@ class DatabaseFeatures(BaseDatabaseFeatures): has_case_insensitive_like = False requires_sqlparse_for_splitting = False greatest_least_ignores_nulls = True + can_clone_databases = True diff --git a/django/db/backends/sqlite3/creation.py b/django/db/backends/sqlite3/creation.py index d8fde0c9b7..d52349e57f 100644 --- a/django/db/backends/sqlite3/creation.py +++ b/django/db/backends/sqlite3/creation.py @@ -1,4 +1,5 @@ import os +import shutil import sys from django.core.exceptions import ImproperlyConfigured @@ -47,6 +48,39 @@ class DatabaseCreation(BaseDatabaseCreation): sys.exit(1) return test_database_name + def get_test_db_clone_settings(self, number): + orig_settings_dict = self.connection.settings_dict + source_database_name = orig_settings_dict['NAME'] + if self.connection.is_in_memory_db(source_database_name): + return orig_settings_dict + else: + new_settings_dict = orig_settings_dict.copy() + root, ext = os.path.splitext(orig_settings_dict['NAME']) + new_settings_dict['NAME'] = '{}_{}.{}'.format(root, number, ext) + return new_settings_dict + + def _clone_test_db(self, number, verbosity, keepdb=False): + source_database_name = self.connection.settings_dict['NAME'] + target_database_name = self.get_test_db_clone_settings(number)['NAME'] + # Forking automatically makes a copy of an in-memory database. + if not self.connection.is_in_memory_db(source_database_name): + # Erase the old test database + if os.access(target_database_name, os.F_OK): + if keepdb: + return + if verbosity >= 1: + print("Destroying old test database '%s'..." % target_database_name) + try: + os.remove(target_database_name) + except Exception as e: + sys.stderr.write("Got an error deleting the old test database: %s\n" % e) + sys.exit(2) + try: + shutil.copy(source_database_name, target_database_name) + except Exception as e: + sys.stderr.write("Got an error cloning the test database: %s\n" % e) + sys.exit(2) + def _destroy_test_db(self, test_database_name, verbosity): if test_database_name and not self.connection.is_in_memory_db(test_database_name): # Remove the SQLite database file diff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py index 5697b8b7eb..dffc48afb8 100644 --- a/django/db/backends/sqlite3/features.py +++ b/django/db/backends/sqlite3/features.py @@ -37,6 +37,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_rollback_ddl = True supports_paramstyle_pyformat = False supports_sequence_reset = False + can_clone_databases = True @cached_property def uses_savepoints(self): diff --git a/django/test/runner.py b/django/test/runner.py index 6a8a918984..243ab714a4 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -1,9 +1,13 @@ import collections +import ctypes +import itertools import logging +import multiprocessing import os +import pickle +import textwrap import unittest from importlib import import_module -from unittest import TestSuite, defaultTestLoader from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -13,6 +17,11 @@ from django.test.utils import setup_test_environment, teardown_test_environment from django.utils.datastructures import OrderedSet from django.utils.six import StringIO +try: + import tblib.pickling_support +except ImportError: + tblib = None + class DebugSQLTextTestResult(unittest.TextTestResult): def __init__(self, stream, descriptions, verbosity): @@ -54,19 +63,295 @@ class DebugSQLTextTestResult(unittest.TextTestResult): self.stream.writeln("%s" % sql_debug) +class RemoteTestResult(object): + """ + Record information about which tests have succeeded and which have failed. + + The sole purpose of this class is to record events in the child processes + so they can be replayed in the master process. As a consequence it doesn't + inherit unittest.TestResult and doesn't attempt to implement all its API. + + The implementation matches the unpythonic coding style of unittest2. + """ + + def __init__(self): + self.events = [] + self.failfast = False + self.shouldStop = False + self.testsRun = 0 + + @property + def test_index(self): + return self.testsRun - 1 + + def check_pickleable(self, test, err): + # Ensure that sys.exc_info() tuples are picklable. This displays a + # clear multiprocessing.pool.RemoteTraceback generated in the child + # process instead of a multiprocessing.pool.MaybeEncodingError, making + # the root cause easier to figure out for users who aren't familiar + # with the multiprocessing module. Since we're in a forked process, + # our best chance to communicate with them is to print to stdout. + try: + pickle.dumps(err) + except Exception as exc: + original_exc_txt = repr(err[1]) + original_exc_txt = textwrap.fill(original_exc_txt, 75) + original_exc_txt = textwrap.indent(original_exc_txt, ' ') + pickle_exc_txt = repr(exc) + pickle_exc_txt = textwrap.fill(pickle_exc_txt, 75) + pickle_exc_txt = textwrap.indent(pickle_exc_txt, ' ') + if tblib is None: + print(""" + +{} failed: + +{} + +Unfortunately, tracebacks cannot be pickled, making it impossible for the +parallel test runner to handle this exception cleanly. + +In order to see the traceback, you should install tblib: + + pip install tblib +""".format(test, original_exc_txt)) + else: + print(""" + +{} failed: + +{} + +Unfortunately, the exception it raised cannot be pickled, making it impossible +for the parallel test runner to handle it cleanly. + +Here's the error encountered while trying to pickle the exception: + +{} + +You should re-run this test without the --parallel option to reproduce the +failure and get a correct traceback. +""".format(test, original_exc_txt, pickle_exc_txt)) + raise + + def stop_if_failfast(self): + if self.failfast: + self.stop() + + def stop(self): + self.shouldStop = True + + def startTestRun(self): + self.events.append(('startTestRun',)) + + def stopTestRun(self): + self.events.append(('stopTestRun',)) + + def startTest(self, test): + self.testsRun += 1 + self.events.append(('startTest', self.test_index)) + + def stopTest(self, test): + self.events.append(('stopTest', self.test_index)) + + def addError(self, test, err): + self.check_pickleable(test, err) + self.events.append(('addError', self.test_index, err)) + self.stop_if_failfast() + + def addFailure(self, test, err): + self.check_pickleable(test, err) + self.events.append(('addFailure', self.test_index, err)) + self.stop_if_failfast() + + def addSubTest(self, test, subtest, err): + raise NotImplementedError("subtests aren't supported at this time") + + def addSuccess(self, test): + self.events.append(('addSuccess', self.test_index)) + + def addSkip(self, test, reason): + self.events.append(('addSkip', self.test_index, reason)) + + def addExpectedFailure(self, test, err): + self.check_pickleable(test, err) + self.events.append(('addExpectedFailure', self.test_index, err)) + + def addUnexpectedSuccess(self, test): + self.events.append(('addUnexpectedSuccess', self.test_index)) + self.stop_if_failfast() + + +class RemoteTestRunner(object): + """ + Run tests and record everything but don't display anything. + + The implementation matches the unpythonic coding style of unittest2. + """ + + resultclass = RemoteTestResult + + def __init__(self, failfast=False, resultclass=None): + self.failfast = failfast + if resultclass is not None: + self.resultclass = resultclass + + def run(self, test): + result = self.resultclass() + unittest.registerResult(result) + result.failfast = self.failfast + test(result) + return result + + +def default_test_processes(): + """ + Default number of test processes when using the --parallel option. + """ + try: + return int(os.environ['DJANGO_TEST_PROCESSES']) + except KeyError: + return multiprocessing.cpu_count() + + +_worker_id = 0 + + +def _init_worker(counter): + """ + Switch to databases dedicated to this worker. + + This helper lives at module-level because of the multiprocessing module's + requirements. + """ + + global _worker_id + + with counter.get_lock(): + counter.value += 1 + _worker_id = counter.value + + for alias in connections: + connection = connections[alias] + settings_dict = connection.creation.get_test_db_clone_settings(_worker_id) + # connection.settings_dict must be updated in place for changes to be + # reflected in django.db.connections. If the following line assigned + # connection.settings_dict = settings_dict, new threads would connect + # to the default database instead of the appropriate clone. + connection.settings_dict.update(settings_dict) + connection.close() + + +def _run_subsuite(args): + """ + Run a suite of tests with a RemoteTestRunner and return a RemoteTestResult. + + This helper lives at module-level and its arguments are wrapped in a tuple + because of the multiprocessing module's requirements. + """ + subsuite_index, subsuite, failfast = args + runner = RemoteTestRunner(failfast=failfast) + result = runner.run(subsuite) + return subsuite_index, result.events + + +class ParallelTestSuite(unittest.TestSuite): + """ + Run a series of tests in parallel in several processes. + + While the unittest module's documentation implies that orchestrating the + execution of tests is the responsibility of the test runner, in practice, + it appears that TestRunner classes are more concerned with formatting and + displaying test results. + + Since there are fewer use cases for customizing TestSuite than TestRunner, + implementing parallelization at the level of the TestSuite improves + interoperability with existing custom test runners. A single instance of a + test runner can still collect results from all tests without being aware + that they have been run in parallel. + """ + + # In case someone wants to modify these in a subclass. + init_worker = _init_worker + run_subsuite = _run_subsuite + + def __init__(self, suite, processes, failfast=False): + self.subsuites = partition_suite_by_case(suite) + self.processes = processes + self.failfast = failfast + super(ParallelTestSuite, self).__init__() + + def run(self, result): + """ + Distribute test cases across workers. + + Return an identifier of each test case with its result in order to use + imap_unordered to show results as soon as they're available. + + To minimize pickling errors when getting results from workers: + + - pass back numeric indexes in self.subsuites instead of tests + - make tracebacks pickleable with tblib, if available + + Even with tblib, errors may still occur for dynamically created + exception classes such Model.DoesNotExist which cannot be unpickled. + """ + if tblib is not None: + tblib.pickling_support.install() + + counter = multiprocessing.Value(ctypes.c_int, 0) + pool = multiprocessing.Pool( + processes=self.processes, + initializer=self.init_worker.__func__, + initargs=[counter]) + args = [ + (index, subsuite, self.failfast) + for index, subsuite in enumerate(self.subsuites) + ] + test_results = pool.imap_unordered(self.run_subsuite.__func__, args) + + while True: + if result.shouldStop: + pool.terminate() + break + + try: + subsuite_index, events = test_results.next(timeout=0.1) + except multiprocessing.TimeoutError: + continue + except StopIteration: + pool.close() + break + + tests = list(self.subsuites[subsuite_index]) + for event in events: + event_name = event[0] + handler = getattr(result, event_name, None) + if handler is None: + continue + test = tests[event[1]] + args = event[2:] + handler(test, *args) + + pool.join() + + return result + + class DiscoverRunner(object): """ A Django test runner that uses unittest2 test discovery. """ - test_suite = TestSuite + test_suite = unittest.TestSuite + parallel_test_suite = ParallelTestSuite test_runner = unittest.TextTestRunner - test_loader = defaultTestLoader + test_loader = unittest.defaultTestLoader reorder_by = (TestCase, SimpleTestCase) def __init__(self, pattern=None, top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, - reverse=False, debug_sql=False, **kwargs): + reverse=False, debug_sql=False, parallel=0, + **kwargs): self.pattern = pattern self.top_level = top_level @@ -77,6 +362,7 @@ class DiscoverRunner(object): self.keepdb = keepdb self.reverse = reverse self.debug_sql = debug_sql + self.parallel = parallel @classmethod def add_arguments(cls, parser): @@ -95,6 +381,10 @@ class DiscoverRunner(object): parser.add_argument('-d', '--debug-sql', action='store_true', dest='debug_sql', default=False, help='Prints logged SQL queries on failure.') + parser.add_argument( + '--parallel', dest='parallel', nargs='?', default=1, type=int, + const=default_test_processes(), + help='Run tests in parallel processes.') def setup_test_environment(self, **kwargs): setup_test_environment() @@ -160,12 +450,27 @@ class DiscoverRunner(object): for test in extra_tests: suite.addTest(test) - return reorder_suite(suite, self.reorder_by, self.reverse) + suite = reorder_suite(suite, self.reorder_by, self.reverse) + + if self.parallel > 1: + parallel_suite = self.parallel_test_suite(suite, self.parallel, self.failfast) + + # Since tests are distributed across processes on a per-TestCase + # basis, there's no need for more processes than TestCases. + parallel_units = len(parallel_suite.subsuites) + if self.parallel > parallel_units: + self.parallel = parallel_units + + # If there's only one TestCase, parallelization isn't needed. + if self.parallel > 1: + suite = parallel_suite + + return suite def setup_databases(self, **kwargs): return setup_databases( self.verbosity, self.interactive, self.keepdb, self.debug_sql, - **kwargs + self.parallel, **kwargs ) def get_resultclass(self): @@ -185,6 +490,13 @@ class DiscoverRunner(object): """ for connection, old_name, destroy in old_config: if destroy: + if self.parallel > 1: + for index in range(self.parallel): + connection.creation.destroy_test_db( + number=index + 1, + verbosity=self.verbosity, + keepdb=self.keepdb, + ) connection.creation.destroy_test_db(old_name, self.verbosity, self.keepdb) def teardown_test_environment(self, **kwargs): @@ -288,14 +600,14 @@ def reorder_suite(suite, classes, reverse=False): class_count = len(classes) suite_class = type(suite) bins = [OrderedSet() for i in range(class_count + 1)] - partition_suite(suite, classes, bins, reverse=reverse) + partition_suite_by_type(suite, classes, bins, reverse=reverse) reordered_suite = suite_class() for i in range(class_count + 1): reordered_suite.addTests(bins[i]) return reordered_suite -def partition_suite(suite, classes, bins, reverse=False): +def partition_suite_by_type(suite, classes, bins, reverse=False): """ Partitions a test suite by test type. Also prevents duplicated tests. @@ -311,7 +623,7 @@ def partition_suite(suite, classes, bins, reverse=False): suite = reversed(tuple(suite)) for test in suite: if isinstance(test, suite_class): - partition_suite(test, classes, bins, reverse=reverse) + partition_suite_by_type(test, classes, bins, reverse=reverse) else: for i in range(len(classes)): if isinstance(test, classes[i]): @@ -321,6 +633,21 @@ def partition_suite(suite, classes, bins, reverse=False): bins[-1].add(test) +def partition_suite_by_case(suite): + """ + Partitions a test suite by test case, preserving the order of tests. + """ + groups = [] + suite_class = type(suite) + for test_type, test_group in itertools.groupby(suite, type): + if issubclass(test_type, unittest.TestCase): + groups.append(suite_class(test_group)) + else: + for item in test_group: + groups.extend(partition_suite_by_case(item)) + return groups + + def get_unique_databases(): """ Figure out which databases actually need to be created. @@ -363,7 +690,7 @@ def get_unique_databases(): return test_databases -def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, **kwargs): +def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, parallel=0, **kwargs): """ Creates the test databases. """ @@ -381,11 +708,18 @@ def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, **kwa if first_alias is None: first_alias = alias connection.creation.create_test_db( - verbosity, + verbosity=verbosity, autoclobber=not interactive, keepdb=keepdb, serialize=connection.settings_dict.get("TEST", {}).get("SERIALIZE", True), ) + if parallel > 1: + for index in range(parallel): + connection.creation.clone_test_db( + number=index + 1, + verbosity=verbosity, + keepdb=keepdb, + ) # Configure all other connections as mirrors of the first one else: connections[alias].creation.set_as_test_mirror( diff --git a/django/test/testcases.py b/django/test/testcases.py index 0c704ad951..302c284edb 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -20,6 +20,7 @@ from django.apps import apps from django.conf import settings from django.core import mail from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.core.files import locks from django.core.handlers.wsgi import WSGIHandler, get_path_info from django.core.management import call_command from django.core.management.color import no_style @@ -1318,7 +1319,7 @@ class LiveServerTestCase(TransactionTestCase): # Launch the live server's thread specified_address = os.environ.get( - 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081') + 'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081-8179') # The specified ports may be of the form '8000-8010,8080,9200-9300' # i.e. a comma-separated list of ports or ranges of ports, so we break @@ -1379,3 +1380,31 @@ class LiveServerTestCase(TransactionTestCase): def tearDownClass(cls): cls._tearDownClassInternal() super(LiveServerTestCase, cls).tearDownClass() + + +class SerializeMixin(object): + """ + Mixin to enforce serialization of TestCases that share a common resource. + + Define a common 'lockfile' for each set of TestCases to serialize. This + file must exist on the filesystem. + + Place it early in the MRO in order to isolate setUpClass / tearDownClass. + """ + + lockfile = None + + @classmethod + def setUpClass(cls): + if cls.lockfile is None: + raise ValueError( + "{}.lockfile isn't set. Set it to a unique value " + "in the base class.".format(cls.__name__)) + cls._lockfile = open(cls.lockfile) + locks.lock(cls._lockfile, locks.LOCK_EX) + super(SerializeMixin, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + super(SerializeMixin, cls).tearDownClass() + cls._lockfile.close() diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 8a097bba8c..55e3a2d5c7 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -284,3 +284,16 @@ combine this with ``--verbosity=2``, all SQL queries will be output:: .. versionadded:: 1.8 The ``--reverse`` and ``--debug-sql`` options were added. + +By default tests are run in parallel with one process per core. You can adjust +this behavior with the ``--parallel`` option:: + + $ ./runtests.py basic --parallel=1 + +You can also use the ``DJANGO_TEST_PROCESSES`` environment variable for this +purpose. + +.. versionadded:: 1.9 + + Support for running tests in parallel and the ``--parallel`` option were + added. diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 35a02eecc8..26702d7db7 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -1227,7 +1227,11 @@ provided by the :setting:`TEST_RUNNER` setting. The ``--liveserver`` option can be used to override the default address where the live server (used with :class:`~django.test.LiveServerTestCase`) is -expected to run from. The default value is ``localhost:8081``. +expected to run from. The default value is ``localhost:8081-8179``. + +.. versionchanged:: 1.9 + + In earlier versions, the default value was ``localhost:8081``. .. django-admin-option:: --keepdb @@ -1257,6 +1261,48 @@ The ``--debug-sql`` option can be used to enable :ref:`SQL logging ` for failing tests. If :djadminopt:`--verbosity` is ``2``, then queries in passing tests are also output. +.. django-admin-option:: --parallel + +.. versionadded:: 1.9 + +The ``--parallel`` option can be used to run tests in parallel in separate +processes. Since modern processors have multiple cores, this allows running +tests significantly faster. + +By default ``--parallel`` runs one process per core according to +:func:`multiprocessing.cpu_count()`. You can adjust the number of processes +either by providing it as the option's value, e.g. ``--parallel=4``, or by +setting the ``DJANGO_TEST_PROCESSES`` environment variable. + +Django distributes test cases — :class:`unittest.TestCase` subclasses — to +subprocesses. If there are fewer test cases than configured processes, Django +will reduce the number of processes accordingly. + +Each process gets its own database. You must ensure that different test cases +don't access the same resources. For instance, test cases that touch the +filesystem should create a temporary directory for their own use. + +This option requires the third-party ``tblib`` package to display tracebacks +correctly: + +.. code-block:: console + + $ pip install tblib + +This feature isn't available on Windows. It doesn't work with the Oracle +database backend either. + +.. warning:: + + When test parallelization is enabled and a test fails, Django may be + unable to display the exception traceback. This can make debugging + difficult. If you encounter this problem, run the affected test without + parallelization to see the traceback of the failure. + + This is a known limitation. It arises from the need to serialize objects + in order to exchange them between processes. See + :ref:`python:pickle-picklable` for details. + testserver -------------------------------- diff --git a/docs/releases/1.9.txt b/docs/releases/1.9.txt index 366e1ac934..3d256ae29d 100644 --- a/docs/releases/1.9.txt +++ b/docs/releases/1.9.txt @@ -125,6 +125,21 @@ degradation. .. _YUI's A-grade: https://github.com/yui/yui3/wiki/Graded-Browser-Support +Running tests in parallel +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :djadmin:`test` command now supports a :djadminopt:`--parallel` option to +run a project's tests in multiple processes in parallel. + +Each process gets its own database. You must ensure that different test cases +don't access the same resources. For instance, test cases that touch the +filesystem should create a temporary directory for their own use. + +This option is enabled by default for Django's own test suite provided: + +- the OS supports it (all but Windows) +- the database backend supports it (all the built-in backends but Oracle) + Minor features ~~~~~~~~~~~~~~ @@ -675,6 +690,11 @@ Database backend API before 1.0, but hasn't been overridden by any core backend in years and hasn't been called anywhere in Django's code or tests. +* In order to support test parallelization, you must implement the + ``DatabaseCreation._clone_test_db()`` method and set + ``DatabaseFeatures.can_clone_databases = True``. You may have to adjust + ``DatabaseCreation.get_test_db_clone_settings()``. + Default settings that were tuples are now lists ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1041,6 +1061,9 @@ Miscellaneous changed. They used to be ``(old_names, mirrors)`` tuples. Now they're just the first item, ``old_names``. +* By default :class:`~django.test.LiveServerTestCase` attempts to find an + available port in the 8081-8179 range instead of just trying port 8081. + .. _deprecated-features-1.9: Features deprecated in 1.9 diff --git a/docs/topics/testing/tools.txt b/docs/topics/testing/tools.txt index 28a4921198..52a00b7ded 100644 --- a/docs/topics/testing/tools.txt +++ b/docs/topics/testing/tools.txt @@ -812,11 +812,18 @@ This allows the use of automated test clients other than the client, to execute a series of functional tests inside a browser and simulate a real user's actions. -By default the live server's address is ``'localhost:8081'`` and the full URL -can be accessed during the tests with ``self.live_server_url``. If you'd like -to change the default address (in the case, for example, where the 8081 port is -already taken) then you may pass a different one to the :djadmin:`test` command -via the :djadminopt:`--liveserver` option, for example: +By default the live server listens on ``localhost`` and picks the first +available port in the ``8081-8179`` range. Its full URL can be accessed with +``self.live_server_url`` during the tests. + +.. versionchanged:: 1.9 + + In earlier versions, the live server's default address was always + ``'localhost:8081'``. + +If you'd like to select another address then you may pass a different one to +the :djadmin:`test` command via the :djadminopt:`--liveserver` option, for +example: .. code-block:: console diff --git a/tests/admin_scripts/tests.py b/tests/admin_scripts/tests.py index 616082346b..397856912e 100644 --- a/tests/admin_scripts/tests.py +++ b/tests/admin_scripts/tests.py @@ -33,23 +33,38 @@ from django.utils._os import npath, upath from django.utils.encoding import force_text from django.utils.six import PY3, StringIO -test_dir = os.path.realpath(os.path.join(tempfile.gettempdir(), 'test_project')) -if not os.path.exists(test_dir): - os.mkdir(test_dir) - open(os.path.join(test_dir, '__init__.py'), 'w').close() - custom_templates_dir = os.path.join(os.path.dirname(upath(__file__)), 'custom_templates') + SYSTEM_CHECK_MSG = 'System check identified no issues' class AdminScriptTestCase(unittest.TestCase): + + @classmethod + def setUpClass(cls): + super(AdminScriptTestCase, cls).setUpClass() + cls.test_dir = os.path.realpath(os.path.join( + tempfile.gettempdir(), + cls.__name__, + 'test_project', + )) + if not os.path.exists(cls.test_dir): + os.makedirs(cls.test_dir) + with open(os.path.join(cls.test_dir, '__init__.py'), 'w'): + pass + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.test_dir) + super(AdminScriptTestCase, cls).tearDownClass() + def write_settings(self, filename, apps=None, is_dir=False, sdict=None, extra=None): if is_dir: - settings_dir = os.path.join(test_dir, filename) + settings_dir = os.path.join(self.test_dir, filename) os.mkdir(settings_dir) settings_file_path = os.path.join(settings_dir, '__init__.py') else: - settings_file_path = os.path.join(test_dir, filename) + settings_file_path = os.path.join(self.test_dir, filename) with open(settings_file_path, 'w') as settings_file: settings_file.write('# -*- coding: utf-8 -*\n') @@ -78,7 +93,7 @@ class AdminScriptTestCase(unittest.TestCase): settings_file.write("%s = %s\n" % (k, v)) def remove_settings(self, filename, is_dir=False): - full_name = os.path.join(test_dir, filename) + full_name = os.path.join(self.test_dir, filename) if is_dir: shutil.rmtree(full_name) else: @@ -96,7 +111,7 @@ class AdminScriptTestCase(unittest.TestCase): except OSError: pass # Also remove a __pycache__ directory, if it exists - cache_name = os.path.join(test_dir, '__pycache__') + cache_name = os.path.join(self.test_dir, '__pycache__') if os.path.isdir(cache_name): shutil.rmtree(cache_name) @@ -115,7 +130,7 @@ class AdminScriptTestCase(unittest.TestCase): return paths def run_test(self, script, args, settings_file=None, apps=None): - base_dir = os.path.dirname(test_dir) + base_dir = os.path.dirname(self.test_dir) # The base dir for Django's tests is one level up. tests_dir = os.path.dirname(os.path.dirname(upath(__file__))) # The base dir for Django is one level above the test dir. We don't use @@ -145,7 +160,7 @@ class AdminScriptTestCase(unittest.TestCase): test_environ[str('PYTHONWARNINGS')] = str('') # Move to the test directory and run - os.chdir(test_dir) + os.chdir(self.test_dir) out, err = subprocess.Popen([sys.executable, script] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=test_environ, universal_newlines=True).communicate() @@ -168,7 +183,7 @@ class AdminScriptTestCase(unittest.TestCase): conf_dir = os.path.dirname(upath(conf.__file__)) template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py') - test_manage_py = os.path.join(test_dir, 'manage.py') + test_manage_py = os.path.join(self.test_dir, 'manage.py') shutil.copyfile(template_manage_py, test_manage_py) with open(test_manage_py, 'r') as fp: @@ -590,7 +605,7 @@ class DjangoAdminSettingsDirectory(AdminScriptTestCase): def test_setup_environ(self): "directory: startapp creates the correct directory" args = ['startapp', 'settings_test'] - app_path = os.path.join(test_dir, 'settings_test') + app_path = os.path.join(self.test_dir, 'settings_test') out, err = self.run_django_admin(args, 'test_project.settings') self.addCleanup(shutil.rmtree, app_path) self.assertNoOutput(err) @@ -611,7 +626,7 @@ class DjangoAdminSettingsDirectory(AdminScriptTestCase): "directory: startapp creates the correct directory with a custom template" template_path = os.path.join(custom_templates_dir, 'app_template') args = ['startapp', '--template', template_path, 'custom_settings_test'] - app_path = os.path.join(test_dir, 'custom_settings_test') + app_path = os.path.join(self.test_dir, 'custom_settings_test') out, err = self.run_django_admin(args, 'test_project.settings') self.addCleanup(shutil.rmtree, app_path) self.assertNoOutput(err) @@ -1047,7 +1062,7 @@ class ManageSettingsWithSettingsErrors(AdminScriptTestCase): self.remove_settings('settings.py') def write_settings_with_import_error(self, filename): - settings_file_path = os.path.join(test_dir, filename) + settings_file_path = os.path.join(self.test_dir, filename) with open(settings_file_path, 'w') as settings_file: settings_file.write('# Settings file automatically generated by admin_scripts test case\n') settings_file.write('# The next line will cause an import error:\nimport foo42bar\n') @@ -1802,7 +1817,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): def test_simple_project(self): "Make sure the startproject management command creates a project" args = ['startproject', 'testproject'] - testproject_dir = os.path.join(test_dir, 'testproject') + testproject_dir = os.path.join(self.test_dir, 'testproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1818,7 +1833,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure the startproject management command validates a project name" for bad_name in ('7testproject', '../testproject'): args = ['startproject', bad_name] - testproject_dir = os.path.join(test_dir, bad_name) + testproject_dir = os.path.join(self.test_dir, bad_name) self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1829,7 +1844,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): def test_simple_project_different_directory(self): "Make sure the startproject management command creates a project in a specific directory" args = ['startproject', 'testproject', 'othertestproject'] - testproject_dir = os.path.join(test_dir, 'othertestproject') + testproject_dir = os.path.join(self.test_dir, 'othertestproject') os.mkdir(testproject_dir) self.addCleanup(shutil.rmtree, testproject_dir) @@ -1846,7 +1861,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure the startproject management command is able to use a different project template" template_path = os.path.join(custom_templates_dir, 'project_template') args = ['startproject', '--template', template_path, 'customtestproject'] - testproject_dir = os.path.join(test_dir, 'customtestproject') + testproject_dir = os.path.join(self.test_dir, 'customtestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1858,7 +1873,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Ticket 17475: Template dir passed has a trailing path separator" template_path = os.path.join(custom_templates_dir, 'project_template' + os.sep) args = ['startproject', '--template', template_path, 'customtestproject'] - testproject_dir = os.path.join(test_dir, 'customtestproject') + testproject_dir = os.path.join(self.test_dir, 'customtestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1870,7 +1885,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure the startproject management command is able to use a different project template from a tarball" template_path = os.path.join(custom_templates_dir, 'project_template.tgz') args = ['startproject', '--template', template_path, 'tarballtestproject'] - testproject_dir = os.path.join(test_dir, 'tarballtestproject') + testproject_dir = os.path.join(self.test_dir, 'tarballtestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1882,7 +1897,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Startproject can use a project template from a tarball and create it in a specified location" template_path = os.path.join(custom_templates_dir, 'project_template.tgz') args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation'] - testproject_dir = os.path.join(test_dir, 'altlocation') + testproject_dir = os.path.join(self.test_dir, 'altlocation') os.mkdir(testproject_dir) self.addCleanup(shutil.rmtree, testproject_dir) @@ -1896,7 +1911,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_url = '%s/custom_templates/project_template.tgz' % self.live_server_url args = ['startproject', '--template', template_url, 'urltestproject'] - testproject_dir = os.path.join(test_dir, 'urltestproject') + testproject_dir = os.path.join(self.test_dir, 'urltestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1909,7 +1924,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_url = '%s/custom_templates/project_template.tgz/' % self.live_server_url args = ['startproject', '--template', template_url, 'urltestproject'] - testproject_dir = os.path.join(test_dir, 'urltestproject') + testproject_dir = os.path.join(self.test_dir, 'urltestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1921,7 +1936,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure the startproject management command is able to render custom files" template_path = os.path.join(custom_templates_dir, 'project_template') args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile'] - testproject_dir = os.path.join(test_dir, 'customtestproject') + testproject_dir = os.path.join(self.test_dir, 'customtestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) @@ -1939,7 +1954,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure template context variables are rendered with proper values" template_path = os.path.join(custom_templates_dir, 'project_template') args = ['startproject', '--template', template_path, 'another_project', 'project_dir'] - testproject_dir = os.path.join(test_dir, 'project_dir') + testproject_dir = os.path.join(self.test_dir, 'project_dir') os.mkdir(testproject_dir) self.addCleanup(shutil.rmtree, testproject_dir) out, err = self.run_django_admin(args) @@ -1957,7 +1972,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): self.addCleanup(self.remove_settings, 'alternate_settings.py') template_path = os.path.join(custom_templates_dir, 'project_template') args = ['custom_startproject', '--template', template_path, 'another_project', 'project_dir', '--extra', '<&>', '--settings=alternate_settings'] - testproject_dir = os.path.join(test_dir, 'project_dir') + testproject_dir = os.path.join(self.test_dir, 'project_dir') os.mkdir(testproject_dir) self.addCleanup(shutil.rmtree, testproject_dir) out, err = self.run_manage(args) @@ -1974,7 +1989,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): """ template_path = os.path.join(custom_templates_dir, 'project_template') args = ['startproject', '--template', template_path, 'yet_another_project', 'project_dir2'] - testproject_dir = os.path.join(test_dir, 'project_dir2') + testproject_dir = os.path.join(self.test_dir, 'project_dir2') out, err = self.run_django_admin(args) self.assertNoOutput(out) self.assertOutput(err, "Destination directory '%s' does not exist, please create it first." % testproject_dir) @@ -1984,7 +1999,7 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Ticket 18091: Make sure the startproject management command is able to render templates with non-ASCII content" template_path = os.path.join(custom_templates_dir, 'project_template') args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject'] - testproject_dir = os.path.join(test_dir, 'customtestproject') + testproject_dir = os.path.join(self.test_dir, 'customtestproject') self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) diff --git a/tests/backends/tests.py b/tests/backends/tests.py index ed42d6317a..c0af1bdb60 100644 --- a/tests/backends/tests.py +++ b/tests/backends/tests.py @@ -2,7 +2,6 @@ # Unit and doctests for specific database backends. from __future__ import unicode_literals -import copy import datetime import re import threading @@ -10,7 +9,6 @@ import unittest import warnings from decimal import Decimal, Rounded -from django.conf import settings from django.core.exceptions import ImproperlyConfigured from django.core.management.color import no_style from django.db import ( @@ -182,8 +180,8 @@ class PostgreSQLTests(TestCase): nodb_conn = connection._nodb_connection del connection._nodb_connection self.assertIsNotNone(nodb_conn.settings_dict['NAME']) - self.assertEqual(nodb_conn.settings_dict['NAME'], settings.DATABASES[DEFAULT_DB_ALIAS]['NAME']) - # Check a RuntimeWarning nas been emitted + self.assertEqual(nodb_conn.settings_dict['NAME'], connection.settings_dict['NAME']) + # Check a RuntimeWarning has been emitted self.assertEqual(len(w), 1) self.assertEqual(w[0].message.__class__, RuntimeWarning) @@ -219,9 +217,7 @@ class PostgreSQLTests(TestCase): PostgreSQL shouldn't roll back SET TIME ZONE, even if the first transaction is rolled back (#17062). """ - databases = copy.deepcopy(settings.DATABASES) - new_connections = ConnectionHandler(databases) - new_connection = new_connections[DEFAULT_DB_ALIAS] + new_connection = connection.copy() try: # Ensure the database default time zone is different than @@ -258,10 +254,9 @@ class PostgreSQLTests(TestCase): The connection wrapper shouldn't believe that autocommit is enabled after setting the time zone when AUTOCOMMIT is False (#21452). """ - databases = copy.deepcopy(settings.DATABASES) - databases[DEFAULT_DB_ALIAS]['AUTOCOMMIT'] = False - new_connections = ConnectionHandler(databases) - new_connection = new_connections[DEFAULT_DB_ALIAS] + new_connection = connection.copy() + new_connection.settings_dict['AUTOCOMMIT'] = False + try: # Open a database connection. new_connection.cursor() @@ -285,10 +280,8 @@ class PostgreSQLTests(TestCase): # Check the level on the psycopg2 connection, not the Django wrapper. self.assertEqual(connection.connection.isolation_level, read_committed) - databases = copy.deepcopy(settings.DATABASES) - databases[DEFAULT_DB_ALIAS]['OPTIONS']['isolation_level'] = serializable - new_connections = ConnectionHandler(databases) - new_connection = new_connections[DEFAULT_DB_ALIAS] + new_connection = connection.copy() + new_connection.settings_dict['OPTIONS']['isolation_level'] = serializable try: # Start a transaction so the isolation level isn't reported as 0. new_connection.set_autocommit(False) @@ -748,8 +741,7 @@ class BackendTestCase(TransactionTestCase): """ old_queries_limit = BaseDatabaseWrapper.queries_limit BaseDatabaseWrapper.queries_limit = 3 - new_connections = ConnectionHandler(settings.DATABASES) - new_connection = new_connections[DEFAULT_DB_ALIAS] + new_connection = connection.copy() # Initialize the connection and clear initialization statements. with new_connection.cursor(): diff --git a/tests/delete_regress/tests.py b/tests/delete_regress/tests.py index 7cbd94f427..2128733798 100644 --- a/tests/delete_regress/tests.py +++ b/tests/delete_regress/tests.py @@ -2,9 +2,7 @@ from __future__ import unicode_literals import datetime -from django.conf import settings -from django.db import DEFAULT_DB_ALIAS, models, transaction -from django.db.utils import ConnectionHandler +from django.db import connection, models, transaction from django.test import TestCase, TransactionTestCase, skipUnlessDBFeature from .models import ( @@ -24,8 +22,7 @@ class DeleteLockingTest(TransactionTestCase): def setUp(self): # Create a second connection to the default database - new_connections = ConnectionHandler(settings.DATABASES) - self.conn2 = new_connections[DEFAULT_DB_ALIAS] + self.conn2 = connection.copy() self.conn2.set_autocommit(False) def tearDown(self): diff --git a/tests/i18n/test_extraction.py b/tests/i18n/test_extraction.py index 4a612c546b..b9c4b4fc3e 100644 --- a/tests/i18n/test_extraction.py +++ b/tests/i18n/test_extraction.py @@ -17,6 +17,7 @@ from django.core.management.commands.makemessages import \ Command as MakeMessagesCommand from django.core.management.utils import find_command from django.test import SimpleTestCase, mock, override_settings +from django.test.testcases import SerializeMixin from django.test.utils import captured_stderr, captured_stdout from django.utils import six from django.utils._os import upath @@ -30,7 +31,13 @@ this_directory = os.path.dirname(upath(__file__)) @skipUnless(has_xgettext, 'xgettext is mandatory for extraction tests') -class ExtractorTests(SimpleTestCase): +class ExtractorTests(SerializeMixin, SimpleTestCase): + + # makemessages scans the current working directory and writes in the + # locale subdirectory. There aren't any options to control this. As a + # consequence tests can't run in parallel. Since i18n tests run in less + # than 4 seconds, serializing them with SerializeMixin is acceptable. + lockfile = __file__ test_dir = os.path.abspath(os.path.join(this_directory, 'commands')) @@ -610,9 +617,6 @@ class KeepPotFileExtractorTests(ExtractorTests): POT_FILE = 'locale/django.pot' - def setUp(self): - super(KeepPotFileExtractorTests, self).setUp() - def tearDown(self): super(KeepPotFileExtractorTests, self).tearDown() os.chdir(self.test_dir) @@ -646,6 +650,7 @@ class MultipleLocaleExtractionTests(ExtractorTests): LOCALES = ['pt', 'de', 'ch'] def tearDown(self): + super(MultipleLocaleExtractionTests, self).tearDown() os.chdir(self.test_dir) for locale in self.LOCALES: try: @@ -677,7 +682,6 @@ class ExcludedLocaleExtractionTests(ExtractorTests): def setUp(self): super(ExcludedLocaleExtractionTests, self).setUp() - os.chdir(self.test_dir) # ExtractorTests.tearDown() takes care of restoring. shutil.copytree('canned_locale', 'locale') self._set_times_for_all_po_files() @@ -719,7 +723,7 @@ class ExcludedLocaleExtractionTests(ExtractorTests): class CustomLayoutExtractionTests(ExtractorTests): def setUp(self): - self._cwd = os.getcwd() + super(CustomLayoutExtractionTests, self).setUp() self.test_dir = os.path.join(this_directory, 'project_dir') def test_no_locale_raises(self): diff --git a/tests/model_fields/test_imagefield.py b/tests/model_fields/test_imagefield.py index c2908b06e4..8320fafaa8 100644 --- a/tests/model_fields/test_imagefield.py +++ b/tests/model_fields/test_imagefield.py @@ -8,6 +8,7 @@ from django.core.exceptions import ImproperlyConfigured from django.core.files import File from django.core.files.images import ImageFile from django.test import TestCase +from django.test.testcases import SerializeMixin from django.utils._os import upath try: @@ -27,11 +28,13 @@ else: PersonTwoImages = Person -class ImageFieldTestMixin(object): +class ImageFieldTestMixin(SerializeMixin): """ Mixin class to provide common functionality to ImageField test classes. """ + lockfile = __file__ + # Person model to use for tests. PersonModel = PersonWithHeightAndWidth # File class to use for file instances. diff --git a/tests/requirements/base.txt b/tests/requirements/base.txt index af828306c3..845aefbb87 100644 --- a/tests/requirements/base.txt +++ b/tests/requirements/base.txt @@ -8,3 +8,4 @@ PyYAML pytz > dev selenium sqlparse +tblib diff --git a/tests/runtests.py b/tests/runtests.py index 97c09c7372..4d6e22e3b2 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import atexit import logging import os import shutil @@ -11,8 +12,9 @@ from argparse import ArgumentParser import django from django.apps import apps from django.conf import settings -from django.db import connection +from django.db import connection, connections from django.test import TestCase, TransactionTestCase +from django.test.runner import default_test_processes from django.test.utils import get_runner from django.utils import six from django.utils._os import upath @@ -34,6 +36,12 @@ TMPDIR = tempfile.mkdtemp(prefix='django_') # so that children processes inherit it. tempfile.tempdir = os.environ['TMPDIR'] = TMPDIR +# Removing the temporary TMPDIR. Ensure we pass in unicode so that it will +# successfully remove temp trees containing non-ASCII filenames on Windows. +# (We're assuming the temp dir name itself only contains ASCII characters.) +atexit.register(shutil.rmtree, six.text_type(TMPDIR)) + + SUBDIRS_TO_SKIP = [ 'data', 'import_error_package', @@ -93,9 +101,12 @@ def get_installed(): return [app_config.name for app_config in apps.get_app_configs()] -def setup(verbosity, test_labels): +def setup(verbosity, test_labels, parallel): if verbosity >= 1: - print("Testing against Django installed in '%s'" % os.path.dirname(django.__file__)) + msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__) + if parallel > 1: + msg += " with %d processes" % parallel + print(msg) # Force declaring available_apps in TransactionTestCase for faster tests. def no_available_apps(self): @@ -218,22 +229,28 @@ def setup(verbosity, test_labels): def teardown(state): - try: - # Removing the temporary TMPDIR. Ensure we pass in unicode - # so that it will successfully remove temp trees containing - # non-ASCII filenames on Windows. (We're assuming the temp dir - # name itself does not contain non-ASCII characters.) - shutil.rmtree(six.text_type(TMPDIR)) - except OSError: - print('Failed to remove temp directory: %s' % TMPDIR) - # Restore the old settings. for key, value in state.items(): setattr(settings, key, value) -def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels, debug_sql): - state = setup(verbosity, test_labels) +def actual_test_processes(parallel): + if parallel == 0: + # On Python 3.4+: if multiprocessing.get_start_method() != 'fork': + if not hasattr(os, 'fork'): + return 1 + # This doesn't work before django.setup() on some databases. + elif all(conn.features.can_clone_databases for conn in connections.all()): + return default_test_processes() + else: + return 1 + else: + return parallel + + +def django_tests(verbosity, interactive, failfast, keepdb, reverse, + test_labels, debug_sql, parallel): + state = setup(verbosity, test_labels, parallel) extra_tests = [] # Run the test suite, including the extra validation tests. @@ -248,6 +265,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels, keepdb=keepdb, reverse=reverse, debug_sql=debug_sql, + parallel=actual_test_processes(parallel), ) failures = test_runner.run_tests( test_labels or get_installed(), @@ -386,13 +404,18 @@ if __name__ == "__main__": parser.add_argument('--liveserver', help='Overrides the default address where the live server (used with ' 'LiveServerTestCase) is expected to run from. The default value ' - 'is localhost:8081.') + 'is localhost:8081-8179.') parser.add_argument( '--selenium', action='store_true', dest='selenium', default=False, - help='Run the Selenium tests as well (if Selenium is installed)') + help='Run the Selenium tests as well (if Selenium is installed).') parser.add_argument( '--debug-sql', action='store_true', dest='debug_sql', default=False, - help='Turn on the SQL query logger within tests') + help='Turn on the SQL query logger within tests.') + parser.add_argument( + '--parallel', dest='parallel', nargs='?', default=0, type=int, + const=default_test_processes(), + help='Run tests in parallel processes.') + options = parser.parse_args() # mock is a required dependency @@ -429,6 +452,6 @@ if __name__ == "__main__": failures = django_tests(options.verbosity, options.interactive, options.failfast, options.keepdb, options.reverse, options.modules, - options.debug_sql) + options.debug_sql, options.parallel) if failures: sys.exit(bool(failures)) diff --git a/tests/select_for_update/tests.py b/tests/select_for_update/tests.py index af408f8ffe..70cd21d594 100644 --- a/tests/select_for_update/tests.py +++ b/tests/select_for_update/tests.py @@ -5,9 +5,7 @@ import time from multiple_database.routers import TestRouter -from django.conf import settings -from django.db import connection, router, transaction -from django.db.utils import DEFAULT_DB_ALIAS, ConnectionHandler, DatabaseError +from django.db import DatabaseError, connection, router, transaction from django.test import ( TransactionTestCase, override_settings, skipIfDBFeature, skipUnlessDBFeature, @@ -30,8 +28,7 @@ class SelectForUpdateTests(TransactionTestCase): # We need another database connection in transaction to test that one # connection issuing a SELECT ... FOR UPDATE will block. - new_connections = ConnectionHandler(settings.DATABASES) - self.new_connection = new_connections[DEFAULT_DB_ALIAS] + self.new_connection = connection.copy() def tearDown(self): try: diff --git a/tests/staticfiles_tests/settings.py b/tests/staticfiles_tests/settings.py index 1978d6529d..ea20a84c6f 100644 --- a/tests/staticfiles_tests/settings.py +++ b/tests/staticfiles_tests/settings.py @@ -6,8 +6,6 @@ from django.utils._os import upath TEST_ROOT = os.path.dirname(upath(__file__)) -TESTFILES_PATH = os.path.join(TEST_ROOT, 'apps', 'test', 'static', 'test') - TEST_SETTINGS = { 'DEBUG': True, 'MEDIA_URL': '/media/', diff --git a/tests/staticfiles_tests/test_management.py b/tests/staticfiles_tests/test_management.py index 784e5b74fd..09557185ed 100644 --- a/tests/staticfiles_tests/test_management.py +++ b/tests/staticfiles_tests/test_management.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import codecs import os import shutil +import tempfile import unittest from django.conf import settings @@ -11,6 +12,7 @@ from django.contrib.staticfiles.management.commands import collectstatic from django.core.exceptions import ImproperlyConfigured from django.core.management import call_command from django.test import override_settings +from django.test.utils import extend_sys_path from django.utils import six from django.utils._os import symlinks_supported from django.utils.encoding import force_text @@ -197,31 +199,45 @@ class TestCollectionFilesOverride(CollectionTestCase): Test overriding duplicated files by ``collectstatic`` management command. Check for proper handling of apps order in installed apps even if file modification dates are in different order: - 'staticfiles_tests.apps.test', + 'staticfiles_test_app', 'staticfiles_tests.apps.no_label', """ def setUp(self): - self.orig_path = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'file2.txt') + self.temp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.temp_dir) + # get modification and access times for no_label/static/file2.txt + self.orig_path = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'file2.txt') self.orig_mtime = os.path.getmtime(self.orig_path) self.orig_atime = os.path.getatime(self.orig_path) - # prepare duplicate of file2.txt from no_label app + # prepare duplicate of file2.txt from a temporary app # this file will have modification time older than no_label/static/file2.txt - # anyway it should be taken to STATIC_ROOT because 'test' app is before + # anyway it should be taken to STATIC_ROOT because the temporary app is before # 'no_label' app in installed apps - self.testfile_path = os.path.join(TEST_ROOT, 'apps', 'test', 'static', 'file2.txt') + self.temp_app_path = os.path.join(self.temp_dir, 'staticfiles_test_app') + self.testfile_path = os.path.join(self.temp_app_path, 'static', 'file2.txt') + + os.makedirs(self.temp_app_path) + with open(os.path.join(self.temp_app_path, '__init__.py'), 'w+'): + pass + + os.makedirs(os.path.dirname(self.testfile_path)) with open(self.testfile_path, 'w+') as f: f.write('duplicate of file2.txt') + os.utime(self.testfile_path, (self.orig_atime - 1, self.orig_mtime - 1)) + + self.settings_with_test_app = self.modify_settings( + INSTALLED_APPS={'prepend': 'staticfiles_test_app'}) + with extend_sys_path(self.temp_dir): + self.settings_with_test_app.enable() + super(TestCollectionFilesOverride, self).setUp() def tearDown(self): - if os.path.exists(self.testfile_path): - os.unlink(self.testfile_path) - # set back original modification time - os.utime(self.orig_path, (self.orig_atime, self.orig_mtime)) super(TestCollectionFilesOverride, self).tearDown() + self.settings_with_test_app.disable() def test_ordering_override(self): """ @@ -234,22 +250,11 @@ class TestCollectionFilesOverride(CollectionTestCase): self.assertFileContains('file2.txt', 'duplicate of file2.txt') - # and now change modification time of no_label/static/file2.txt - # test app is first in installed apps so file2.txt should remain unmodified - mtime = os.path.getmtime(self.testfile_path) - atime = os.path.getatime(self.testfile_path) - os.utime(self.orig_path, (mtime + 1, atime + 1)) - - # run collectstatic again - self.run_collectstatic() - - self.assertFileContains('file2.txt', 'duplicate of file2.txt') - # The collectstatic test suite already has conflicting files since both # project/test/file.txt and apps/test/static/test/file.txt are collected. To # properly test for the warning not happening unless we tell it to explicitly, -# we only include static files from the default finders. +# we remove the project directory and will add back a conflicting file later. @override_settings(STATICFILES_DIRS=[]) class TestCollectionOverwriteWarning(CollectionTestCase): """ @@ -268,41 +273,37 @@ class TestCollectionOverwriteWarning(CollectionTestCase): """ out = six.StringIO() call_command('collectstatic', interactive=False, verbosity=3, stdout=out, **kwargs) - out.seek(0) - return out.read() + return force_text(out.getvalue()) def test_no_warning(self): """ There isn't a warning if there isn't a duplicate destination. """ output = self._collectstatic_output(clear=True) - self.assertNotIn(self.warning_string, force_text(output)) + self.assertNotIn(self.warning_string, output) def test_warning(self): """ There is a warning when there are duplicate destinations. """ - # Create new file in the no_label app that also exists in the test app. - test_dir = os.path.join(TEST_ROOT, 'apps', 'no_label', 'static', 'test') - if not os.path.exists(test_dir): - os.mkdir(test_dir) + static_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, static_dir) - try: - duplicate_path = os.path.join(test_dir, 'file.txt') - with open(duplicate_path, 'w+') as f: - f.write('duplicate of file.txt') + duplicate = os.path.join(static_dir, 'test', 'file.txt') + os.mkdir(os.path.dirname(duplicate)) + with open(duplicate, 'w+') as f: + f.write('duplicate of file.txt') + + with self.settings(STATICFILES_DIRS=[static_dir]): output = self._collectstatic_output(clear=True) - self.assertIn(self.warning_string, force_text(output)) - finally: - if os.path.exists(duplicate_path): - os.unlink(duplicate_path) + self.assertIn(self.warning_string, output) - if os.path.exists(test_dir): - os.rmdir(test_dir) + os.remove(duplicate) # Make sure the warning went away again. - output = self._collectstatic_output(clear=True) - self.assertNotIn(self.warning_string, force_text(output)) + with self.settings(STATICFILES_DIRS=[static_dir]): + output = self._collectstatic_output(clear=True) + self.assertNotIn(self.warning_string, output) @override_settings(STATICFILES_STORAGE='staticfiles_tests.storage.DummyStorage') diff --git a/tests/staticfiles_tests/test_storage.py b/tests/staticfiles_tests/test_storage.py index c3318512f6..b96d9d0f12 100644 --- a/tests/staticfiles_tests/test_storage.py +++ b/tests/staticfiles_tests/test_storage.py @@ -1,7 +1,9 @@ from __future__ import unicode_literals import os +import shutil import sys +import tempfile import unittest from django.conf import settings @@ -18,7 +20,7 @@ from django.utils.encoding import force_text from .cases import ( BaseCollectionTestCase, BaseStaticFilesTestCase, StaticFilesTestCase, ) -from .settings import TEST_ROOT, TEST_SETTINGS, TESTFILES_PATH +from .settings import TEST_ROOT, TEST_SETTINGS def hashed_file_path(test, path): @@ -252,15 +254,25 @@ class TestCollectionManifestStorage(TestHashedFiles, BaseCollectionTestCase, def setUp(self): super(TestCollectionManifestStorage, self).setUp() - self._clear_filename = os.path.join(TESTFILES_PATH, 'cleared.txt') + temp_dir = tempfile.mkdtemp() + os.makedirs(os.path.join(temp_dir, 'test')) + self._clear_filename = os.path.join(temp_dir, 'test', 'cleared.txt') with open(self._clear_filename, 'w') as f: f.write('to be deleted in one test') + self.patched_settings = self.settings( + STATICFILES_DIRS=settings.STATICFILES_DIRS + [temp_dir]) + self.patched_settings.enable() + self.addCleanup(shutil.rmtree, six.text_type(temp_dir)) + def tearDown(self): - super(TestCollectionManifestStorage, self).tearDown() + self.patched_settings.disable() + if os.path.exists(self._clear_filename): os.unlink(self._clear_filename) + super(TestCollectionManifestStorage, self).tearDown() + def test_manifest_exists(self): filename = storage.staticfiles_storage.manifest_name path = storage.staticfiles_storage.path(filename) diff --git a/tests/test_runner/tests.py b/tests/test_runner/tests.py index 8e9991d654..bf0b8795cc 100644 --- a/tests/test_runner/tests.py +++ b/tests/test_runner/tests.py @@ -319,7 +319,7 @@ class SetupDatabasesTests(unittest.TestCase): with mock.patch('django.test.runner.connections', new=tested_connections): self.runner_instance.setup_databases() mocked_db_creation.return_value.create_test_db.assert_called_once_with( - 0, autoclobber=False, serialize=True, keepdb=False + verbosity=0, autoclobber=False, serialize=True, keepdb=False ) def test_serialized_off(self): @@ -333,7 +333,7 @@ class SetupDatabasesTests(unittest.TestCase): with mock.patch('django.test.runner.connections', new=tested_connections): self.runner_instance.setup_databases() mocked_db_creation.return_value.create_test_db.assert_called_once_with( - 0, autoclobber=False, serialize=False, keepdb=False + verbosity=0, autoclobber=False, serialize=False, keepdb=False ) diff --git a/tests/timezones/tests.py b/tests/timezones/tests.py index 18539687a8..65a052f2cb 100644 --- a/tests/timezones/tests.py +++ b/tests/timezones/tests.py @@ -4,6 +4,7 @@ import datetime import re import sys import warnings +from contextlib import contextmanager from unittest import SkipTest, skipIf from xml.dom.minidom import parseString @@ -611,27 +612,41 @@ class ForcedTimeZoneDatabaseTests(TransactionTestCase): raise SkipTest("Database doesn't support feature(s): test_db_allows_multiple_connections") super(ForcedTimeZoneDatabaseTests, cls).setUpClass() - connections.databases['tz'] = connections.databases['default'].copy() - connections.databases['tz']['TIME_ZONE'] = 'Asia/Bangkok' - @classmethod - def tearDownClass(cls): - connections['tz'].close() - del connections['tz'] - del connections.databases['tz'] - super(ForcedTimeZoneDatabaseTests, cls).tearDownClass() + @contextmanager + def override_database_connection_timezone(self, timezone): + try: + orig_timezone = connection.settings_dict['TIME_ZONE'] + connection.settings_dict['TIME_ZONE'] = timezone + # Clear cached properties, after first accessing them to ensure they exist. + connection.timezone + del connection.timezone + connection.timezone_name + del connection.timezone_name + + yield + + finally: + connection.settings_dict['TIME_ZONE'] = orig_timezone + # Clear cached properties, after first accessing them to ensure they exist. + connection.timezone + del connection.timezone + connection.timezone_name + del connection.timezone_name def test_read_datetime(self): fake_dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=UTC) Event.objects.create(dt=fake_dt) - event = Event.objects.using('tz').get() - dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC) + with self.override_database_connection_timezone('Asia/Bangkok'): + event = Event.objects.get() + dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC) self.assertEqual(event.dt, dt) def test_write_datetime(self): dt = datetime.datetime(2011, 9, 1, 10, 20, 30, tzinfo=UTC) - Event.objects.using('tz').create(dt=dt) + with self.override_database_connection_timezone('Asia/Bangkok'): + Event.objects.create(dt=dt) event = Event.objects.get() fake_dt = datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=UTC)