From 33c7c2a55770fe8ccc297a8ae13e04487b72b3a1 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 6 Sep 2015 11:12:08 +0200 Subject: [PATCH] Enabled parallel testing by default in runtests.py. --- django/db/backends/base/features.py | 4 ++++ django/db/backends/mysql/features.py | 1 + django/db/backends/postgresql/features.py | 1 + django/db/backends/sqlite3/features.py | 1 + .../contributing/writing-code/unit-tests.txt | 13 +++++++++++ docs/releases/1.9.txt | 7 +++++- tests/requirements/base.txt | 1 + tests/runtests.py | 23 +++++++++++++++---- 8 files changed, 45 insertions(+), 6 deletions(-) diff --git a/django/db/backends/base/features.py b/django/db/backends/base/features.py index b7861865ac..095164770f 100644 --- a/django/db/backends/base/features.py +++ b/django/db/backends/base/features.py @@ -212,6 +212,10 @@ class BaseDatabaseFeatures(object): # every expression is null? greatest_least_ignores_nulls = False + # Can the backend clone databases for parallel test execution? + # Defaults to False to allow third-party backends to opt-in. + can_clone_databases = False + def __init__(self, connection): self.connection = connection diff --git a/django/db/backends/mysql/features.py b/django/db/backends/mysql/features.py index 8f2e99a791..c8ea8f7fb6 100644 --- a/django/db/backends/mysql/features.py +++ b/django/db/backends/mysql/features.py @@ -31,6 +31,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_release_savepoints = True atomic_transactions = False supports_column_check_constraints = False + can_clone_databases = True @cached_property def _mysql_storage_engine(self): diff --git a/django/db/backends/postgresql/features.py b/django/db/backends/postgresql/features.py index a07700e601..9130a4521e 100644 --- a/django/db/backends/postgresql/features.py +++ b/django/db/backends/postgresql/features.py @@ -28,3 +28,4 @@ class DatabaseFeatures(BaseDatabaseFeatures): has_case_insensitive_like = False requires_sqlparse_for_splitting = False greatest_least_ignores_nulls = True + can_clone_databases = True diff --git a/django/db/backends/sqlite3/features.py b/django/db/backends/sqlite3/features.py index 5697b8b7eb..dffc48afb8 100644 --- a/django/db/backends/sqlite3/features.py +++ b/django/db/backends/sqlite3/features.py @@ -37,6 +37,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): can_rollback_ddl = True supports_paramstyle_pyformat = False supports_sequence_reset = False + can_clone_databases = True @cached_property def uses_savepoints(self): diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index 8a097bba8c..55e3a2d5c7 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -284,3 +284,16 @@ combine this with ``--verbosity=2``, all SQL queries will be output:: .. versionadded:: 1.8 The ``--reverse`` and ``--debug-sql`` options were added. + +By default tests are run in parallel with one process per core. You can adjust +this behavior with the ``--parallel`` option:: + + $ ./runtests.py basic --parallel=1 + +You can also use the ``DJANGO_TEST_PROCESSES`` environment variable for this +purpose. + +.. versionadded:: 1.9 + + Support for running tests in parallel and the ``--parallel`` option were + added. diff --git a/docs/releases/1.9.txt b/docs/releases/1.9.txt index 99b6e44bc3..41e7701e99 100644 --- a/docs/releases/1.9.txt +++ b/docs/releases/1.9.txt @@ -135,6 +135,10 @@ Each process gets its own database. You must ensure that different test cases don't access the same resources. For instance, test cases that touch the filesystem should create a temporary directory for their own use. +This option is enabled by default for Django's own test suite on database +backends that support it (this is all the built-in backends except for +Oracle). + Minor features ~~~~~~~~~~~~~~ @@ -686,7 +690,8 @@ Database backend API and hasn't been called anywhere in Django's code or tests. * In order to support test parallelization, you must implement the - ``DatabaseCreation._clone_test_db()`` method. You may have to adjust + ``DatabaseCreation._clone_test_db()`` method and set + ``DatabaseFeatures.can_clone_databases = True``. You may have to adjust ``DatabaseCreation.get_test_db_clone_settings()``. Default settings that were tuples are now lists diff --git a/tests/requirements/base.txt b/tests/requirements/base.txt index af828306c3..845aefbb87 100644 --- a/tests/requirements/base.txt +++ b/tests/requirements/base.txt @@ -8,3 +8,4 @@ PyYAML pytz > dev selenium sqlparse +tblib diff --git a/tests/runtests.py b/tests/runtests.py index 0577737a5f..cfc8722ae2 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -12,7 +12,7 @@ from argparse import ArgumentParser import django from django.apps import apps from django.conf import settings -from django.db import connection +from django.db import connection, connections from django.test import TestCase, TransactionTestCase from django.test.runner import default_test_processes from django.test.utils import get_runner @@ -103,8 +103,10 @@ def get_installed(): def setup(verbosity, test_labels, parallel): if verbosity >= 1: - print("Testing against Django installed in '%s' with %d processes" % ( - os.path.dirname(django.__file__), parallel)) + msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__) + if parallel > 1: + msg += " with %d processes" % parallel + print(msg) # Force declaring available_apps in TransactionTestCase for faster tests. def no_available_apps(self): @@ -232,6 +234,17 @@ def teardown(state): setattr(settings, key, value) +def actual_test_processes(parallel): + if parallel == 0: + # This doesn't work before django.setup() on some databases. + if all(conn.features.can_clone_databases for conn in connections.all()): + return default_test_processes() + else: + return 1 + else: + return parallel + + def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels, debug_sql, parallel): state = setup(verbosity, test_labels, parallel) @@ -249,7 +262,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse, keepdb=keepdb, reverse=reverse, debug_sql=debug_sql, - parallel=parallel, + parallel=actual_test_processes(parallel), ) failures = test_runner.run_tests( test_labels or get_installed(), @@ -396,7 +409,7 @@ if __name__ == "__main__": '--debug-sql', action='store_true', dest='debug_sql', default=False, help='Turn on the SQL query logger within tests.') parser.add_argument( - '--parallel', dest='parallel', nargs='?', default=1, type=int, + '--parallel', dest='parallel', nargs='?', default=0, type=int, const=default_test_processes(), help='Run tests in parallel processes.')