Fixed #23742 -- Added an option to reverse tests order.

This is useful for debugging side effects affecting tests that
are usually executed before a given test. Full suite and pair
tests sort cases more or less deterministically, thus some test
cross-dependencies are easier to reveal by reversing the order.

Thanks Preston Timmons for the review.
This commit is contained in:
wrwrwr 2014-11-22 17:59:05 +01:00 committed by Tim Graham
parent ca801b8c8f
commit e22c64dfc0
9 changed files with 142 additions and 16 deletions

View File

@ -20,7 +20,7 @@ class DiscoverRunner(object):
reorder_by = (TestCase, SimpleTestCase) reorder_by = (TestCase, SimpleTestCase)
def __init__(self, pattern=None, top_level=None, def __init__(self, pattern=None, top_level=None,
verbosity=1, interactive=True, failfast=False, keepdb=False, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False,
**kwargs): **kwargs):
self.pattern = pattern self.pattern = pattern
@ -30,6 +30,7 @@ class DiscoverRunner(object):
self.interactive = interactive self.interactive = interactive
self.failfast = failfast self.failfast = failfast
self.keepdb = keepdb self.keepdb = keepdb
self.reverse = reverse
@classmethod @classmethod
def add_arguments(cls, parser): def add_arguments(cls, parser):
@ -41,7 +42,10 @@ class DiscoverRunner(object):
help='The test matching pattern. Defaults to test*.py.') help='The test matching pattern. Defaults to test*.py.')
parser.add_argument('-k', '--keepdb', action='store_true', dest='keepdb', parser.add_argument('-k', '--keepdb', action='store_true', dest='keepdb',
default=False, default=False,
help='Preserve the test DB between runs. Defaults to False') help='Preserves the test DB between runs.')
parser.add_argument('-r', '--reverse', action='store_true', dest='reverse',
default=False,
help='Reverses test cases order.')
def setup_test_environment(self, **kwargs): def setup_test_environment(self, **kwargs):
setup_test_environment() setup_test_environment()
@ -107,7 +111,7 @@ class DiscoverRunner(object):
for test in extra_tests: for test in extra_tests:
suite.addTest(test) suite.addTest(test)
return reorder_suite(suite, self.reorder_by) return reorder_suite(suite, self.reorder_by, self.reverse)
def setup_databases(self, **kwargs): def setup_databases(self, **kwargs):
return setup_databases(self.verbosity, self.interactive, self.keepdb, **kwargs) return setup_databases(self.verbosity, self.interactive, self.keepdb, **kwargs)
@ -213,7 +217,7 @@ def dependency_ordered(test_databases, dependencies):
return ordered_test_databases return ordered_test_databases
def reorder_suite(suite, classes): def reorder_suite(suite, classes, reverse=False):
""" """
Reorders a test suite by test type. Reorders a test suite by test type.
@ -221,30 +225,36 @@ def reorder_suite(suite, classes):
All tests of type classes[0] are placed first, then tests of type All tests of type classes[0] are placed first, then tests of type
classes[1], etc. Tests with no match in classes are placed last. classes[1], etc. Tests with no match in classes are placed last.
If `reverse` is True, tests within classes are sorted in opposite order,
but test classes are not reversed.
""" """
class_count = len(classes) class_count = len(classes)
suite_class = type(suite) suite_class = type(suite)
bins = [suite_class() for i in range(class_count + 1)] bins = [suite_class() for i in range(class_count + 1)]
partition_suite(suite, classes, bins) partition_suite(suite, classes, bins, reverse=reverse)
for i in range(class_count): for i in range(class_count):
bins[0].addTests(bins[i + 1]) bins[0].addTests(bins[i + 1])
return bins[0] return bins[0]
def partition_suite(suite, classes, bins): def partition_suite(suite, classes, bins, reverse=False):
""" """
Partitions a test suite by test type. Also prevents duplicated tests. Partitions a test suite by test type. Also prevents duplicated tests.
classes is a sequence of types classes is a sequence of types
bins is a sequence of TestSuites, one more than classes bins is a sequence of TestSuites, one more than classes
reverse changes the ordering of tests within bins
Tests of type classes[i] are added to bins[i], Tests of type classes[i] are added to bins[i],
tests with no match found in classes are place in bins[-1] tests with no match found in classes are place in bins[-1]
""" """
suite_class = type(suite) suite_class = type(suite)
if reverse:
suite = reversed(tuple(suite))
for test in suite: for test in suite:
if isinstance(test, suite_class): if isinstance(test, suite_class):
partition_suite(test, classes, bins) partition_suite(test, classes, bins, reverse=reverse)
else: else:
for i in range(len(classes)): for i in range(len(classes)):
if isinstance(test, classes[i]): if isinstance(test, classes[i]):

View File

@ -307,3 +307,15 @@ the first one:
.. code-block:: bash .. code-block:: bash
$ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions $ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions
You can also try running any set of tests in reverse using the ``--reverse``
option in order to verify that executing tests in a different order does not
cause any trouble:
.. code-block:: bash
$ ./runtests.py basic --reverse
.. versionadded:: 1.8
The ``--reverse`` option was added.

View File

@ -1411,6 +1411,15 @@ test suite. If the test database does not exist, it will be created on the first
run and then preserved for each subsequent run. Any unapplied migrations will also run and then preserved for each subsequent run. Any unapplied migrations will also
be applied to the test database before running the test suite. be applied to the test database before running the test suite.
.. django-admin-option:: --reverse
.. versionadded:: 1.8
The ``--reverse`` option can be used to sort test cases in the opposite order.
This may help in debugging tests that aren't properly isolated and have side
effects. :ref:`Grouping by test class <order-of-tests>` is preserved when using
this option.
testserver <fixture fixture ...> testserver <fixture fixture ...>
-------------------------------- --------------------------------

View File

@ -471,8 +471,9 @@ Tests
* The new :meth:`~django.test.SimpleTestCase.assertJSONNotEqual` assertion * The new :meth:`~django.test.SimpleTestCase.assertJSONNotEqual` assertion
allows you to test that two JSON fragments are not equal. allows you to test that two JSON fragments are not equal.
* Added the ability to preserve the test database by adding the * Added options to the :djadmin:`test` command to preserve the test database
:djadminopt:`--keepdb` flag. (:djadminopt:`--keepdb`) and to run the test cases in reverse order
(:djadminopt:`--reverse`).
* Added the :attr:`~django.test.Response.resolver_match` attribute to test * Added the :attr:`~django.test.Response.resolver_match` attribute to test
client responses. client responses.

View File

@ -355,7 +355,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a
selection of other methods that are used to by ``run_tests()`` to set up, selection of other methods that are used to by ``run_tests()`` to set up,
execute and tear down the test suite. execute and tear down the test suite.
.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=True, keepdb=False **kwargs) .. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=True, keepdb=False, reverse=False, **kwargs)
``DiscoverRunner`` will search for tests in any file matching ``pattern``. ``DiscoverRunner`` will search for tests in any file matching ``pattern``.
@ -381,6 +381,11 @@ execute and tear down the test suite.
or create one if necessary. If ``False``, a new database will be created, or create one if necessary. If ``False``, a new database will be created,
prompting the user to remove the existing one, if present. prompting the user to remove the existing one, if present.
If ``reverse`` is ``True``, test cases will be executed in the opposite
order. This could be useful to debug tests that aren't properly isolated
and have side effects. :ref:`Grouping by test class <order-of-tests>` is
preserved when using this option.
Django may, from time to time, extend the capabilities of the test runner Django may, from time to time, extend the capabilities of the test runner
by adding new arguments. The ``**kwargs`` declaration allows for this by adding new arguments. The ``**kwargs`` declaration allows for this
expansion. If you subclass ``DiscoverRunner`` or write your own test expansion. If you subclass ``DiscoverRunner`` or write your own test
@ -397,7 +402,7 @@ execute and tear down the test suite.
subclassed test runner to add options to the list of command-line subclassed test runner to add options to the list of command-line
options that the :djadmin:`test` command could use. options that the :djadmin:`test` command could use.
The ``keepdb`` argument was added. The ``keepdb`` and the ``reverse`` arguments were added.
Attributes Attributes
~~~~~~~~~~ ~~~~~~~~~~

View File

@ -234,6 +234,12 @@ the Django test runner reorders tests in the following way:
database by a given :class:`~django.test.TransactionTestCase` test, they database by a given :class:`~django.test.TransactionTestCase` test, they
must be updated to be able to run independently. must be updated to be able to run independently.
.. versionadded:: 1.8
You may reverse the execution order inside groups by passing
:djadminopt:`--reverse` to the test command. This can help with ensuring
your tests are independent from each other.
.. _test-case-serialized-rollback: .. _test-case-serialized-rollback:
Rollback emulation Rollback emulation

View File

@ -198,7 +198,7 @@ def teardown(state):
setattr(settings, key, value) setattr(settings, key, value)
def django_tests(verbosity, interactive, failfast, keepdb, test_labels): def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels):
state = setup(verbosity, test_labels) state = setup(verbosity, test_labels)
extra_tests = [] extra_tests = []
@ -212,6 +212,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, test_labels):
interactive=interactive, interactive=interactive,
failfast=failfast, failfast=failfast,
keepdb=keepdb, keepdb=keepdb,
reverse=reverse,
) )
# Catch warnings thrown in test DB setup -- remove in Django 1.9 # Catch warnings thrown in test DB setup -- remove in Django 1.9
with warnings.catch_warnings(): with warnings.catch_warnings():
@ -361,6 +362,9 @@ if __name__ == "__main__":
parser.add_argument('--pair', parser.add_argument('--pair',
help='Run the test suite in pairs with the named test to find problem ' help='Run the test suite in pairs with the named test to find problem '
'pairs.') 'pairs.')
parser.add_argument('--reverse', action='store_true', default=False,
help='Sort test suites and test cases in opposite order to debug '
'test side effects not apparent with normal execution lineup.')
parser.add_argument('--liveserver', parser.add_argument('--liveserver',
help='Overrides the default address where the live server (used with ' help='Overrides the default address where the live server (used with '
'LiveServerTestCase) is expected to run from. The default value ' 'LiveServerTestCase) is expected to run from. The default value '
@ -393,6 +397,6 @@ if __name__ == "__main__":
else: else:
failures = django_tests(options.verbosity, options.interactive, failures = django_tests(options.verbosity, options.interactive,
options.failfast, options.keepdb, options.failfast, options.keepdb,
options.modules) options.reverse, options.modules)
if failures: if failures:
sys.exit(bool(failures)) sys.exit(bool(failures))

View File

@ -1,7 +1,57 @@
from django.test import TestCase from unittest import TestCase
from django.test import SimpleTestCase, TestCase as DjangoTestCase
class Test(TestCase): class DjangoCase1(DjangoTestCase):
def test_sample(self): def test_1(self):
pass
def test_2(self):
pass
class DjangoCase2(DjangoTestCase):
def test_1(self):
pass
def test_2(self):
pass
class SimpleCase1(SimpleTestCase):
def test_1(self):
pass
def test_2(self):
pass
class SimpleCase2(SimpleTestCase):
def test_1(self):
pass
def test_2(self):
pass
class UnittestCase1(TestCase):
def test_1(self):
pass
def test_2(self):
pass
class UnittestCase2(TestCase):
def test_1(self):
pass
def test_2(self):
pass pass

View File

@ -126,6 +126,35 @@ class DiscoverRunnerTest(TestCase):
["django.contrib.gis", "django.contrib.gis.tests.geo3d"]).countTestCases() ["django.contrib.gis", "django.contrib.gis.tests.geo3d"]).countTestCases()
self.assertEqual(single, dups) self.assertEqual(single, dups)
def test_reverse(self):
"""
Reverse should reorder tests while maintaining the grouping specified
by ``DiscoverRunner.reorder_by``.
"""
runner = DiscoverRunner(reverse=True)
suite = runner.build_suite(
test_labels=('test_discovery_sample', 'test_discovery_sample2'))
self.assertIn('test_discovery_sample2', next(iter(suite)).id(),
msg="Test labels should be reversed.")
suite = runner.build_suite(test_labels=('test_discovery_sample2',))
suite = tuple(suite)
self.assertIn('DjangoCase', suite[0].id(),
msg="Test groups should not be reversed.")
self.assertIn('SimpleCase', suite[4].id(),
msg="Test groups order should be preserved.")
self.assertIn('DjangoCase2', suite[0].id(),
msg="Django test cases should be reversed.")
self.assertIn('SimpleCase2', suite[4].id(),
msg="Simple test cases should be reversed.")
self.assertIn('UnittestCase2', suite[8].id(),
msg="Unittest test cases should be reversed.")
self.assertIn('test_2', suite[0].id(),
msg="Methods of Django cases should be reversed.")
self.assertIn('test_2', suite[4].id(),
msg="Methods of simple cases should be reversed.")
self.assertIn('test_2', suite[8].id(),
msg="Methods of unittest cases should be reversed.")
def test_overrideable_test_suite(self): def test_overrideable_test_suite(self):
self.assertEqual(DiscoverRunner().test_suite, TestSuite) self.assertEqual(DiscoverRunner().test_suite, TestSuite)