mirror of https://github.com/django/django.git
Fixed #24522 -- Added a --shuffle option to DiscoverRunner.
This commit is contained in:
Chris Jerdonek
Mariusz Felisiak
parent
77b88fe621
commit
90ba716bf0
|
|
@ -1,11 +1,13 @@
|
|||
import ctypes
|
||||
import faulthandler
|
||||
import hashlib
|
||||
import io
|
||||
import itertools
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import pickle
|
||||
import random
|
||||
import sys
|
||||
import textwrap
|
||||
import unittest
|
||||
|
|
@ -469,6 +471,64 @@ class ParallelTestSuite(unittest.TestSuite):
|
|||
return iter(self.subsuites)
|
||||
|
||||
|
||||
class Shuffler:
|
||||
"""
|
||||
This class implements shuffling with a special consistency property.
|
||||
Consistency means that, for a given seed and key function, if two sets of
|
||||
items are shuffled, the resulting order will agree on the intersection of
|
||||
the two sets. For example, if items are removed from an original set, the
|
||||
shuffled order for the new set will be the shuffled order of the original
|
||||
set restricted to the smaller set.
|
||||
"""
|
||||
|
||||
# This doesn't need to be cryptographically strong, so use what's fastest.
|
||||
hash_algorithm = 'md5'
|
||||
|
||||
@classmethod
|
||||
def _hash_text(cls, text):
|
||||
h = hashlib.new(cls.hash_algorithm)
|
||||
h.update(text.encode('utf-8'))
|
||||
return h.hexdigest()
|
||||
|
||||
def __init__(self, seed=None):
|
||||
if seed is None:
|
||||
# Limit seeds to 9 digits for simpler output.
|
||||
seed = random.randint(0, 10**10 - 1)
|
||||
seed_source = 'generated'
|
||||
else:
|
||||
seed_source = 'given'
|
||||
self.seed = seed
|
||||
self.seed_source = seed_source
|
||||
|
||||
@property
|
||||
def seed_display(self):
|
||||
return f'{self.seed!r} ({self.seed_source})'
|
||||
|
||||
def _hash_item(self, item, key):
|
||||
text = '{}{}'.format(self.seed, key(item))
|
||||
return self._hash_text(text)
|
||||
|
||||
def shuffle(self, items, key):
|
||||
"""
|
||||
Return a new list of the items in a shuffled order.
|
||||
|
||||
The `key` is a function that accepts an item in `items` and returns
|
||||
a string unique for that item that can be viewed as a string id. The
|
||||
order of the return value is deterministic. It depends on the seed
|
||||
and key function but not on the original order.
|
||||
"""
|
||||
hashes = {}
|
||||
for item in items:
|
||||
hashed = self._hash_item(item, key)
|
||||
if hashed in hashes:
|
||||
msg = 'item {!r} has same hash {!r} as item {!r}'.format(
|
||||
item, hashed, hashes[hashed],
|
||||
)
|
||||
raise RuntimeError(msg)
|
||||
hashes[hashed] = item
|
||||
return [hashes[hashed] for hashed in sorted(hashes)]
|
||||
|
||||
|
||||
class DiscoverRunner:
|
||||
"""A Django test runner that uses unittest2 test discovery."""
|
||||
|
||||
|
|
@ -483,7 +543,7 @@ class DiscoverRunner:
|
|||
reverse=False, debug_mode=False, debug_sql=False, parallel=0,
|
||||
tags=None, exclude_tags=None, test_name_patterns=None,
|
||||
pdb=False, buffer=False, enable_faulthandler=True,
|
||||
timing=False, **kwargs):
|
||||
timing=False, shuffle=False, **kwargs):
|
||||
|
||||
self.pattern = pattern
|
||||
self.top_level = top_level
|
||||
|
|
@ -515,6 +575,8 @@ class DiscoverRunner:
|
|||
pattern if '*' in pattern else '*%s*' % pattern
|
||||
for pattern in test_name_patterns
|
||||
}
|
||||
self.shuffle = shuffle
|
||||
self._shuffler = None
|
||||
|
||||
@classmethod
|
||||
def add_arguments(cls, parser):
|
||||
|
|
@ -530,6 +592,10 @@ class DiscoverRunner:
|
|||
'--keepdb', action='store_true',
|
||||
help='Preserves the test DB between runs.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--shuffle', nargs='?', default=False, type=int, metavar='SEED',
|
||||
help='Shuffles test case order.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'-r', '--reverse', action='store_true',
|
||||
help='Reverses test case order.',
|
||||
|
|
@ -582,6 +648,12 @@ class DiscoverRunner:
|
|||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def shuffle_seed(self):
|
||||
if self._shuffler is None:
|
||||
return None
|
||||
return self._shuffler.seed
|
||||
|
||||
def log(self, msg, level=None):
|
||||
"""
|
||||
Log the given message at the given logging level.
|
||||
|
|
@ -599,6 +671,13 @@ class DiscoverRunner:
|
|||
setup_test_environment(debug=self.debug_mode)
|
||||
unittest.installHandler()
|
||||
|
||||
def setup_shuffler(self):
|
||||
if self.shuffle is False:
|
||||
return
|
||||
shuffler = Shuffler(seed=self.shuffle)
|
||||
self.log(f'Using shuffle seed: {shuffler.seed_display}')
|
||||
self._shuffler = shuffler
|
||||
|
||||
@contextmanager
|
||||
def load_with_patterns(self):
|
||||
original_test_name_patterns = self.test_loader.testNamePatterns
|
||||
|
|
@ -655,6 +734,7 @@ class DiscoverRunner:
|
|||
discover_kwargs['pattern'] = self.pattern
|
||||
if self.top_level is not None:
|
||||
discover_kwargs['top_level_dir'] = self.top_level
|
||||
self.setup_shuffler()
|
||||
|
||||
all_tests = []
|
||||
for label in test_labels:
|
||||
|
|
@ -680,7 +760,12 @@ class DiscoverRunner:
|
|||
# _FailedTest objects include things like test modules that couldn't be
|
||||
# found or that couldn't be loaded due to syntax errors.
|
||||
test_types = (unittest.loader._FailedTest, *self.reorder_by)
|
||||
all_tests = list(reorder_tests(all_tests, test_types, self.reverse))
|
||||
all_tests = list(reorder_tests(
|
||||
all_tests,
|
||||
test_types,
|
||||
shuffler=self._shuffler,
|
||||
reverse=self.reverse,
|
||||
))
|
||||
self.log('Found %d test(s).' % len(all_tests))
|
||||
suite = self.test_suite(all_tests)
|
||||
|
||||
|
|
@ -726,7 +811,12 @@ class DiscoverRunner:
|
|||
def run_suite(self, suite, **kwargs):
|
||||
kwargs = self.get_test_runner_kwargs()
|
||||
runner = self.test_runner(**kwargs)
|
||||
try:
|
||||
return runner.run(suite)
|
||||
finally:
|
||||
if self._shuffler is not None:
|
||||
seed_display = self._shuffler.seed_display
|
||||
self.log(f'Used shuffle seed: {seed_display}')
|
||||
|
||||
def teardown_databases(self, old_config, **kwargs):
|
||||
"""Destroy all the non-mirror databases."""
|
||||
|
|
@ -851,17 +941,64 @@ def find_top_level(top_level):
|
|||
return top_level
|
||||
|
||||
|
||||
def reorder_tests(tests, classes, reverse=False):
|
||||
def _class_shuffle_key(cls):
|
||||
return f'{cls.__module__}.{cls.__qualname__}'
|
||||
|
||||
|
||||
def shuffle_tests(tests, shuffler):
|
||||
"""
|
||||
Return an iterator over the given tests in a shuffled order, keeping tests
|
||||
next to other tests of their class.
|
||||
|
||||
`tests` should be an iterable of tests.
|
||||
"""
|
||||
tests_by_type = {}
|
||||
for _, class_tests in itertools.groupby(tests, type):
|
||||
class_tests = list(class_tests)
|
||||
test_type = type(class_tests[0])
|
||||
class_tests = shuffler.shuffle(class_tests, key=lambda test: test.id())
|
||||
tests_by_type[test_type] = class_tests
|
||||
|
||||
classes = shuffler.shuffle(tests_by_type, key=_class_shuffle_key)
|
||||
|
||||
return itertools.chain(*(tests_by_type[cls] for cls in classes))
|
||||
|
||||
|
||||
def reorder_test_bin(tests, shuffler=None, reverse=False):
|
||||
"""
|
||||
Return an iterator that reorders the given tests, keeping tests next to
|
||||
other tests of their class.
|
||||
|
||||
`tests` should be an iterable of tests that supports reversed().
|
||||
"""
|
||||
if shuffler is None:
|
||||
if reverse:
|
||||
return reversed(tests)
|
||||
# The function must return an iterator.
|
||||
return iter(tests)
|
||||
|
||||
tests = shuffle_tests(tests, shuffler)
|
||||
if not reverse:
|
||||
return tests
|
||||
# Arguments to reversed() must be reversible.
|
||||
return reversed(list(tests))
|
||||
|
||||
|
||||
def reorder_tests(tests, classes, reverse=False, shuffler=None):
|
||||
"""
|
||||
Reorder an iterable of tests by test type, removing any duplicates.
|
||||
|
||||
`classes` is a sequence of types. The result is returned as an iterator.
|
||||
|
||||
The result is returned as an iterator. `classes` is a sequence of types.
|
||||
All tests of type classes[0] are placed first, then tests of type
|
||||
classes[1], etc. Tests with no match in classes are placed last.
|
||||
|
||||
If `reverse` is True, sort tests within classes in opposite order but
|
||||
don't reverse test classes.
|
||||
|
||||
The `shuffler` argument is an optional instance of this module's `Shuffler`
|
||||
class. If provided, tests will be shuffled within each `classes` group, but
|
||||
keeping tests with other tests of their TestCase class. Reversing is
|
||||
applied after shuffling to allow reversing the same random order.
|
||||
"""
|
||||
bins = [OrderedSet() for i in range(len(classes) + 1)]
|
||||
*class_bins, last_bin = bins
|
||||
|
|
@ -874,9 +1011,8 @@ def reorder_tests(tests, classes, reverse=False):
|
|||
test_bin = last_bin
|
||||
test_bin.add(test)
|
||||
|
||||
if reverse:
|
||||
bins = (reversed(tests) for tests in bins)
|
||||
return itertools.chain(*bins)
|
||||
for tests in bins:
|
||||
yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse)
|
||||
|
||||
|
||||
def partition_suite_by_case(suite):
|
||||
|
|
|
|||
|
|
@ -470,12 +470,13 @@ the first one:
|
|||
|
||||
$ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions
|
||||
|
||||
You can also try running any set of tests in reverse using the ``--reverse``
|
||||
option in order to verify that executing tests in a different order does not
|
||||
cause any trouble:
|
||||
You can also try running any set of tests in a random or reverse order using
|
||||
the ``--shuffle`` and ``--reverse`` options. This can help verify that
|
||||
executing tests in a different order does not cause any trouble:
|
||||
|
||||
.. console::
|
||||
|
||||
$ ./runtests.py basic --shuffle
|
||||
$ ./runtests.py basic --reverse
|
||||
|
||||
Seeing the SQL queries run during a test
|
||||
|
|
|
|||
|
|
@ -1425,11 +1425,30 @@ subsequent run. Unless the :setting:`MIGRATE <TEST_MIGRATE>` test setting is
|
|||
``False``, any unapplied migrations will also be applied to the test database
|
||||
before running the test suite.
|
||||
|
||||
.. django-admin-option:: --shuffle [SEED]
|
||||
|
||||
.. versionadded:: 4.0
|
||||
|
||||
Randomizes the order of tests before running them. This can help detect tests
|
||||
that aren't properly isolated. The test order generated by this option is a
|
||||
deterministic function of the integer seed given. When no seed is passed, a
|
||||
seed is chosen randomly and printed to the console. To repeat a particular test
|
||||
order, pass a seed. The test orders generated by this option preserve Django's
|
||||
:ref:`guarantees on test order <order-of-tests>`. They also keep tests grouped
|
||||
by test case class.
|
||||
|
||||
The shuffled orderings also have a special consistency property useful when
|
||||
narrowing down isolation issues. Namely, for a given seed and when running a
|
||||
subset of tests, the new order will be the original shuffling restricted to the
|
||||
smaller set. Similarly, when adding tests while keeping the seed the same, the
|
||||
order of the original tests will be the same in the new order.
|
||||
|
||||
.. django-admin-option:: --reverse, -r
|
||||
|
||||
Sorts test cases in the opposite execution order. This may help in debugging
|
||||
the side effects of tests that aren't properly isolated. :ref:`Grouping by test
|
||||
class <order-of-tests>` is preserved when using this option.
|
||||
class <order-of-tests>` is preserved when using this option. This can be used
|
||||
in conjunction with ``--shuffle`` to reverse the order for a particular seed.
|
||||
|
||||
.. django-admin-option:: --debug-mode
|
||||
|
||||
|
|
|
|||
|
|
@ -325,6 +325,9 @@ Tests
|
|||
* The new :meth:`.DiscoverRunner.log` method allows customizing the way
|
||||
messages are logged.
|
||||
|
||||
* Django test runner now supports a :option:`--shuffle <test --shuffle>` option
|
||||
to execute tests in a random order.
|
||||
|
||||
URLs
|
||||
~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -510,7 +510,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a
|
|||
selection of other methods that are used by ``run_tests()`` to set up, execute
|
||||
and tear down the test suite.
|
||||
|
||||
.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, **kwargs)
|
||||
.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, shuffle=False, **kwargs)
|
||||
|
||||
``DiscoverRunner`` will search for tests in any file matching ``pattern``.
|
||||
|
||||
|
|
@ -539,7 +539,8 @@ and tear down the test suite.
|
|||
If ``reverse`` is ``True``, test cases will be executed in the opposite
|
||||
order. This could be useful to debug tests that aren't properly isolated
|
||||
and have side effects. :ref:`Grouping by test class <order-of-tests>` is
|
||||
preserved when using this option.
|
||||
preserved when using this option. This option can be used in conjunction
|
||||
with ``--shuffle`` to reverse the order for a particular random seed.
|
||||
|
||||
``debug_mode`` specifies what the :setting:`DEBUG` setting should be
|
||||
set to prior to running tests.
|
||||
|
|
@ -576,6 +577,14 @@ and tear down the test suite.
|
|||
If ``timing`` is ``True``, test timings, including database setup and total
|
||||
run time, will be shown.
|
||||
|
||||
If ``shuffle`` is an integer, test cases will be shuffled in a random order
|
||||
prior to execution, using the integer as a random seed. If ``shuffle`` is
|
||||
``None``, the seed will be generated randomly. In both cases, the seed will
|
||||
be logged to the console and set to ``self.shuffle_seed`` prior to running
|
||||
tests. This option can be used to help detect tests that aren't properly
|
||||
isolated. :ref:`Grouping by test class <order-of-tests>` is preserved when
|
||||
using this option.
|
||||
|
||||
Django may, from time to time, extend the capabilities of the test runner
|
||||
by adding new arguments. The ``**kwargs`` declaration allows for this
|
||||
expansion. If you subclass ``DiscoverRunner`` or write your own test
|
||||
|
|
@ -590,6 +599,10 @@ and tear down the test suite.
|
|||
|
||||
The ``enable_faulthandler`` and ``timing`` arguments were added.
|
||||
|
||||
.. versionadded:: 4.0
|
||||
|
||||
The ``shuffle`` argument was added.
|
||||
|
||||
Attributes
|
||||
~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -235,9 +235,9 @@ the Django test runner reorders tests in the following way:
|
|||
for quicker feedback. This includes things like test modules that couldn't
|
||||
be found or that couldn't be loaded due to syntax errors.
|
||||
|
||||
You may reverse the execution order inside groups using the :option:`test
|
||||
--reverse` option. This can help with ensuring your tests are independent from
|
||||
each other.
|
||||
You may randomize and/or reverse the execution order inside groups using the
|
||||
:option:`test --shuffle` and :option:`--reverse <test --reverse>` options. This
|
||||
can help with ensuring your tests are independent from each other.
|
||||
|
||||
.. versionchanged:: 4.0
|
||||
|
||||
|
|
|
|||
|
|
@ -353,7 +353,7 @@ class ActionSelenium(argparse.Action):
|
|||
def django_tests(verbosity, interactive, failfast, keepdb, reverse,
|
||||
test_labels, debug_sql, parallel, tags, exclude_tags,
|
||||
test_name_patterns, start_at, start_after, pdb, buffer,
|
||||
timing):
|
||||
timing, shuffle):
|
||||
if verbosity >= 1:
|
||||
msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__)
|
||||
max_parallel = default_test_processes() if parallel == 0 else parallel
|
||||
|
|
@ -380,6 +380,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse,
|
|||
pdb=pdb,
|
||||
buffer=buffer,
|
||||
timing=timing,
|
||||
shuffle=shuffle,
|
||||
)
|
||||
failures = test_runner.run_tests(test_labels)
|
||||
teardown_run_tests(state)
|
||||
|
|
@ -406,6 +407,11 @@ def get_subprocess_args(options):
|
|||
subprocess_args.append('--tag=%s' % options.tags)
|
||||
if options.exclude_tags:
|
||||
subprocess_args.append('--exclude_tag=%s' % options.exclude_tags)
|
||||
if options.shuffle is not False:
|
||||
if options.shuffle is None:
|
||||
subprocess_args.append('--shuffle')
|
||||
else:
|
||||
subprocess_args.append('--shuffle=%s' % options.shuffle)
|
||||
return subprocess_args
|
||||
|
||||
|
||||
|
|
@ -523,6 +529,13 @@ if __name__ == "__main__":
|
|||
'--pair',
|
||||
help='Run the test suite in pairs with the named test to find problem pairs.',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--shuffle', nargs='?', default=False, type=int, metavar='SEED',
|
||||
help=(
|
||||
'Shuffle the order of test cases to help check that tests are '
|
||||
'properly isolated.'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
'--reverse', action='store_true',
|
||||
help='Sort test suites and test cases in opposite order to debug '
|
||||
|
|
@ -650,7 +663,7 @@ if __name__ == "__main__":
|
|||
options.exclude_tags,
|
||||
getattr(options, 'test_name_patterns', None),
|
||||
options.start_at, options.start_after, options.pdb, options.buffer,
|
||||
options.timing,
|
||||
options.timing, options.shuffle,
|
||||
)
|
||||
time_keeper.print_results()
|
||||
if failures:
|
||||
|
|
|
|||
|
|
@ -49,6 +49,16 @@ class DiscoverRunnerTests(SimpleTestCase):
|
|||
runner = DiscoverRunner()
|
||||
self.assertFalse(runner.debug_mode)
|
||||
|
||||
def test_add_arguments_shuffle(self):
|
||||
parser = ArgumentParser()
|
||||
DiscoverRunner.add_arguments(parser)
|
||||
ns = parser.parse_args([])
|
||||
self.assertIs(ns.shuffle, False)
|
||||
ns = parser.parse_args(['--shuffle'])
|
||||
self.assertIsNone(ns.shuffle)
|
||||
ns = parser.parse_args(['--shuffle', '5'])
|
||||
self.assertEqual(ns.shuffle, 5)
|
||||
|
||||
def test_add_arguments_debug_mode(self):
|
||||
parser = ArgumentParser()
|
||||
DiscoverRunner.add_arguments(parser)
|
||||
|
|
@ -58,6 +68,30 @@ class DiscoverRunnerTests(SimpleTestCase):
|
|||
ns = parser.parse_args(["--debug-mode"])
|
||||
self.assertTrue(ns.debug_mode)
|
||||
|
||||
def test_setup_shuffler_no_shuffle_argument(self):
|
||||
runner = DiscoverRunner()
|
||||
self.assertIs(runner.shuffle, False)
|
||||
runner.setup_shuffler()
|
||||
self.assertIsNone(runner.shuffle_seed)
|
||||
|
||||
def test_setup_shuffler_shuffle_none(self):
|
||||
runner = DiscoverRunner(shuffle=None)
|
||||
self.assertIsNone(runner.shuffle)
|
||||
with mock.patch('random.randint', return_value=1):
|
||||
with captured_stdout() as stdout:
|
||||
runner.setup_shuffler()
|
||||
self.assertEqual(stdout.getvalue(), 'Using shuffle seed: 1 (generated)\n')
|
||||
self.assertEqual(runner.shuffle_seed, 1)
|
||||
|
||||
def test_setup_shuffler_shuffle_int(self):
|
||||
runner = DiscoverRunner(shuffle=2)
|
||||
self.assertEqual(runner.shuffle, 2)
|
||||
with captured_stdout() as stdout:
|
||||
runner.setup_shuffler()
|
||||
expected_out = 'Using shuffle seed: 2 (given)\n'
|
||||
self.assertEqual(stdout.getvalue(), expected_out)
|
||||
self.assertEqual(runner.shuffle_seed, 2)
|
||||
|
||||
def test_load_tests_for_label_file_path(self):
|
||||
with change_cwd('.'):
|
||||
msg = (
|
||||
|
|
@ -266,6 +300,25 @@ class DiscoverRunnerTests(SimpleTestCase):
|
|||
self.assertIsInstance(tests[0], unittest.loader._FailedTest)
|
||||
self.assertNotIsInstance(tests[-1], unittest.loader._FailedTest)
|
||||
|
||||
def test_build_suite_shuffling(self):
|
||||
# These will result in unittest.loader._FailedTest instances rather
|
||||
# than TestCase objects, but they are sufficient for testing.
|
||||
labels = ['label1', 'label2', 'label3', 'label4']
|
||||
cases = [
|
||||
({}, ['label1', 'label2', 'label3', 'label4']),
|
||||
({'reverse': True}, ['label4', 'label3', 'label2', 'label1']),
|
||||
({'shuffle': 8}, ['label4', 'label1', 'label3', 'label2']),
|
||||
({'shuffle': 8, 'reverse': True}, ['label2', 'label3', 'label1', 'label4']),
|
||||
]
|
||||
for kwargs, expected in cases:
|
||||
with self.subTest(kwargs=kwargs):
|
||||
# Prevent writing the seed to stdout.
|
||||
runner = DiscoverRunner(**kwargs, verbosity=0)
|
||||
tests = runner.build_suite(test_labels=labels)
|
||||
# The ids have the form "unittest.loader._FailedTest.label1".
|
||||
names = [test.id().split('.')[-1] for test in tests]
|
||||
self.assertEqual(names, expected)
|
||||
|
||||
def test_overridable_get_test_runner_kwargs(self):
|
||||
self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict)
|
||||
|
||||
|
|
@ -374,6 +427,52 @@ class DiscoverRunnerTests(SimpleTestCase):
|
|||
self.assertIn('Write to stderr.', stderr.getvalue())
|
||||
self.assertIn('Write to stdout.', stdout.getvalue())
|
||||
|
||||
def run_suite_with_runner(self, runner_class, **kwargs):
|
||||
class MyRunner(DiscoverRunner):
|
||||
def test_runner(self, *args, **kwargs):
|
||||
return runner_class()
|
||||
|
||||
runner = MyRunner(**kwargs)
|
||||
# Suppress logging "Using shuffle seed" to the console.
|
||||
with captured_stdout():
|
||||
runner.setup_shuffler()
|
||||
with captured_stdout() as stdout:
|
||||
try:
|
||||
result = runner.run_suite(None)
|
||||
except RuntimeError as exc:
|
||||
result = str(exc)
|
||||
output = stdout.getvalue()
|
||||
return result, output
|
||||
|
||||
def test_run_suite_logs_seed(self):
|
||||
class TestRunner:
|
||||
def run(self, suite):
|
||||
return '<fake-result>'
|
||||
|
||||
expected_prefix = 'Used shuffle seed'
|
||||
# Test with and without shuffling enabled.
|
||||
result, output = self.run_suite_with_runner(TestRunner)
|
||||
self.assertEqual(result, '<fake-result>')
|
||||
self.assertNotIn(expected_prefix, output)
|
||||
|
||||
result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
|
||||
self.assertEqual(result, '<fake-result>')
|
||||
expected_output = f'{expected_prefix}: 2 (given)\n'
|
||||
self.assertEqual(output, expected_output)
|
||||
|
||||
def test_run_suite_logs_seed_exception(self):
|
||||
"""
|
||||
run_suite() logs the seed when TestRunner.run() raises an exception.
|
||||
"""
|
||||
class TestRunner:
|
||||
def run(self, suite):
|
||||
raise RuntimeError('my exception')
|
||||
|
||||
result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
|
||||
self.assertEqual(result, 'my exception')
|
||||
expected_output = 'Used shuffle seed: 2 (given)\n'
|
||||
self.assertEqual(output, expected_output)
|
||||
|
||||
@mock.patch('faulthandler.enable')
|
||||
def test_faulthandler_enabled(self, mocked_enable):
|
||||
with mock.patch('faulthandler.is_enabled', return_value=False):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,102 @@
|
|||
from unittest import mock
|
||||
|
||||
from django.test import SimpleTestCase
|
||||
from django.test.runner import Shuffler
|
||||
|
||||
|
||||
class ShufflerTests(SimpleTestCase):
|
||||
|
||||
def test_hash_text(self):
|
||||
actual = Shuffler._hash_text('abcd')
|
||||
self.assertEqual(actual, 'e2fc714c4727ee9395f324cd2e7f331f')
|
||||
|
||||
def test_hash_text_hash_algorithm(self):
|
||||
class MyShuffler(Shuffler):
|
||||
hash_algorithm = 'sha1'
|
||||
|
||||
actual = MyShuffler._hash_text('abcd')
|
||||
self.assertEqual(actual, '81fe8bfe87576c3ecb22426f8e57847382917acf')
|
||||
|
||||
def test_init(self):
|
||||
shuffler = Shuffler(100)
|
||||
self.assertEqual(shuffler.seed, 100)
|
||||
self.assertEqual(shuffler.seed_source, 'given')
|
||||
|
||||
def test_init_none_seed(self):
|
||||
with mock.patch('random.randint', return_value=200):
|
||||
shuffler = Shuffler(None)
|
||||
self.assertEqual(shuffler.seed, 200)
|
||||
self.assertEqual(shuffler.seed_source, 'generated')
|
||||
|
||||
def test_init_no_seed_argument(self):
|
||||
with mock.patch('random.randint', return_value=300):
|
||||
shuffler = Shuffler()
|
||||
self.assertEqual(shuffler.seed, 300)
|
||||
self.assertEqual(shuffler.seed_source, 'generated')
|
||||
|
||||
def test_seed_display(self):
|
||||
shuffler = Shuffler(100)
|
||||
shuffler.seed_source = 'test'
|
||||
self.assertEqual(shuffler.seed_display, '100 (test)')
|
||||
|
||||
def test_hash_item_seed(self):
|
||||
cases = [
|
||||
(1234, '64ad3fb166ddb41a2ca24f1803b8b722'),
|
||||
# Passing a string gives the same value.
|
||||
('1234', '64ad3fb166ddb41a2ca24f1803b8b722'),
|
||||
(5678, '4dde450ad339b6ce45a0a2666e35b975'),
|
||||
]
|
||||
for seed, expected in cases:
|
||||
with self.subTest(seed=seed):
|
||||
shuffler = Shuffler(seed=seed)
|
||||
actual = shuffler._hash_item('abc', lambda x: x)
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_hash_item_key(self):
|
||||
cases = [
|
||||
(lambda x: x, '64ad3fb166ddb41a2ca24f1803b8b722'),
|
||||
(lambda x: x.upper(), 'ee22e8597bff91742affe4befbf4649a'),
|
||||
]
|
||||
for key, expected in cases:
|
||||
with self.subTest(key=key):
|
||||
shuffler = Shuffler(seed=1234)
|
||||
actual = shuffler._hash_item('abc', key)
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_shuffle_key(self):
|
||||
cases = [
|
||||
(lambda x: x, ['a', 'd', 'b', 'c']),
|
||||
(lambda x: x.upper(), ['d', 'c', 'a', 'b']),
|
||||
]
|
||||
for num, (key, expected) in enumerate(cases, start=1):
|
||||
with self.subTest(num=num):
|
||||
shuffler = Shuffler(seed=1234)
|
||||
actual = shuffler.shuffle(['a', 'b', 'c', 'd'], key)
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_shuffle_consistency(self):
|
||||
seq = [str(n) for n in range(5)]
|
||||
cases = [
|
||||
(None, ['3', '0', '2', '4', '1']),
|
||||
(0, ['3', '2', '4', '1']),
|
||||
(1, ['3', '0', '2', '4']),
|
||||
(2, ['3', '0', '4', '1']),
|
||||
(3, ['0', '2', '4', '1']),
|
||||
(4, ['3', '0', '2', '1']),
|
||||
]
|
||||
shuffler = Shuffler(seed=1234)
|
||||
for index, expected in cases:
|
||||
with self.subTest(index=index):
|
||||
if index is None:
|
||||
new_seq = seq
|
||||
else:
|
||||
new_seq = seq.copy()
|
||||
del new_seq[index]
|
||||
actual = shuffler.shuffle(new_seq, lambda x: x)
|
||||
self.assertEqual(actual, expected)
|
||||
|
||||
def test_shuffle_same_hash(self):
|
||||
shuffler = Shuffler(seed=1234)
|
||||
msg = "item 'A' has same hash 'a56ce89262959e151ee2266552f1819c' as item 'a'"
|
||||
with self.assertRaisesMessage(RuntimeError, msg):
|
||||
shuffler.shuffle(['a', 'b', 'A'], lambda x: x.upper())
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
"""
|
||||
Tests for django test runner
|
||||
"""
|
||||
import collections.abc
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -14,7 +15,9 @@ from django.core.management.base import SystemCheckError
|
|||
from django.test import (
|
||||
SimpleTestCase, TransactionTestCase, skipUnlessDBFeature,
|
||||
)
|
||||
from django.test.runner import DiscoverRunner, reorder_tests
|
||||
from django.test.runner import (
|
||||
DiscoverRunner, Shuffler, reorder_test_bin, reorder_tests, shuffle_tests,
|
||||
)
|
||||
from django.test.testcases import connections_support_transactions
|
||||
from django.test.utils import (
|
||||
captured_stderr, dependency_ordered, get_unique_databases_and_mirrors,
|
||||
|
|
@ -126,6 +129,68 @@ class TestSuiteTests(SimpleTestCase):
|
|||
self.assertEqual(len(tests), 4)
|
||||
self.assertNotIsInstance(tests[0], unittest.TestSuite)
|
||||
|
||||
def make_tests(self):
|
||||
"""Return an iterable of tests."""
|
||||
suite = self.make_test_suite()
|
||||
tests = list(iter_test_cases(suite))
|
||||
return tests
|
||||
|
||||
def test_shuffle_tests(self):
|
||||
tests = self.make_tests()
|
||||
# Choose a seed that shuffles both the classes and methods.
|
||||
shuffler = Shuffler(seed=9)
|
||||
shuffled_tests = shuffle_tests(tests, shuffler)
|
||||
self.assertIsInstance(shuffled_tests, collections.abc.Iterator)
|
||||
self.assertTestNames(shuffled_tests, expected=[
|
||||
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
|
||||
])
|
||||
|
||||
def test_reorder_test_bin_no_arguments(self):
|
||||
tests = self.make_tests()
|
||||
reordered_tests = reorder_test_bin(tests)
|
||||
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
|
||||
self.assertTestNames(reordered_tests, expected=[
|
||||
'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
|
||||
])
|
||||
|
||||
def test_reorder_test_bin_reverse(self):
|
||||
tests = self.make_tests()
|
||||
reordered_tests = reorder_test_bin(tests, reverse=True)
|
||||
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
|
||||
self.assertTestNames(reordered_tests, expected=[
|
||||
'Tests2.test2', 'Tests2.test1', 'Tests1.test2', 'Tests1.test1',
|
||||
])
|
||||
|
||||
def test_reorder_test_bin_random(self):
|
||||
tests = self.make_tests()
|
||||
# Choose a seed that shuffles both the classes and methods.
|
||||
shuffler = Shuffler(seed=9)
|
||||
reordered_tests = reorder_test_bin(tests, shuffler=shuffler)
|
||||
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
|
||||
self.assertTestNames(reordered_tests, expected=[
|
||||
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
|
||||
])
|
||||
|
||||
def test_reorder_test_bin_random_and_reverse(self):
|
||||
tests = self.make_tests()
|
||||
# Choose a seed that shuffles both the classes and methods.
|
||||
shuffler = Shuffler(seed=9)
|
||||
reordered_tests = reorder_test_bin(tests, shuffler=shuffler, reverse=True)
|
||||
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
|
||||
self.assertTestNames(reordered_tests, expected=[
|
||||
'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1',
|
||||
])
|
||||
|
||||
def test_reorder_tests_random(self):
|
||||
tests = self.make_tests()
|
||||
# Choose a seed that shuffles both the classes and methods.
|
||||
shuffler = Shuffler(seed=9)
|
||||
reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
|
||||
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
|
||||
self.assertTestNames(reordered_tests, expected=[
|
||||
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
|
||||
])
|
||||
|
||||
def test_reorder_tests_reverse_with_duplicates(self):
|
||||
class Tests1(unittest.TestCase):
|
||||
def test1(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue