From 90ba716bf060ee7fef79dc230b0b20644839069f Mon Sep 17 00:00:00 2001 From: Chris Jerdonek Date: Sat, 24 Apr 2021 16:46:16 -0700 Subject: [PATCH] Fixed #24522 -- Added a --shuffle option to DiscoverRunner. --- django/test/runner.py | 154 +++++++++++++++++- .../contributing/writing-code/unit-tests.txt | 7 +- docs/ref/django-admin.txt | 21 ++- docs/releases/4.0.txt | 3 + docs/topics/testing/advanced.txt | 17 +- docs/topics/testing/overview.txt | 6 +- tests/runtests.py | 17 +- tests/test_runner/test_discover_runner.py | 99 +++++++++++ tests/test_runner/test_shuffler.py | 102 ++++++++++++ tests/test_runner/tests.py | 67 +++++++- 10 files changed, 472 insertions(+), 21 deletions(-) create mode 100644 tests/test_runner/test_shuffler.py diff --git a/django/test/runner.py b/django/test/runner.py index ee570b08922..ab6538b61df 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -1,11 +1,13 @@ import ctypes import faulthandler +import hashlib import io import itertools import logging import multiprocessing import os import pickle +import random import sys import textwrap import unittest @@ -469,6 +471,64 @@ class ParallelTestSuite(unittest.TestSuite): return iter(self.subsuites) +class Shuffler: + """ + This class implements shuffling with a special consistency property. + Consistency means that, for a given seed and key function, if two sets of + items are shuffled, the resulting order will agree on the intersection of + the two sets. For example, if items are removed from an original set, the + shuffled order for the new set will be the shuffled order of the original + set restricted to the smaller set. + """ + + # This doesn't need to be cryptographically strong, so use what's fastest. + hash_algorithm = 'md5' + + @classmethod + def _hash_text(cls, text): + h = hashlib.new(cls.hash_algorithm) + h.update(text.encode('utf-8')) + return h.hexdigest() + + def __init__(self, seed=None): + if seed is None: + # Limit seeds to 9 digits for simpler output. + seed = random.randint(0, 10**10 - 1) + seed_source = 'generated' + else: + seed_source = 'given' + self.seed = seed + self.seed_source = seed_source + + @property + def seed_display(self): + return f'{self.seed!r} ({self.seed_source})' + + def _hash_item(self, item, key): + text = '{}{}'.format(self.seed, key(item)) + return self._hash_text(text) + + def shuffle(self, items, key): + """ + Return a new list of the items in a shuffled order. + + The `key` is a function that accepts an item in `items` and returns + a string unique for that item that can be viewed as a string id. The + order of the return value is deterministic. It depends on the seed + and key function but not on the original order. + """ + hashes = {} + for item in items: + hashed = self._hash_item(item, key) + if hashed in hashes: + msg = 'item {!r} has same hash {!r} as item {!r}'.format( + item, hashed, hashes[hashed], + ) + raise RuntimeError(msg) + hashes[hashed] = item + return [hashes[hashed] for hashed in sorted(hashes)] + + class DiscoverRunner: """A Django test runner that uses unittest2 test discovery.""" @@ -483,7 +543,7 @@ class DiscoverRunner: reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, - timing=False, **kwargs): + timing=False, shuffle=False, **kwargs): self.pattern = pattern self.top_level = top_level @@ -515,6 +575,8 @@ class DiscoverRunner: pattern if '*' in pattern else '*%s*' % pattern for pattern in test_name_patterns } + self.shuffle = shuffle + self._shuffler = None @classmethod def add_arguments(cls, parser): @@ -530,6 +592,10 @@ class DiscoverRunner: '--keepdb', action='store_true', help='Preserves the test DB between runs.' ) + parser.add_argument( + '--shuffle', nargs='?', default=False, type=int, metavar='SEED', + help='Shuffles test case order.', + ) parser.add_argument( '-r', '--reverse', action='store_true', help='Reverses test case order.', @@ -582,6 +648,12 @@ class DiscoverRunner: ), ) + @property + def shuffle_seed(self): + if self._shuffler is None: + return None + return self._shuffler.seed + def log(self, msg, level=None): """ Log the given message at the given logging level. @@ -599,6 +671,13 @@ class DiscoverRunner: setup_test_environment(debug=self.debug_mode) unittest.installHandler() + def setup_shuffler(self): + if self.shuffle is False: + return + shuffler = Shuffler(seed=self.shuffle) + self.log(f'Using shuffle seed: {shuffler.seed_display}') + self._shuffler = shuffler + @contextmanager def load_with_patterns(self): original_test_name_patterns = self.test_loader.testNamePatterns @@ -655,6 +734,7 @@ class DiscoverRunner: discover_kwargs['pattern'] = self.pattern if self.top_level is not None: discover_kwargs['top_level_dir'] = self.top_level + self.setup_shuffler() all_tests = [] for label in test_labels: @@ -680,7 +760,12 @@ class DiscoverRunner: # _FailedTest objects include things like test modules that couldn't be # found or that couldn't be loaded due to syntax errors. test_types = (unittest.loader._FailedTest, *self.reorder_by) - all_tests = list(reorder_tests(all_tests, test_types, self.reverse)) + all_tests = list(reorder_tests( + all_tests, + test_types, + shuffler=self._shuffler, + reverse=self.reverse, + )) self.log('Found %d test(s).' % len(all_tests)) suite = self.test_suite(all_tests) @@ -726,7 +811,12 @@ class DiscoverRunner: def run_suite(self, suite, **kwargs): kwargs = self.get_test_runner_kwargs() runner = self.test_runner(**kwargs) - return runner.run(suite) + try: + return runner.run(suite) + finally: + if self._shuffler is not None: + seed_display = self._shuffler.seed_display + self.log(f'Used shuffle seed: {seed_display}') def teardown_databases(self, old_config, **kwargs): """Destroy all the non-mirror databases.""" @@ -851,17 +941,64 @@ def find_top_level(top_level): return top_level -def reorder_tests(tests, classes, reverse=False): +def _class_shuffle_key(cls): + return f'{cls.__module__}.{cls.__qualname__}' + + +def shuffle_tests(tests, shuffler): + """ + Return an iterator over the given tests in a shuffled order, keeping tests + next to other tests of their class. + + `tests` should be an iterable of tests. + """ + tests_by_type = {} + for _, class_tests in itertools.groupby(tests, type): + class_tests = list(class_tests) + test_type = type(class_tests[0]) + class_tests = shuffler.shuffle(class_tests, key=lambda test: test.id()) + tests_by_type[test_type] = class_tests + + classes = shuffler.shuffle(tests_by_type, key=_class_shuffle_key) + + return itertools.chain(*(tests_by_type[cls] for cls in classes)) + + +def reorder_test_bin(tests, shuffler=None, reverse=False): + """ + Return an iterator that reorders the given tests, keeping tests next to + other tests of their class. + + `tests` should be an iterable of tests that supports reversed(). + """ + if shuffler is None: + if reverse: + return reversed(tests) + # The function must return an iterator. + return iter(tests) + + tests = shuffle_tests(tests, shuffler) + if not reverse: + return tests + # Arguments to reversed() must be reversible. + return reversed(list(tests)) + + +def reorder_tests(tests, classes, reverse=False, shuffler=None): """ Reorder an iterable of tests by test type, removing any duplicates. - `classes` is a sequence of types. The result is returned as an iterator. - + The result is returned as an iterator. `classes` is a sequence of types. All tests of type classes[0] are placed first, then tests of type classes[1], etc. Tests with no match in classes are placed last. If `reverse` is True, sort tests within classes in opposite order but don't reverse test classes. + + The `shuffler` argument is an optional instance of this module's `Shuffler` + class. If provided, tests will be shuffled within each `classes` group, but + keeping tests with other tests of their TestCase class. Reversing is + applied after shuffling to allow reversing the same random order. """ bins = [OrderedSet() for i in range(len(classes) + 1)] *class_bins, last_bin = bins @@ -874,9 +1011,8 @@ def reorder_tests(tests, classes, reverse=False): test_bin = last_bin test_bin.add(test) - if reverse: - bins = (reversed(tests) for tests in bins) - return itertools.chain(*bins) + for tests in bins: + yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse) def partition_suite_by_case(suite): diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index d2d4044e7ea..1d38d4f4295 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -470,12 +470,13 @@ the first one: $ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions -You can also try running any set of tests in reverse using the ``--reverse`` -option in order to verify that executing tests in a different order does not -cause any trouble: +You can also try running any set of tests in a random or reverse order using +the ``--shuffle`` and ``--reverse`` options. This can help verify that +executing tests in a different order does not cause any trouble: .. console:: + $ ./runtests.py basic --shuffle $ ./runtests.py basic --reverse Seeing the SQL queries run during a test diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 078f019de30..89d945500db 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -1425,11 +1425,30 @@ subsequent run. Unless the :setting:`MIGRATE ` test setting is ``False``, any unapplied migrations will also be applied to the test database before running the test suite. +.. django-admin-option:: --shuffle [SEED] + +.. versionadded:: 4.0 + +Randomizes the order of tests before running them. This can help detect tests +that aren't properly isolated. The test order generated by this option is a +deterministic function of the integer seed given. When no seed is passed, a +seed is chosen randomly and printed to the console. To repeat a particular test +order, pass a seed. The test orders generated by this option preserve Django's +:ref:`guarantees on test order `. They also keep tests grouped +by test case class. + +The shuffled orderings also have a special consistency property useful when +narrowing down isolation issues. Namely, for a given seed and when running a +subset of tests, the new order will be the original shuffling restricted to the +smaller set. Similarly, when adding tests while keeping the seed the same, the +order of the original tests will be the same in the new order. + .. django-admin-option:: --reverse, -r Sorts test cases in the opposite execution order. This may help in debugging the side effects of tests that aren't properly isolated. :ref:`Grouping by test -class ` is preserved when using this option. +class ` is preserved when using this option. This can be used +in conjunction with ``--shuffle`` to reverse the order for a particular seed. .. django-admin-option:: --debug-mode diff --git a/docs/releases/4.0.txt b/docs/releases/4.0.txt index 4c0efc236fe..3ef008e5fea 100644 --- a/docs/releases/4.0.txt +++ b/docs/releases/4.0.txt @@ -325,6 +325,9 @@ Tests * The new :meth:`.DiscoverRunner.log` method allows customizing the way messages are logged. +* Django test runner now supports a :option:`--shuffle ` option + to execute tests in a random order. + URLs ~~~~ diff --git a/docs/topics/testing/advanced.txt b/docs/topics/testing/advanced.txt index a8a63e7b57d..dcc1be7f44f 100644 --- a/docs/topics/testing/advanced.txt +++ b/docs/topics/testing/advanced.txt @@ -510,7 +510,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a selection of other methods that are used by ``run_tests()`` to set up, execute and tear down the test suite. -.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, **kwargs) +.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, shuffle=False, **kwargs) ``DiscoverRunner`` will search for tests in any file matching ``pattern``. @@ -539,7 +539,8 @@ and tear down the test suite. If ``reverse`` is ``True``, test cases will be executed in the opposite order. This could be useful to debug tests that aren't properly isolated and have side effects. :ref:`Grouping by test class ` is - preserved when using this option. + preserved when using this option. This option can be used in conjunction + with ``--shuffle`` to reverse the order for a particular random seed. ``debug_mode`` specifies what the :setting:`DEBUG` setting should be set to prior to running tests. @@ -576,6 +577,14 @@ and tear down the test suite. If ``timing`` is ``True``, test timings, including database setup and total run time, will be shown. + If ``shuffle`` is an integer, test cases will be shuffled in a random order + prior to execution, using the integer as a random seed. If ``shuffle`` is + ``None``, the seed will be generated randomly. In both cases, the seed will + be logged to the console and set to ``self.shuffle_seed`` prior to running + tests. This option can be used to help detect tests that aren't properly + isolated. :ref:`Grouping by test class ` is preserved when + using this option. + Django may, from time to time, extend the capabilities of the test runner by adding new arguments. The ``**kwargs`` declaration allows for this expansion. If you subclass ``DiscoverRunner`` or write your own test @@ -590,6 +599,10 @@ and tear down the test suite. The ``enable_faulthandler`` and ``timing`` arguments were added. + .. versionadded:: 4.0 + + The ``shuffle`` argument was added. + Attributes ~~~~~~~~~~ diff --git a/docs/topics/testing/overview.txt b/docs/topics/testing/overview.txt index 7461fdaaf01..cc552247617 100644 --- a/docs/topics/testing/overview.txt +++ b/docs/topics/testing/overview.txt @@ -235,9 +235,9 @@ the Django test runner reorders tests in the following way: for quicker feedback. This includes things like test modules that couldn't be found or that couldn't be loaded due to syntax errors. -You may reverse the execution order inside groups using the :option:`test ---reverse` option. This can help with ensuring your tests are independent from -each other. +You may randomize and/or reverse the execution order inside groups using the +:option:`test --shuffle` and :option:`--reverse ` options. This +can help with ensuring your tests are independent from each other. .. versionchanged:: 4.0 diff --git a/tests/runtests.py b/tests/runtests.py index 2a9adbbc703..648ac27e052 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -353,7 +353,7 @@ class ActionSelenium(argparse.Action): def django_tests(verbosity, interactive, failfast, keepdb, reverse, test_labels, debug_sql, parallel, tags, exclude_tags, test_name_patterns, start_at, start_after, pdb, buffer, - timing): + timing, shuffle): if verbosity >= 1: msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__) max_parallel = default_test_processes() if parallel == 0 else parallel @@ -380,6 +380,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse, pdb=pdb, buffer=buffer, timing=timing, + shuffle=shuffle, ) failures = test_runner.run_tests(test_labels) teardown_run_tests(state) @@ -406,6 +407,11 @@ def get_subprocess_args(options): subprocess_args.append('--tag=%s' % options.tags) if options.exclude_tags: subprocess_args.append('--exclude_tag=%s' % options.exclude_tags) + if options.shuffle is not False: + if options.shuffle is None: + subprocess_args.append('--shuffle') + else: + subprocess_args.append('--shuffle=%s' % options.shuffle) return subprocess_args @@ -523,6 +529,13 @@ if __name__ == "__main__": '--pair', help='Run the test suite in pairs with the named test to find problem pairs.', ) + parser.add_argument( + '--shuffle', nargs='?', default=False, type=int, metavar='SEED', + help=( + 'Shuffle the order of test cases to help check that tests are ' + 'properly isolated.' + ), + ) parser.add_argument( '--reverse', action='store_true', help='Sort test suites and test cases in opposite order to debug ' @@ -650,7 +663,7 @@ if __name__ == "__main__": options.exclude_tags, getattr(options, 'test_name_patterns', None), options.start_at, options.start_after, options.pdb, options.buffer, - options.timing, + options.timing, options.shuffle, ) time_keeper.print_results() if failures: diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py index 4f7aed059f1..2fc4df31333 100644 --- a/tests/test_runner/test_discover_runner.py +++ b/tests/test_runner/test_discover_runner.py @@ -49,6 +49,16 @@ class DiscoverRunnerTests(SimpleTestCase): runner = DiscoverRunner() self.assertFalse(runner.debug_mode) + def test_add_arguments_shuffle(self): + parser = ArgumentParser() + DiscoverRunner.add_arguments(parser) + ns = parser.parse_args([]) + self.assertIs(ns.shuffle, False) + ns = parser.parse_args(['--shuffle']) + self.assertIsNone(ns.shuffle) + ns = parser.parse_args(['--shuffle', '5']) + self.assertEqual(ns.shuffle, 5) + def test_add_arguments_debug_mode(self): parser = ArgumentParser() DiscoverRunner.add_arguments(parser) @@ -58,6 +68,30 @@ class DiscoverRunnerTests(SimpleTestCase): ns = parser.parse_args(["--debug-mode"]) self.assertTrue(ns.debug_mode) + def test_setup_shuffler_no_shuffle_argument(self): + runner = DiscoverRunner() + self.assertIs(runner.shuffle, False) + runner.setup_shuffler() + self.assertIsNone(runner.shuffle_seed) + + def test_setup_shuffler_shuffle_none(self): + runner = DiscoverRunner(shuffle=None) + self.assertIsNone(runner.shuffle) + with mock.patch('random.randint', return_value=1): + with captured_stdout() as stdout: + runner.setup_shuffler() + self.assertEqual(stdout.getvalue(), 'Using shuffle seed: 1 (generated)\n') + self.assertEqual(runner.shuffle_seed, 1) + + def test_setup_shuffler_shuffle_int(self): + runner = DiscoverRunner(shuffle=2) + self.assertEqual(runner.shuffle, 2) + with captured_stdout() as stdout: + runner.setup_shuffler() + expected_out = 'Using shuffle seed: 2 (given)\n' + self.assertEqual(stdout.getvalue(), expected_out) + self.assertEqual(runner.shuffle_seed, 2) + def test_load_tests_for_label_file_path(self): with change_cwd('.'): msg = ( @@ -266,6 +300,25 @@ class DiscoverRunnerTests(SimpleTestCase): self.assertIsInstance(tests[0], unittest.loader._FailedTest) self.assertNotIsInstance(tests[-1], unittest.loader._FailedTest) + def test_build_suite_shuffling(self): + # These will result in unittest.loader._FailedTest instances rather + # than TestCase objects, but they are sufficient for testing. + labels = ['label1', 'label2', 'label3', 'label4'] + cases = [ + ({}, ['label1', 'label2', 'label3', 'label4']), + ({'reverse': True}, ['label4', 'label3', 'label2', 'label1']), + ({'shuffle': 8}, ['label4', 'label1', 'label3', 'label2']), + ({'shuffle': 8, 'reverse': True}, ['label2', 'label3', 'label1', 'label4']), + ] + for kwargs, expected in cases: + with self.subTest(kwargs=kwargs): + # Prevent writing the seed to stdout. + runner = DiscoverRunner(**kwargs, verbosity=0) + tests = runner.build_suite(test_labels=labels) + # The ids have the form "unittest.loader._FailedTest.label1". + names = [test.id().split('.')[-1] for test in tests] + self.assertEqual(names, expected) + def test_overridable_get_test_runner_kwargs(self): self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict) @@ -374,6 +427,52 @@ class DiscoverRunnerTests(SimpleTestCase): self.assertIn('Write to stderr.', stderr.getvalue()) self.assertIn('Write to stdout.', stdout.getvalue()) + def run_suite_with_runner(self, runner_class, **kwargs): + class MyRunner(DiscoverRunner): + def test_runner(self, *args, **kwargs): + return runner_class() + + runner = MyRunner(**kwargs) + # Suppress logging "Using shuffle seed" to the console. + with captured_stdout(): + runner.setup_shuffler() + with captured_stdout() as stdout: + try: + result = runner.run_suite(None) + except RuntimeError as exc: + result = str(exc) + output = stdout.getvalue() + return result, output + + def test_run_suite_logs_seed(self): + class TestRunner: + def run(self, suite): + return '' + + expected_prefix = 'Used shuffle seed' + # Test with and without shuffling enabled. + result, output = self.run_suite_with_runner(TestRunner) + self.assertEqual(result, '') + self.assertNotIn(expected_prefix, output) + + result, output = self.run_suite_with_runner(TestRunner, shuffle=2) + self.assertEqual(result, '') + expected_output = f'{expected_prefix}: 2 (given)\n' + self.assertEqual(output, expected_output) + + def test_run_suite_logs_seed_exception(self): + """ + run_suite() logs the seed when TestRunner.run() raises an exception. + """ + class TestRunner: + def run(self, suite): + raise RuntimeError('my exception') + + result, output = self.run_suite_with_runner(TestRunner, shuffle=2) + self.assertEqual(result, 'my exception') + expected_output = 'Used shuffle seed: 2 (given)\n' + self.assertEqual(output, expected_output) + @mock.patch('faulthandler.enable') def test_faulthandler_enabled(self, mocked_enable): with mock.patch('faulthandler.is_enabled', return_value=False): diff --git a/tests/test_runner/test_shuffler.py b/tests/test_runner/test_shuffler.py new file mode 100644 index 00000000000..f1772497069 --- /dev/null +++ b/tests/test_runner/test_shuffler.py @@ -0,0 +1,102 @@ +from unittest import mock + +from django.test import SimpleTestCase +from django.test.runner import Shuffler + + +class ShufflerTests(SimpleTestCase): + + def test_hash_text(self): + actual = Shuffler._hash_text('abcd') + self.assertEqual(actual, 'e2fc714c4727ee9395f324cd2e7f331f') + + def test_hash_text_hash_algorithm(self): + class MyShuffler(Shuffler): + hash_algorithm = 'sha1' + + actual = MyShuffler._hash_text('abcd') + self.assertEqual(actual, '81fe8bfe87576c3ecb22426f8e57847382917acf') + + def test_init(self): + shuffler = Shuffler(100) + self.assertEqual(shuffler.seed, 100) + self.assertEqual(shuffler.seed_source, 'given') + + def test_init_none_seed(self): + with mock.patch('random.randint', return_value=200): + shuffler = Shuffler(None) + self.assertEqual(shuffler.seed, 200) + self.assertEqual(shuffler.seed_source, 'generated') + + def test_init_no_seed_argument(self): + with mock.patch('random.randint', return_value=300): + shuffler = Shuffler() + self.assertEqual(shuffler.seed, 300) + self.assertEqual(shuffler.seed_source, 'generated') + + def test_seed_display(self): + shuffler = Shuffler(100) + shuffler.seed_source = 'test' + self.assertEqual(shuffler.seed_display, '100 (test)') + + def test_hash_item_seed(self): + cases = [ + (1234, '64ad3fb166ddb41a2ca24f1803b8b722'), + # Passing a string gives the same value. + ('1234', '64ad3fb166ddb41a2ca24f1803b8b722'), + (5678, '4dde450ad339b6ce45a0a2666e35b975'), + ] + for seed, expected in cases: + with self.subTest(seed=seed): + shuffler = Shuffler(seed=seed) + actual = shuffler._hash_item('abc', lambda x: x) + self.assertEqual(actual, expected) + + def test_hash_item_key(self): + cases = [ + (lambda x: x, '64ad3fb166ddb41a2ca24f1803b8b722'), + (lambda x: x.upper(), 'ee22e8597bff91742affe4befbf4649a'), + ] + for key, expected in cases: + with self.subTest(key=key): + shuffler = Shuffler(seed=1234) + actual = shuffler._hash_item('abc', key) + self.assertEqual(actual, expected) + + def test_shuffle_key(self): + cases = [ + (lambda x: x, ['a', 'd', 'b', 'c']), + (lambda x: x.upper(), ['d', 'c', 'a', 'b']), + ] + for num, (key, expected) in enumerate(cases, start=1): + with self.subTest(num=num): + shuffler = Shuffler(seed=1234) + actual = shuffler.shuffle(['a', 'b', 'c', 'd'], key) + self.assertEqual(actual, expected) + + def test_shuffle_consistency(self): + seq = [str(n) for n in range(5)] + cases = [ + (None, ['3', '0', '2', '4', '1']), + (0, ['3', '2', '4', '1']), + (1, ['3', '0', '2', '4']), + (2, ['3', '0', '4', '1']), + (3, ['0', '2', '4', '1']), + (4, ['3', '0', '2', '1']), + ] + shuffler = Shuffler(seed=1234) + for index, expected in cases: + with self.subTest(index=index): + if index is None: + new_seq = seq + else: + new_seq = seq.copy() + del new_seq[index] + actual = shuffler.shuffle(new_seq, lambda x: x) + self.assertEqual(actual, expected) + + def test_shuffle_same_hash(self): + shuffler = Shuffler(seed=1234) + msg = "item 'A' has same hash 'a56ce89262959e151ee2266552f1819c' as item 'a'" + with self.assertRaisesMessage(RuntimeError, msg): + shuffler.shuffle(['a', 'b', 'A'], lambda x: x.upper()) diff --git a/tests/test_runner/tests.py b/tests/test_runner/tests.py index 6348569dfc5..77a756a1ded 100644 --- a/tests/test_runner/tests.py +++ b/tests/test_runner/tests.py @@ -1,6 +1,7 @@ """ Tests for django test runner """ +import collections.abc import unittest from unittest import mock @@ -14,7 +15,9 @@ from django.core.management.base import SystemCheckError from django.test import ( SimpleTestCase, TransactionTestCase, skipUnlessDBFeature, ) -from django.test.runner import DiscoverRunner, reorder_tests +from django.test.runner import ( + DiscoverRunner, Shuffler, reorder_test_bin, reorder_tests, shuffle_tests, +) from django.test.testcases import connections_support_transactions from django.test.utils import ( captured_stderr, dependency_ordered, get_unique_databases_and_mirrors, @@ -126,6 +129,68 @@ class TestSuiteTests(SimpleTestCase): self.assertEqual(len(tests), 4) self.assertNotIsInstance(tests[0], unittest.TestSuite) + def make_tests(self): + """Return an iterable of tests.""" + suite = self.make_test_suite() + tests = list(iter_test_cases(suite)) + return tests + + def test_shuffle_tests(self): + tests = self.make_tests() + # Choose a seed that shuffles both the classes and methods. + shuffler = Shuffler(seed=9) + shuffled_tests = shuffle_tests(tests, shuffler) + self.assertIsInstance(shuffled_tests, collections.abc.Iterator) + self.assertTestNames(shuffled_tests, expected=[ + 'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1', + ]) + + def test_reorder_test_bin_no_arguments(self): + tests = self.make_tests() + reordered_tests = reorder_test_bin(tests) + self.assertIsInstance(reordered_tests, collections.abc.Iterator) + self.assertTestNames(reordered_tests, expected=[ + 'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2', + ]) + + def test_reorder_test_bin_reverse(self): + tests = self.make_tests() + reordered_tests = reorder_test_bin(tests, reverse=True) + self.assertIsInstance(reordered_tests, collections.abc.Iterator) + self.assertTestNames(reordered_tests, expected=[ + 'Tests2.test2', 'Tests2.test1', 'Tests1.test2', 'Tests1.test1', + ]) + + def test_reorder_test_bin_random(self): + tests = self.make_tests() + # Choose a seed that shuffles both the classes and methods. + shuffler = Shuffler(seed=9) + reordered_tests = reorder_test_bin(tests, shuffler=shuffler) + self.assertIsInstance(reordered_tests, collections.abc.Iterator) + self.assertTestNames(reordered_tests, expected=[ + 'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1', + ]) + + def test_reorder_test_bin_random_and_reverse(self): + tests = self.make_tests() + # Choose a seed that shuffles both the classes and methods. + shuffler = Shuffler(seed=9) + reordered_tests = reorder_test_bin(tests, shuffler=shuffler, reverse=True) + self.assertIsInstance(reordered_tests, collections.abc.Iterator) + self.assertTestNames(reordered_tests, expected=[ + 'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1', + ]) + + def test_reorder_tests_random(self): + tests = self.make_tests() + # Choose a seed that shuffles both the classes and methods. + shuffler = Shuffler(seed=9) + reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler) + self.assertIsInstance(reordered_tests, collections.abc.Iterator) + self.assertTestNames(reordered_tests, expected=[ + 'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1', + ]) + def test_reorder_tests_reverse_with_duplicates(self): class Tests1(unittest.TestCase): def test1(self):