Fixed #24522 -- Added a --shuffle option to DiscoverRunner.

django/test/runner.py

@@ -1,11 +1,13 @@
 import ctypes
 import faulthandler
+import hashlib
 import io
 import itertools
 import logging
 import multiprocessing
 import os
 import pickle
+import random
 import sys
 import textwrap
 import unittest
@@ -469,6 +471,64 @@ class ParallelTestSuite(unittest.TestSuite):
         return iter(self.subsuites)
 
 
+class Shuffler:
+    """
+    This class implements shuffling with a special consistency property.
+    Consistency means that, for a given seed and key function, if two sets of
+    items are shuffled, the resulting order will agree on the intersection of
+    the two sets. For example, if items are removed from an original set, the
+    shuffled order for the new set will be the shuffled order of the original
+    set restricted to the smaller set.
+    """
+
+    # This doesn't need to be cryptographically strong, so use what's fastest.
+    hash_algorithm = 'md5'
+
+    @classmethod
+    def _hash_text(cls, text):
+        h = hashlib.new(cls.hash_algorithm)
+        h.update(text.encode('utf-8'))
+        return h.hexdigest()
+
+    def __init__(self, seed=None):
+        if seed is None:
+            # Limit seeds to 9 digits for simpler output.
+            seed = random.randint(0, 10**10 - 1)
+            seed_source = 'generated'
+        else:
+            seed_source = 'given'
+        self.seed = seed
+        self.seed_source = seed_source
+
+    @property
+    def seed_display(self):
+        return f'{self.seed!r} ({self.seed_source})'
+
+    def _hash_item(self, item, key):
+        text = '{}{}'.format(self.seed, key(item))
+        return self._hash_text(text)
+
+    def shuffle(self, items, key):
+        """
+        Return a new list of the items in a shuffled order.
+
+        The `key` is a function that accepts an item in `items` and returns
+        a string unique for that item that can be viewed as a string id. The
+        order of the return value is deterministic. It depends on the seed
+        and key function but not on the original order.
+        """
+        hashes = {}
+        for item in items:
+            hashed = self._hash_item(item, key)
+            if hashed in hashes:
+                msg = 'item {!r} has same hash {!r} as item {!r}'.format(
+                    item, hashed, hashes[hashed],
+                )
+                raise RuntimeError(msg)
+            hashes[hashed] = item
+        return [hashes[hashed] for hashed in sorted(hashes)]
+
+
 class DiscoverRunner:
     """A Django test runner that uses unittest2 test discovery."""
 
@@ -483,7 +543,7 @@ class DiscoverRunner:
                  reverse=False, debug_mode=False, debug_sql=False, parallel=0,
                  tags=None, exclude_tags=None, test_name_patterns=None,
                  pdb=False, buffer=False, enable_faulthandler=True,
-                 timing=False, **kwargs):
+                 timing=False, shuffle=False, **kwargs):
 
         self.pattern = pattern
         self.top_level = top_level
@@ -515,6 +575,8 @@ class DiscoverRunner:
                 pattern if '*' in pattern else '*%s*' % pattern
                 for pattern in test_name_patterns
             }
+        self.shuffle = shuffle
+        self._shuffler = None
 
     @classmethod
     def add_arguments(cls, parser):
@@ -530,6 +592,10 @@ class DiscoverRunner:
             '--keepdb', action='store_true',
             help='Preserves the test DB between runs.'
         )
+        parser.add_argument(
+            '--shuffle', nargs='?', default=False, type=int, metavar='SEED',
+            help='Shuffles test case order.',
+        )
         parser.add_argument(
             '-r', '--reverse', action='store_true',
             help='Reverses test case order.',
@@ -582,6 +648,12 @@ class DiscoverRunner:
             ),
         )
 
+    @property
+    def shuffle_seed(self):
+        if self._shuffler is None:
+            return None
+        return self._shuffler.seed
+
     def log(self, msg, level=None):
         """
         Log the given message at the given logging level.
@@ -599,6 +671,13 @@ class DiscoverRunner:
         setup_test_environment(debug=self.debug_mode)
         unittest.installHandler()
 
+    def setup_shuffler(self):
+        if self.shuffle is False:
+            return
+        shuffler = Shuffler(seed=self.shuffle)
+        self.log(f'Using shuffle seed: {shuffler.seed_display}')
+        self._shuffler = shuffler
+
     @contextmanager
     def load_with_patterns(self):
         original_test_name_patterns = self.test_loader.testNamePatterns
@@ -655,6 +734,7 @@ class DiscoverRunner:
             discover_kwargs['pattern'] = self.pattern
         if self.top_level is not None:
             discover_kwargs['top_level_dir'] = self.top_level
+        self.setup_shuffler()
 
         all_tests = []
         for label in test_labels:
@@ -680,7 +760,12 @@ class DiscoverRunner:
         # _FailedTest objects include things like test modules that couldn't be
         # found or that couldn't be loaded due to syntax errors.
         test_types = (unittest.loader._FailedTest, *self.reorder_by)
-        all_tests = list(reorder_tests(all_tests, test_types, self.reverse))
+        all_tests = list(reorder_tests(
+            all_tests,
+            test_types,
+            shuffler=self._shuffler,
+            reverse=self.reverse,
+        ))
         self.log('Found %d test(s).' % len(all_tests))
         suite = self.test_suite(all_tests)
 
@@ -726,7 +811,12 @@ class DiscoverRunner:
     def run_suite(self, suite, **kwargs):
         kwargs = self.get_test_runner_kwargs()
         runner = self.test_runner(**kwargs)
+        try:
             return runner.run(suite)
+        finally:
+            if self._shuffler is not None:
+                seed_display = self._shuffler.seed_display
+                self.log(f'Used shuffle seed: {seed_display}')
 
     def teardown_databases(self, old_config, **kwargs):
         """Destroy all the non-mirror databases."""
@@ -851,17 +941,64 @@ def find_top_level(top_level):
     return top_level
 
 
-def reorder_tests(tests, classes, reverse=False):
+def _class_shuffle_key(cls):
+    return f'{cls.__module__}.{cls.__qualname__}'
+
+
+def shuffle_tests(tests, shuffler):
+    """
+    Return an iterator over the given tests in a shuffled order, keeping tests
+    next to other tests of their class.
+
+    `tests` should be an iterable of tests.
+    """
+    tests_by_type = {}
+    for _, class_tests in itertools.groupby(tests, type):
+        class_tests = list(class_tests)
+        test_type = type(class_tests[0])
+        class_tests = shuffler.shuffle(class_tests, key=lambda test: test.id())
+        tests_by_type[test_type] = class_tests
+
+    classes = shuffler.shuffle(tests_by_type, key=_class_shuffle_key)
+
+    return itertools.chain(*(tests_by_type[cls] for cls in classes))
+
+
+def reorder_test_bin(tests, shuffler=None, reverse=False):
+    """
+    Return an iterator that reorders the given tests, keeping tests next to
+    other tests of their class.
+
+    `tests` should be an iterable of tests that supports reversed().
+    """
+    if shuffler is None:
+        if reverse:
+            return reversed(tests)
+        # The function must return an iterator.
+        return iter(tests)
+
+    tests = shuffle_tests(tests, shuffler)
+    if not reverse:
+        return tests
+    # Arguments to reversed() must be reversible.
+    return reversed(list(tests))
+
+
+def reorder_tests(tests, classes, reverse=False, shuffler=None):
     """
     Reorder an iterable of tests by test type, removing any duplicates.
 
-    `classes` is a sequence of types. The result is returned as an iterator.
+    The result is returned as an iterator. `classes` is a sequence of types.
-
     All tests of type classes[0] are placed first, then tests of type
     classes[1], etc. Tests with no match in classes are placed last.
 
     If `reverse` is True, sort tests within classes in opposite order but
     don't reverse test classes.
+
+    The `shuffler` argument is an optional instance of this module's `Shuffler`
+    class. If provided, tests will be shuffled within each `classes` group, but
+    keeping tests with other tests of their TestCase class. Reversing is
+    applied after shuffling to allow reversing the same random order.
     """
     bins = [OrderedSet() for i in range(len(classes) + 1)]
     *class_bins, last_bin = bins
@@ -874,9 +1011,8 @@ def reorder_tests(tests, classes, reverse=False):
             test_bin = last_bin
         test_bin.add(test)
 
-    if reverse:
+    for tests in bins:
-        bins = (reversed(tests) for tests in bins)
+        yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse)
-    return itertools.chain(*bins)
 
 
 def partition_suite_by_case(suite):

docs/internals/contributing/writing-code/unit-tests.txt

@@ -470,12 +470,13 @@ the first one:
 
     $ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions
 
-You can also try running any set of tests in reverse using the ``--reverse``
+You can also try running any set of tests in a random or reverse order using
-option in order to verify that executing tests in a different order does not
+the ``--shuffle`` and ``--reverse`` options. This can help verify that
-cause any trouble:
+executing tests in a different order does not cause any trouble:
 
 .. console::
 
+    $ ./runtests.py basic --shuffle
     $ ./runtests.py basic --reverse
 
 Seeing the SQL queries run during a test

docs/ref/django-admin.txt

@@ -1425,11 +1425,30 @@ subsequent run. Unless the :setting:`MIGRATE <TEST_MIGRATE>` test setting is
 ``False``, any unapplied migrations will also be applied to the test database
 before running the test suite.
 
+.. django-admin-option:: --shuffle [SEED]
+
+.. versionadded:: 4.0
+
+Randomizes the order of tests before running them. This can help detect tests
+that aren't properly isolated. The test order generated by this option is a
+deterministic function of the integer seed given. When no seed is passed, a
+seed is chosen randomly and printed to the console. To repeat a particular test
+order, pass a seed. The test orders generated by this option preserve Django's
+:ref:`guarantees on test order <order-of-tests>`. They also keep tests grouped
+by test case class.
+
+The shuffled orderings also have a special consistency property useful when
+narrowing down isolation issues. Namely, for a given seed and when running a
+subset of tests, the new order will be the original shuffling restricted to the
+smaller set. Similarly, when adding tests while keeping the seed the same, the
+order of the original tests will be the same in the new order.
+
 .. django-admin-option:: --reverse, -r
 
 Sorts test cases in the opposite execution order. This may help in debugging
 the side effects of tests that aren't properly isolated. :ref:`Grouping by test
-class <order-of-tests>` is preserved when using this option.
+class <order-of-tests>` is preserved when using this option. This can be used
+in conjunction with ``--shuffle`` to reverse the order for a particular seed.
 
 .. django-admin-option:: --debug-mode
 

docs/releases/4.0.txt

@@ -325,6 +325,9 @@ Tests
 * The new :meth:`.DiscoverRunner.log` method allows customizing the way
   messages are logged.
 
+* Django test runner now supports a :option:`--shuffle <test --shuffle>` option
+  to execute tests in a random order.
+
 URLs
 ~~~~
 

docs/topics/testing/advanced.txt

@@ -510,7 +510,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a
 selection of other methods that are used by ``run_tests()`` to set up, execute
 and tear down the test suite.
 
-.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, **kwargs)
+.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, shuffle=False, **kwargs)
 
     ``DiscoverRunner`` will search for tests in any file matching ``pattern``.
 
@@ -539,7 +539,8 @@ and tear down the test suite.
     If ``reverse`` is ``True``, test cases will be executed in the opposite
     order. This could be useful to debug tests that aren't properly isolated
     and have side effects. :ref:`Grouping by test class <order-of-tests>` is
-    preserved when using this option.
+    preserved when using this option. This option can be used in conjunction
+    with ``--shuffle`` to reverse the order for a particular random seed.
 
     ``debug_mode`` specifies what the :setting:`DEBUG` setting should be
     set to prior to running tests.
@@ -576,6 +577,14 @@ and tear down the test suite.
     If ``timing`` is ``True``, test timings, including database setup and total
     run time, will be shown.
 
+    If ``shuffle`` is an integer, test cases will be shuffled in a random order
+    prior to execution, using the integer as a random seed. If ``shuffle`` is
+    ``None``, the seed will be generated randomly. In both cases, the seed will
+    be logged to the console and set to ``self.shuffle_seed`` prior to running
+    tests. This option can be used to help detect tests that aren't properly
+    isolated. :ref:`Grouping by test class <order-of-tests>` is preserved when
+    using this option.
+
     Django may, from time to time, extend the capabilities of the test runner
     by adding new arguments. The ``**kwargs`` declaration allows for this
     expansion. If you subclass ``DiscoverRunner`` or write your own test
@@ -590,6 +599,10 @@ and tear down the test suite.
 
         The ``enable_faulthandler`` and ``timing`` arguments were added.
 
+    .. versionadded:: 4.0
+
+        The ``shuffle`` argument was added.
+
 Attributes
 ~~~~~~~~~~
 

docs/topics/testing/overview.txt

@@ -235,9 +235,9 @@ the Django test runner reorders tests in the following way:
     for quicker feedback. This includes things like test modules that couldn't
     be found or that couldn't be loaded due to syntax errors.
 
-You may reverse the execution order inside groups using the :option:`test
+You may randomize and/or reverse the execution order inside groups using the
---reverse` option. This can help with ensuring your tests are independent from
+:option:`test --shuffle` and :option:`--reverse <test --reverse>` options. This
-each other.
+can help with ensuring your tests are independent from each other.
 
 .. versionchanged:: 4.0
 

tests/runtests.py

@@ -353,7 +353,7 @@ class ActionSelenium(argparse.Action):
 def django_tests(verbosity, interactive, failfast, keepdb, reverse,
                  test_labels, debug_sql, parallel, tags, exclude_tags,
                  test_name_patterns, start_at, start_after, pdb, buffer,
-                 timing):
+                 timing, shuffle):
     if verbosity >= 1:
         msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__)
         max_parallel = default_test_processes() if parallel == 0 else parallel
@@ -380,6 +380,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse,
         pdb=pdb,
         buffer=buffer,
         timing=timing,
+        shuffle=shuffle,
     )
     failures = test_runner.run_tests(test_labels)
     teardown_run_tests(state)
@@ -406,6 +407,11 @@ def get_subprocess_args(options):
         subprocess_args.append('--tag=%s' % options.tags)
     if options.exclude_tags:
         subprocess_args.append('--exclude_tag=%s' % options.exclude_tags)
+    if options.shuffle is not False:
+        if options.shuffle is None:
+            subprocess_args.append('--shuffle')
+        else:
+            subprocess_args.append('--shuffle=%s' % options.shuffle)
     return subprocess_args
 
 
@@ -523,6 +529,13 @@ if __name__ == "__main__":
         '--pair',
         help='Run the test suite in pairs with the named test to find problem pairs.',
     )
+    parser.add_argument(
+        '--shuffle', nargs='?', default=False, type=int, metavar='SEED',
+        help=(
+            'Shuffle the order of test cases to help check that tests are '
+            'properly isolated.'
+        ),
+    )
     parser.add_argument(
         '--reverse', action='store_true',
         help='Sort test suites and test cases in opposite order to debug '
@@ -650,7 +663,7 @@ if __name__ == "__main__":
                 options.exclude_tags,
                 getattr(options, 'test_name_patterns', None),
                 options.start_at, options.start_after, options.pdb, options.buffer,
-                options.timing,
+                options.timing, options.shuffle,
             )
         time_keeper.print_results()
         if failures:

tests/test_runner/test_discover_runner.py

@@ -49,6 +49,16 @@ class DiscoverRunnerTests(SimpleTestCase):
         runner = DiscoverRunner()
         self.assertFalse(runner.debug_mode)
 
+    def test_add_arguments_shuffle(self):
+        parser = ArgumentParser()
+        DiscoverRunner.add_arguments(parser)
+        ns = parser.parse_args([])
+        self.assertIs(ns.shuffle, False)
+        ns = parser.parse_args(['--shuffle'])
+        self.assertIsNone(ns.shuffle)
+        ns = parser.parse_args(['--shuffle', '5'])
+        self.assertEqual(ns.shuffle, 5)
+
     def test_add_arguments_debug_mode(self):
         parser = ArgumentParser()
         DiscoverRunner.add_arguments(parser)
@@ -58,6 +68,30 @@ class DiscoverRunnerTests(SimpleTestCase):
         ns = parser.parse_args(["--debug-mode"])
         self.assertTrue(ns.debug_mode)
 
+    def test_setup_shuffler_no_shuffle_argument(self):
+        runner = DiscoverRunner()
+        self.assertIs(runner.shuffle, False)
+        runner.setup_shuffler()
+        self.assertIsNone(runner.shuffle_seed)
+
+    def test_setup_shuffler_shuffle_none(self):
+        runner = DiscoverRunner(shuffle=None)
+        self.assertIsNone(runner.shuffle)
+        with mock.patch('random.randint', return_value=1):
+            with captured_stdout() as stdout:
+                runner.setup_shuffler()
+        self.assertEqual(stdout.getvalue(), 'Using shuffle seed: 1 (generated)\n')
+        self.assertEqual(runner.shuffle_seed, 1)
+
+    def test_setup_shuffler_shuffle_int(self):
+        runner = DiscoverRunner(shuffle=2)
+        self.assertEqual(runner.shuffle, 2)
+        with captured_stdout() as stdout:
+            runner.setup_shuffler()
+        expected_out = 'Using shuffle seed: 2 (given)\n'
+        self.assertEqual(stdout.getvalue(), expected_out)
+        self.assertEqual(runner.shuffle_seed, 2)
+
     def test_load_tests_for_label_file_path(self):
         with change_cwd('.'):
             msg = (
@@ -266,6 +300,25 @@ class DiscoverRunnerTests(SimpleTestCase):
         self.assertIsInstance(tests[0], unittest.loader._FailedTest)
         self.assertNotIsInstance(tests[-1], unittest.loader._FailedTest)
 
+    def test_build_suite_shuffling(self):
+        # These will result in unittest.loader._FailedTest instances rather
+        # than TestCase objects, but they are sufficient for testing.
+        labels = ['label1', 'label2', 'label3', 'label4']
+        cases = [
+            ({}, ['label1', 'label2', 'label3', 'label4']),
+            ({'reverse': True}, ['label4', 'label3', 'label2', 'label1']),
+            ({'shuffle': 8}, ['label4', 'label1', 'label3', 'label2']),
+            ({'shuffle': 8, 'reverse': True}, ['label2', 'label3', 'label1', 'label4']),
+        ]
+        for kwargs, expected in cases:
+            with self.subTest(kwargs=kwargs):
+                # Prevent writing the seed to stdout.
+                runner = DiscoverRunner(**kwargs, verbosity=0)
+                tests = runner.build_suite(test_labels=labels)
+                # The ids have the form "unittest.loader._FailedTest.label1".
+                names = [test.id().split('.')[-1] for test in tests]
+                self.assertEqual(names, expected)
+
     def test_overridable_get_test_runner_kwargs(self):
         self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict)
 
@@ -374,6 +427,52 @@ class DiscoverRunnerTests(SimpleTestCase):
         self.assertIn('Write to stderr.', stderr.getvalue())
         self.assertIn('Write to stdout.', stdout.getvalue())
 
+    def run_suite_with_runner(self, runner_class, **kwargs):
+        class MyRunner(DiscoverRunner):
+            def test_runner(self, *args, **kwargs):
+                return runner_class()
+
+        runner = MyRunner(**kwargs)
+        # Suppress logging "Using shuffle seed" to the console.
+        with captured_stdout():
+            runner.setup_shuffler()
+        with captured_stdout() as stdout:
+            try:
+                result = runner.run_suite(None)
+            except RuntimeError as exc:
+                result = str(exc)
+        output = stdout.getvalue()
+        return result, output
+
+    def test_run_suite_logs_seed(self):
+        class TestRunner:
+            def run(self, suite):
+                return '<fake-result>'
+
+        expected_prefix = 'Used shuffle seed'
+        # Test with and without shuffling enabled.
+        result, output = self.run_suite_with_runner(TestRunner)
+        self.assertEqual(result, '<fake-result>')
+        self.assertNotIn(expected_prefix, output)
+
+        result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
+        self.assertEqual(result, '<fake-result>')
+        expected_output = f'{expected_prefix}: 2 (given)\n'
+        self.assertEqual(output, expected_output)
+
+    def test_run_suite_logs_seed_exception(self):
+        """
+        run_suite() logs the seed when TestRunner.run() raises an exception.
+        """
+        class TestRunner:
+            def run(self, suite):
+                raise RuntimeError('my exception')
+
+        result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
+        self.assertEqual(result, 'my exception')
+        expected_output = 'Used shuffle seed: 2 (given)\n'
+        self.assertEqual(output, expected_output)
+
     @mock.patch('faulthandler.enable')
     def test_faulthandler_enabled(self, mocked_enable):
         with mock.patch('faulthandler.is_enabled', return_value=False):

tests/test_runner/test_shuffler.py

@@ -0,0 +1,102 @@
+from unittest import mock
+
+from django.test import SimpleTestCase
+from django.test.runner import Shuffler
+
+
+class ShufflerTests(SimpleTestCase):
+
+    def test_hash_text(self):
+        actual = Shuffler._hash_text('abcd')
+        self.assertEqual(actual, 'e2fc714c4727ee9395f324cd2e7f331f')
+
+    def test_hash_text_hash_algorithm(self):
+        class MyShuffler(Shuffler):
+            hash_algorithm = 'sha1'
+
+        actual = MyShuffler._hash_text('abcd')
+        self.assertEqual(actual, '81fe8bfe87576c3ecb22426f8e57847382917acf')
+
+    def test_init(self):
+        shuffler = Shuffler(100)
+        self.assertEqual(shuffler.seed, 100)
+        self.assertEqual(shuffler.seed_source, 'given')
+
+    def test_init_none_seed(self):
+        with mock.patch('random.randint', return_value=200):
+            shuffler = Shuffler(None)
+        self.assertEqual(shuffler.seed, 200)
+        self.assertEqual(shuffler.seed_source, 'generated')
+
+    def test_init_no_seed_argument(self):
+        with mock.patch('random.randint', return_value=300):
+            shuffler = Shuffler()
+        self.assertEqual(shuffler.seed, 300)
+        self.assertEqual(shuffler.seed_source, 'generated')
+
+    def test_seed_display(self):
+        shuffler = Shuffler(100)
+        shuffler.seed_source = 'test'
+        self.assertEqual(shuffler.seed_display, '100 (test)')
+
+    def test_hash_item_seed(self):
+        cases = [
+            (1234, '64ad3fb166ddb41a2ca24f1803b8b722'),
+            # Passing a string gives the same value.
+            ('1234', '64ad3fb166ddb41a2ca24f1803b8b722'),
+            (5678, '4dde450ad339b6ce45a0a2666e35b975'),
+        ]
+        for seed, expected in cases:
+            with self.subTest(seed=seed):
+                shuffler = Shuffler(seed=seed)
+                actual = shuffler._hash_item('abc', lambda x: x)
+                self.assertEqual(actual, expected)
+
+    def test_hash_item_key(self):
+        cases = [
+            (lambda x: x, '64ad3fb166ddb41a2ca24f1803b8b722'),
+            (lambda x: x.upper(), 'ee22e8597bff91742affe4befbf4649a'),
+        ]
+        for key, expected in cases:
+            with self.subTest(key=key):
+                shuffler = Shuffler(seed=1234)
+                actual = shuffler._hash_item('abc', key)
+                self.assertEqual(actual, expected)
+
+    def test_shuffle_key(self):
+        cases = [
+            (lambda x: x, ['a', 'd', 'b', 'c']),
+            (lambda x: x.upper(), ['d', 'c', 'a', 'b']),
+        ]
+        for num, (key, expected) in enumerate(cases, start=1):
+            with self.subTest(num=num):
+                shuffler = Shuffler(seed=1234)
+                actual = shuffler.shuffle(['a', 'b', 'c', 'd'], key)
+                self.assertEqual(actual, expected)
+
+    def test_shuffle_consistency(self):
+        seq = [str(n) for n in range(5)]
+        cases = [
+            (None, ['3', '0', '2', '4', '1']),
+            (0, ['3', '2', '4', '1']),
+            (1, ['3', '0', '2', '4']),
+            (2, ['3', '0', '4', '1']),
+            (3, ['0', '2', '4', '1']),
+            (4, ['3', '0', '2', '1']),
+        ]
+        shuffler = Shuffler(seed=1234)
+        for index, expected in cases:
+            with self.subTest(index=index):
+                if index is None:
+                    new_seq = seq
+                else:
+                    new_seq = seq.copy()
+                    del new_seq[index]
+                actual = shuffler.shuffle(new_seq, lambda x: x)
+                self.assertEqual(actual, expected)
+
+    def test_shuffle_same_hash(self):
+        shuffler = Shuffler(seed=1234)
+        msg = "item 'A' has same hash 'a56ce89262959e151ee2266552f1819c' as item 'a'"
+        with self.assertRaisesMessage(RuntimeError, msg):
+            shuffler.shuffle(['a', 'b', 'A'], lambda x: x.upper())

tests/test_runner/tests.py

@@ -1,6 +1,7 @@
 """
 Tests for django test runner
 """
+import collections.abc
 import unittest
 from unittest import mock
 
@@ -14,7 +15,9 @@ from django.core.management.base import SystemCheckError
 from django.test import (
     SimpleTestCase, TransactionTestCase, skipUnlessDBFeature,
 )
-from django.test.runner import DiscoverRunner, reorder_tests
+from django.test.runner import (
+    DiscoverRunner, Shuffler, reorder_test_bin, reorder_tests, shuffle_tests,
+)
 from django.test.testcases import connections_support_transactions
 from django.test.utils import (
     captured_stderr, dependency_ordered, get_unique_databases_and_mirrors,
@@ -126,6 +129,68 @@ class TestSuiteTests(SimpleTestCase):
         self.assertEqual(len(tests), 4)
         self.assertNotIsInstance(tests[0], unittest.TestSuite)
 
+    def make_tests(self):
+        """Return an iterable of tests."""
+        suite = self.make_test_suite()
+        tests = list(iter_test_cases(suite))
+        return tests
+
+    def test_shuffle_tests(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        shuffled_tests = shuffle_tests(tests, shuffler)
+        self.assertIsInstance(shuffled_tests, collections.abc.Iterator)
+        self.assertTestNames(shuffled_tests, expected=[
+            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+        ])
+
+    def test_reorder_test_bin_no_arguments(self):
+        tests = self.make_tests()
+        reordered_tests = reorder_test_bin(tests)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
+        ])
+
+    def test_reorder_test_bin_reverse(self):
+        tests = self.make_tests()
+        reordered_tests = reorder_test_bin(tests, reverse=True)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests2.test2', 'Tests2.test1', 'Tests1.test2', 'Tests1.test1',
+        ])
+
+    def test_reorder_test_bin_random(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        reordered_tests = reorder_test_bin(tests, shuffler=shuffler)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+        ])
+
+    def test_reorder_test_bin_random_and_reverse(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        reordered_tests = reorder_test_bin(tests, shuffler=shuffler, reverse=True)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1',
+        ])
+
+    def test_reorder_tests_random(self):
+        tests = self.make_tests()
+        # Choose a seed that shuffles both the classes and methods.
+        shuffler = Shuffler(seed=9)
+        reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
+        self.assertIsInstance(reordered_tests, collections.abc.Iterator)
+        self.assertTestNames(reordered_tests, expected=[
+            'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
+        ])
+
     def test_reorder_tests_reverse_with_duplicates(self):
         class Tests1(unittest.TestCase):
             def test1(self):