p15693087
/
django
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed #24522 -- Added a --shuffle option to DiscoverRunner.

This commit is contained in:
Chris Jerdonek 2021-04-24 16:46:16 -07:00 committed by Mariusz Felisiak
parent 77b88fe621
commit 90ba716bf0
10 changed files with 472 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

154
django/test/runner.py
View File

@ -1,11 +1,13 @@
import ctypes
import faulthandler
import hashlib
import io
import itertools
import logging
import multiprocessing
import os
import pickle
import random
import sys
import textwrap
import unittest
@ -469,6 +471,64 @@ class ParallelTestSuite(unittest.TestSuite):
return iter(self.subsuites)
class Shuffler:
"""
This class implements shuffling with a special consistency property.
Consistency means that, for a given seed and key function, if two sets of
items are shuffled, the resulting order will agree on the intersection of
the two sets. For example, if items are removed from an original set, the
shuffled order for the new set will be the shuffled order of the original
set restricted to the smaller set.
"""
# This doesn't need to be cryptographically strong, so use what's fastest.
hash_algorithm = 'md5'
@classmethod
def _hash_text(cls, text):
h = hashlib.new(cls.hash_algorithm)
h.update(text.encode('utf-8'))
return h.hexdigest()
def __init__(self, seed=None):
if seed is None:
# Limit seeds to 9 digits for simpler output.
seed = random.randint(0, 10**10 - 1)
seed_source = 'generated'
else:
seed_source = 'given'
self.seed = seed
self.seed_source = seed_source
@property
def seed_display(self):
return f'{self.seed!r} ({self.seed_source})'
def _hash_item(self, item, key):
text = '{}{}'.format(self.seed, key(item))
return self._hash_text(text)
def shuffle(self, items, key):
"""
Return a new list of the items in a shuffled order.
The `key` is a function that accepts an item in `items` and returns
a string unique for that item that can be viewed as a string id. The
order of the return value is deterministic. It depends on the seed
and key function but not on the original order.
"""
hashes = {}
for item in items:
hashed = self._hash_item(item, key)
if hashed in hashes:
msg = 'item {!r} has same hash {!r} as item {!r}'.format(
item, hashed, hashes[hashed],
)
raise RuntimeError(msg)
hashes[hashed] = item
return [hashes[hashed] for hashed in sorted(hashes)]
class DiscoverRunner:
"""A Django test runner that uses unittest2 test discovery."""
@ -483,7 +543,7 @@ class DiscoverRunner:
reverse=False, debug_mode=False, debug_sql=False, parallel=0,
tags=None, exclude_tags=None, test_name_patterns=None,
pdb=False, buffer=False, enable_faulthandler=True,
timing=False, **kwargs):
timing=False, shuffle=False, **kwargs):
self.pattern = pattern
self.top_level = top_level
@ -515,6 +575,8 @@ class DiscoverRunner:
pattern if '*' in pattern else '*%s*' % pattern
for pattern in test_name_patterns
}
self.shuffle = shuffle
self._shuffler = None
@classmethod
def add_arguments(cls, parser):
@ -530,6 +592,10 @@ class DiscoverRunner:
'--keepdb', action='store_true',
help='Preserves the test DB between runs.'
)
parser.add_argument(
'--shuffle', nargs='?', default=False, type=int, metavar='SEED',
help='Shuffles test case order.',
)
parser.add_argument(
'-r', '--reverse', action='store_true',
help='Reverses test case order.',
@ -582,6 +648,12 @@ class DiscoverRunner:
),
)
@property
def shuffle_seed(self):
if self._shuffler is None:
return None
return self._shuffler.seed
def log(self, msg, level=None):
"""
Log the given message at the given logging level.
@ -599,6 +671,13 @@ class DiscoverRunner:
setup_test_environment(debug=self.debug_mode)
unittest.installHandler()
def setup_shuffler(self):
if self.shuffle is False:
return
shuffler = Shuffler(seed=self.shuffle)
self.log(f'Using shuffle seed: {shuffler.seed_display}')
self._shuffler = shuffler
@contextmanager
def load_with_patterns(self):
original_test_name_patterns = self.test_loader.testNamePatterns
@ -655,6 +734,7 @@ class DiscoverRunner:
discover_kwargs['pattern'] = self.pattern
if self.top_level is not None:
discover_kwargs['top_level_dir'] = self.top_level
self.setup_shuffler()
all_tests = []
for label in test_labels:
@ -680,7 +760,12 @@ class DiscoverRunner:
# _FailedTest objects include things like test modules that couldn't be
# found or that couldn't be loaded due to syntax errors.
test_types = (unittest.loader._FailedTest, *self.reorder_by)
all_tests = list(reorder_tests(all_tests, test_types, self.reverse))
all_tests = list(reorder_tests(
all_tests,
test_types,
shuffler=self._shuffler,
reverse=self.reverse,
))
self.log('Found %d test(s).' % len(all_tests))
suite = self.test_suite(all_tests)
@ -726,7 +811,12 @@ class DiscoverRunner:
def run_suite(self, suite, **kwargs):
kwargs = self.get_test_runner_kwargs()
runner = self.test_runner(**kwargs)
return runner.run(suite)
try:
return runner.run(suite)
finally:
if self._shuffler is not None:
seed_display = self._shuffler.seed_display
self.log(f'Used shuffle seed: {seed_display}')
def teardown_databases(self, old_config, **kwargs):
"""Destroy all the non-mirror databases."""
@ -851,17 +941,64 @@ def find_top_level(top_level):
return top_level
def reorder_tests(tests, classes, reverse=False):
def _class_shuffle_key(cls):
return f'{cls.__module__}.{cls.__qualname__}'
def shuffle_tests(tests, shuffler):
"""
Return an iterator over the given tests in a shuffled order, keeping tests
next to other tests of their class.
`tests` should be an iterable of tests.
"""
tests_by_type = {}
for _, class_tests in itertools.groupby(tests, type):
class_tests = list(class_tests)
test_type = type(class_tests[0])
class_tests = shuffler.shuffle(class_tests, key=lambda test: test.id())
tests_by_type[test_type] = class_tests
classes = shuffler.shuffle(tests_by_type, key=_class_shuffle_key)
return itertools.chain(*(tests_by_type[cls] for cls in classes))
def reorder_test_bin(tests, shuffler=None, reverse=False):
"""
Return an iterator that reorders the given tests, keeping tests next to
other tests of their class.
`tests` should be an iterable of tests that supports reversed().
"""
if shuffler is None:
if reverse:
return reversed(tests)
# The function must return an iterator.
return iter(tests)
tests = shuffle_tests(tests, shuffler)
if not reverse:
return tests
# Arguments to reversed() must be reversible.
return reversed(list(tests))
def reorder_tests(tests, classes, reverse=False, shuffler=None):
"""
Reorder an iterable of tests by test type, removing any duplicates.
`classes` is a sequence of types. The result is returned as an iterator.
The result is returned as an iterator. `classes` is a sequence of types.
All tests of type classes[0] are placed first, then tests of type
classes[1], etc. Tests with no match in classes are placed last.
If `reverse` is True, sort tests within classes in opposite order but
don't reverse test classes.
The `shuffler` argument is an optional instance of this module's `Shuffler`
class. If provided, tests will be shuffled within each `classes` group, but
keeping tests with other tests of their TestCase class. Reversing is
applied after shuffling to allow reversing the same random order.
"""
bins = [OrderedSet() for i in range(len(classes) + 1)]
*class_bins, last_bin = bins
@ -874,9 +1011,8 @@ def reorder_tests(tests, classes, reverse=False):
test_bin = last_bin
test_bin.add(test)
if reverse:
bins = (reversed(tests) for tests in bins)
return itertools.chain(*bins)
for tests in bins:
yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse)
def partition_suite_by_case(suite):

7
docs/internals/contributing/writing-code/unit-tests.txt
View File

@ -470,12 +470,13 @@ the first one:
$ ./runtests.py --pair basic.tests.ModelTest.test_eq queries transactions
You can also try running any set of tests in reverse using the ``--reverse``
option in order to verify that executing tests in a different order does not
cause any trouble:
You can also try running any set of tests in a random or reverse order using
the ``--shuffle`` and ``--reverse`` options. This can help verify that
executing tests in a different order does not cause any trouble:
.. console::
$ ./runtests.py basic --shuffle
$ ./runtests.py basic --reverse
Seeing the SQL queries run during a test

21
docs/ref/django-admin.txt
View File

@ -1425,11 +1425,30 @@ subsequent run. Unless the :setting:`MIGRATE <TEST_MIGRATE>` test setting is
``False``, any unapplied migrations will also be applied to the test database
before running the test suite.
.. django-admin-option:: --shuffle [SEED]
.. versionadded:: 4.0
Randomizes the order of tests before running them. This can help detect tests
that aren't properly isolated. The test order generated by this option is a
deterministic function of the integer seed given. When no seed is passed, a
seed is chosen randomly and printed to the console. To repeat a particular test
order, pass a seed. The test orders generated by this option preserve Django's
:ref:`guarantees on test order <order-of-tests>`. They also keep tests grouped
by test case class.
The shuffled orderings also have a special consistency property useful when
narrowing down isolation issues. Namely, for a given seed and when running a
subset of tests, the new order will be the original shuffling restricted to the
smaller set. Similarly, when adding tests while keeping the seed the same, the
order of the original tests will be the same in the new order.
.. django-admin-option:: --reverse, -r
Sorts test cases in the opposite execution order. This may help in debugging
the side effects of tests that aren't properly isolated. :ref:`Grouping by test
class <order-of-tests>` is preserved when using this option.
class <order-of-tests>` is preserved when using this option. This can be used
in conjunction with ``--shuffle`` to reverse the order for a particular seed.
.. django-admin-option:: --debug-mode

3
docs/releases/4.0.txt
View File

@ -325,6 +325,9 @@ Tests
* The new :meth:`.DiscoverRunner.log` method allows customizing the way
messages are logged.
* Django test runner now supports a :option:`--shuffle <test --shuffle>` option
to execute tests in a random order.
URLs
~~~~

17
docs/topics/testing/advanced.txt
View File

@ -510,7 +510,7 @@ behavior. This class defines the ``run_tests()`` entry point, plus a
selection of other methods that are used by ``run_tests()`` to set up, execute
and tear down the test suite.
.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, **kwargs)
.. class:: DiscoverRunner(pattern='test*.py', top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_mode=False, debug_sql=False, parallel=0, tags=None, exclude_tags=None, test_name_patterns=None, pdb=False, buffer=False, enable_faulthandler=True, timing=True, shuffle=False, **kwargs)
``DiscoverRunner`` will search for tests in any file matching ``pattern``.
@ -539,7 +539,8 @@ and tear down the test suite.
If ``reverse`` is ``True``, test cases will be executed in the opposite
order. This could be useful to debug tests that aren't properly isolated
and have side effects. :ref:`Grouping by test class <order-of-tests>` is
preserved when using this option.
preserved when using this option. This option can be used in conjunction
with ``--shuffle`` to reverse the order for a particular random seed.
``debug_mode`` specifies what the :setting:`DEBUG` setting should be
set to prior to running tests.
@ -576,6 +577,14 @@ and tear down the test suite.
If ``timing`` is ``True``, test timings, including database setup and total
run time, will be shown.
If ``shuffle`` is an integer, test cases will be shuffled in a random order
prior to execution, using the integer as a random seed. If ``shuffle`` is
``None``, the seed will be generated randomly. In both cases, the seed will
be logged to the console and set to ``self.shuffle_seed`` prior to running
tests. This option can be used to help detect tests that aren't properly
isolated. :ref:`Grouping by test class <order-of-tests>` is preserved when
using this option.
Django may, from time to time, extend the capabilities of the test runner
by adding new arguments. The ``**kwargs`` declaration allows for this
expansion. If you subclass ``DiscoverRunner`` or write your own test
@ -590,6 +599,10 @@ and tear down the test suite.
The ``enable_faulthandler`` and ``timing`` arguments were added.
.. versionadded:: 4.0
The ``shuffle`` argument was added.
Attributes
~~~~~~~~~~

6
docs/topics/testing/overview.txt
View File

@ -235,9 +235,9 @@ the Django test runner reorders tests in the following way:
for quicker feedback. This includes things like test modules that couldn't
be found or that couldn't be loaded due to syntax errors.
You may reverse the execution order inside groups using the :option:`test
--reverse` option. This can help with ensuring your tests are independent from
each other.
You may randomize and/or reverse the execution order inside groups using the
:option:`test --shuffle` and :option:`--reverse <test --reverse>` options. This
can help with ensuring your tests are independent from each other.
.. versionchanged:: 4.0

17
tests/runtests.py
View File

@ -353,7 +353,7 @@ class ActionSelenium(argparse.Action):
def django_tests(verbosity, interactive, failfast, keepdb, reverse,
test_labels, debug_sql, parallel, tags, exclude_tags,
test_name_patterns, start_at, start_after, pdb, buffer,
timing):
timing, shuffle):
if verbosity >= 1:
msg = "Testing against Django installed in '%s'" % os.path.dirname(django.__file__)
max_parallel = default_test_processes() if parallel == 0 else parallel
@ -380,6 +380,7 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse,
pdb=pdb,
buffer=buffer,
timing=timing,
shuffle=shuffle,
)
failures = test_runner.run_tests(test_labels)
teardown_run_tests(state)
@ -406,6 +407,11 @@ def get_subprocess_args(options):
subprocess_args.append('--tag=%s' % options.tags)
if options.exclude_tags:
subprocess_args.append('--exclude_tag=%s' % options.exclude_tags)
if options.shuffle is not False:
if options.shuffle is None:
subprocess_args.append('--shuffle')
else:
subprocess_args.append('--shuffle=%s' % options.shuffle)
return subprocess_args
@ -523,6 +529,13 @@ if __name__ == "__main__":
'--pair',
help='Run the test suite in pairs with the named test to find problem pairs.',
)
parser.add_argument(
'--shuffle', nargs='?', default=False, type=int, metavar='SEED',
help=(
'Shuffle the order of test cases to help check that tests are '
'properly isolated.'
),
)
parser.add_argument(
'--reverse', action='store_true',
help='Sort test suites and test cases in opposite order to debug '
@ -650,7 +663,7 @@ if __name__ == "__main__":
options.exclude_tags,
getattr(options, 'test_name_patterns', None),
options.start_at, options.start_after, options.pdb, options.buffer,
options.timing,
options.timing, options.shuffle,
)
time_keeper.print_results()
if failures:

99
tests/test_runner/test_discover_runner.py
View File

@ -49,6 +49,16 @@ class DiscoverRunnerTests(SimpleTestCase):
runner = DiscoverRunner()
self.assertFalse(runner.debug_mode)
def test_add_arguments_shuffle(self):
parser = ArgumentParser()
DiscoverRunner.add_arguments(parser)
ns = parser.parse_args([])
self.assertIs(ns.shuffle, False)
ns = parser.parse_args(['--shuffle'])
self.assertIsNone(ns.shuffle)
ns = parser.parse_args(['--shuffle', '5'])
self.assertEqual(ns.shuffle, 5)
def test_add_arguments_debug_mode(self):
parser = ArgumentParser()
DiscoverRunner.add_arguments(parser)
@ -58,6 +68,30 @@ class DiscoverRunnerTests(SimpleTestCase):
ns = parser.parse_args(["--debug-mode"])
self.assertTrue(ns.debug_mode)
def test_setup_shuffler_no_shuffle_argument(self):
runner = DiscoverRunner()
self.assertIs(runner.shuffle, False)
runner.setup_shuffler()
self.assertIsNone(runner.shuffle_seed)
def test_setup_shuffler_shuffle_none(self):
runner = DiscoverRunner(shuffle=None)
self.assertIsNone(runner.shuffle)
with mock.patch('random.randint', return_value=1):
with captured_stdout() as stdout:
runner.setup_shuffler()
self.assertEqual(stdout.getvalue(), 'Using shuffle seed: 1 (generated)\n')
self.assertEqual(runner.shuffle_seed, 1)
def test_setup_shuffler_shuffle_int(self):
runner = DiscoverRunner(shuffle=2)
self.assertEqual(runner.shuffle, 2)
with captured_stdout() as stdout:
runner.setup_shuffler()
expected_out = 'Using shuffle seed: 2 (given)\n'
self.assertEqual(stdout.getvalue(), expected_out)
self.assertEqual(runner.shuffle_seed, 2)
def test_load_tests_for_label_file_path(self):
with change_cwd('.'):
msg = (
@ -266,6 +300,25 @@ class DiscoverRunnerTests(SimpleTestCase):
self.assertIsInstance(tests[0], unittest.loader._FailedTest)
self.assertNotIsInstance(tests[-1], unittest.loader._FailedTest)
def test_build_suite_shuffling(self):
# These will result in unittest.loader._FailedTest instances rather
# than TestCase objects, but they are sufficient for testing.
labels = ['label1', 'label2', 'label3', 'label4']
cases = [
({}, ['label1', 'label2', 'label3', 'label4']),
({'reverse': True}, ['label4', 'label3', 'label2', 'label1']),
({'shuffle': 8}, ['label4', 'label1', 'label3', 'label2']),
({'shuffle': 8, 'reverse': True}, ['label2', 'label3', 'label1', 'label4']),
]
for kwargs, expected in cases:
with self.subTest(kwargs=kwargs):
# Prevent writing the seed to stdout.
runner = DiscoverRunner(**kwargs, verbosity=0)
tests = runner.build_suite(test_labels=labels)
# The ids have the form "unittest.loader._FailedTest.label1".
names = [test.id().split('.')[-1] for test in tests]
self.assertEqual(names, expected)
def test_overridable_get_test_runner_kwargs(self):
self.assertIsInstance(DiscoverRunner().get_test_runner_kwargs(), dict)
@ -374,6 +427,52 @@ class DiscoverRunnerTests(SimpleTestCase):
self.assertIn('Write to stderr.', stderr.getvalue())
self.assertIn('Write to stdout.', stdout.getvalue())
def run_suite_with_runner(self, runner_class, **kwargs):
class MyRunner(DiscoverRunner):
def test_runner(self, *args, **kwargs):
return runner_class()
runner = MyRunner(**kwargs)
# Suppress logging "Using shuffle seed" to the console.
with captured_stdout():
runner.setup_shuffler()
with captured_stdout() as stdout:
try:
result = runner.run_suite(None)
except RuntimeError as exc:
result = str(exc)
output = stdout.getvalue()
return result, output
def test_run_suite_logs_seed(self):
class TestRunner:
def run(self, suite):
return '<fake-result>'
expected_prefix = 'Used shuffle seed'
# Test with and without shuffling enabled.
result, output = self.run_suite_with_runner(TestRunner)
self.assertEqual(result, '<fake-result>')
self.assertNotIn(expected_prefix, output)
result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
self.assertEqual(result, '<fake-result>')
expected_output = f'{expected_prefix}: 2 (given)\n'
self.assertEqual(output, expected_output)
def test_run_suite_logs_seed_exception(self):
"""
run_suite() logs the seed when TestRunner.run() raises an exception.
"""
class TestRunner:
def run(self, suite):
raise RuntimeError('my exception')
result, output = self.run_suite_with_runner(TestRunner, shuffle=2)
self.assertEqual(result, 'my exception')
expected_output = 'Used shuffle seed: 2 (given)\n'
self.assertEqual(output, expected_output)
@mock.patch('faulthandler.enable')
def test_faulthandler_enabled(self, mocked_enable):
with mock.patch('faulthandler.is_enabled', return_value=False):

102
tests/test_runner/test_shuffler.py Normal file
View File

@ -0,0 +1,102 @@
from unittest import mock
from django.test import SimpleTestCase
from django.test.runner import Shuffler
class ShufflerTests(SimpleTestCase):
def test_hash_text(self):
actual = Shuffler._hash_text('abcd')
self.assertEqual(actual, 'e2fc714c4727ee9395f324cd2e7f331f')
def test_hash_text_hash_algorithm(self):
class MyShuffler(Shuffler):
hash_algorithm = 'sha1'
actual = MyShuffler._hash_text('abcd')
self.assertEqual(actual, '81fe8bfe87576c3ecb22426f8e57847382917acf')
def test_init(self):
shuffler = Shuffler(100)
self.assertEqual(shuffler.seed, 100)
self.assertEqual(shuffler.seed_source, 'given')
def test_init_none_seed(self):
with mock.patch('random.randint', return_value=200):
shuffler = Shuffler(None)
self.assertEqual(shuffler.seed, 200)
self.assertEqual(shuffler.seed_source, 'generated')
def test_init_no_seed_argument(self):
with mock.patch('random.randint', return_value=300):
shuffler = Shuffler()
self.assertEqual(shuffler.seed, 300)
self.assertEqual(shuffler.seed_source, 'generated')
def test_seed_display(self):
shuffler = Shuffler(100)
shuffler.seed_source = 'test'
self.assertEqual(shuffler.seed_display, '100 (test)')
def test_hash_item_seed(self):
cases = [
(1234, '64ad3fb166ddb41a2ca24f1803b8b722'),
# Passing a string gives the same value.
('1234', '64ad3fb166ddb41a2ca24f1803b8b722'),
(5678, '4dde450ad339b6ce45a0a2666e35b975'),
]
for seed, expected in cases:
with self.subTest(seed=seed):
shuffler = Shuffler(seed=seed)
actual = shuffler._hash_item('abc', lambda x: x)
self.assertEqual(actual, expected)
def test_hash_item_key(self):
cases = [
(lambda x: x, '64ad3fb166ddb41a2ca24f1803b8b722'),
(lambda x: x.upper(), 'ee22e8597bff91742affe4befbf4649a'),
]
for key, expected in cases:
with self.subTest(key=key):
shuffler = Shuffler(seed=1234)
actual = shuffler._hash_item('abc', key)
self.assertEqual(actual, expected)
def test_shuffle_key(self):
cases = [
(lambda x: x, ['a', 'd', 'b', 'c']),
(lambda x: x.upper(), ['d', 'c', 'a', 'b']),
]
for num, (key, expected) in enumerate(cases, start=1):
with self.subTest(num=num):
shuffler = Shuffler(seed=1234)
actual = shuffler.shuffle(['a', 'b', 'c', 'd'], key)
self.assertEqual(actual, expected)
def test_shuffle_consistency(self):
seq = [str(n) for n in range(5)]
cases = [
(None, ['3', '0', '2', '4', '1']),
(0, ['3', '2', '4', '1']),
(1, ['3', '0', '2', '4']),
(2, ['3', '0', '4', '1']),
(3, ['0', '2', '4', '1']),
(4, ['3', '0', '2', '1']),
]
shuffler = Shuffler(seed=1234)
for index, expected in cases:
with self.subTest(index=index):
if index is None:
new_seq = seq
else:
new_seq = seq.copy()
del new_seq[index]
actual = shuffler.shuffle(new_seq, lambda x: x)
self.assertEqual(actual, expected)
def test_shuffle_same_hash(self):
shuffler = Shuffler(seed=1234)
msg = "item 'A' has same hash 'a56ce89262959e151ee2266552f1819c' as item 'a'"
with self.assertRaisesMessage(RuntimeError, msg):
shuffler.shuffle(['a', 'b', 'A'], lambda x: x.upper())

67
tests/test_runner/tests.py
View File

@ -1,6 +1,7 @@
"""
Tests for django test runner
"""
import collections.abc
import unittest
from unittest import mock
@ -14,7 +15,9 @@ from django.core.management.base import SystemCheckError
from django.test import (
SimpleTestCase, TransactionTestCase, skipUnlessDBFeature,
)
from django.test.runner import DiscoverRunner, reorder_tests
from django.test.runner import (
DiscoverRunner, Shuffler, reorder_test_bin, reorder_tests, shuffle_tests,
)
from django.test.testcases import connections_support_transactions
from django.test.utils import (
captured_stderr, dependency_ordered, get_unique_databases_and_mirrors,
@ -126,6 +129,68 @@ class TestSuiteTests(SimpleTestCase):
self.assertEqual(len(tests), 4)
self.assertNotIsInstance(tests[0], unittest.TestSuite)
def make_tests(self):
"""Return an iterable of tests."""
suite = self.make_test_suite()
tests = list(iter_test_cases(suite))
return tests
def test_shuffle_tests(self):
tests = self.make_tests()
# Choose a seed that shuffles both the classes and methods.
shuffler = Shuffler(seed=9)
shuffled_tests = shuffle_tests(tests, shuffler)
self.assertIsInstance(shuffled_tests, collections.abc.Iterator)
self.assertTestNames(shuffled_tests, expected=[
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
])
def test_reorder_test_bin_no_arguments(self):
tests = self.make_tests()
reordered_tests = reorder_test_bin(tests)
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
self.assertTestNames(reordered_tests, expected=[
'Tests1.test1', 'Tests1.test2', 'Tests2.test1', 'Tests2.test2',
])
def test_reorder_test_bin_reverse(self):
tests = self.make_tests()
reordered_tests = reorder_test_bin(tests, reverse=True)
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
self.assertTestNames(reordered_tests, expected=[
'Tests2.test2', 'Tests2.test1', 'Tests1.test2', 'Tests1.test1',
])
def test_reorder_test_bin_random(self):
tests = self.make_tests()
# Choose a seed that shuffles both the classes and methods.
shuffler = Shuffler(seed=9)
reordered_tests = reorder_test_bin(tests, shuffler=shuffler)
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
self.assertTestNames(reordered_tests, expected=[
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
])
def test_reorder_test_bin_random_and_reverse(self):
tests = self.make_tests()
# Choose a seed that shuffles both the classes and methods.
shuffler = Shuffler(seed=9)
reordered_tests = reorder_test_bin(tests, shuffler=shuffler, reverse=True)
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
self.assertTestNames(reordered_tests, expected=[
'Tests1.test1', 'Tests1.test2', 'Tests2.test2', 'Tests2.test1',
])
def test_reorder_tests_random(self):
tests = self.make_tests()
# Choose a seed that shuffles both the classes and methods.
shuffler = Shuffler(seed=9)
reordered_tests = reorder_tests(tests, classes=[], shuffler=shuffler)
self.assertIsInstance(reordered_tests, collections.abc.Iterator)
self.assertTestNames(reordered_tests, expected=[
'Tests2.test1', 'Tests2.test2', 'Tests1.test2', 'Tests1.test1',
])
def test_reorder_tests_reverse_with_duplicates(self):
class Tests1(unittest.TestCase):
def test1(self):