From d4dc775620fc57e962165eab98a77264e3dd16b2 Mon Sep 17 00:00:00 2001 From: Jakub Paczkowski Date: Sat, 7 Nov 2015 14:57:56 +0100 Subject: [PATCH] Fixed #25735 -- Added support for test tags to DiscoverRunner. Thanks Carl Meyer, Claude Paroz, and Simon Charette for review. --- AUTHORS | 1 + django/test/runner.py | 31 +++++++++++- django/test/utils.py | 10 ++++ docs/ref/django-admin.txt | 14 ++++++ docs/releases/1.10.txt | 4 ++ docs/topics/testing/tools.txt | 54 +++++++++++++++++++++ tests/runtests.py | 24 +++++++-- tests/test_discovery_sample/tests_sample.py | 13 +++++ tests/test_runner/test_discover_runner.py | 20 +++++++- 9 files changed, 162 insertions(+), 9 deletions(-) diff --git a/AUTHORS b/AUTHORS index 67a974765b..8e10fff33d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -307,6 +307,7 @@ answer newbie questions, and generally made Django that much better: Jaap Roes Jacob Burch Jacob Kaplan-Moss + Jakub Paczkowski Jakub Wilk Jakub Wiśniowski james_027@yahoo.com diff --git a/django/test/runner.py b/django/test/runner.py index 7d24aae9dc..b025e2a1f7 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -360,11 +360,10 @@ class DiscoverRunner(object): def __init__(self, pattern=None, top_level=None, verbosity=1, interactive=True, failfast=False, keepdb=False, reverse=False, debug_sql=False, parallel=0, - **kwargs): + tags=None, exclude_tags=None, **kwargs): self.pattern = pattern self.top_level = top_level - self.verbosity = verbosity self.interactive = interactive self.failfast = failfast @@ -372,6 +371,8 @@ class DiscoverRunner(object): self.reverse = reverse self.debug_sql = debug_sql self.parallel = parallel + self.tags = set(tags or []) + self.exclude_tags = set(exclude_tags or []) @classmethod def add_arguments(cls, parser): @@ -394,6 +395,10 @@ class DiscoverRunner(object): '--parallel', dest='parallel', nargs='?', default=1, type=int, const=default_test_processes(), metavar='N', help='Run tests using up to N parallel processes.') + parser.add_argument('--tag', action='append', dest='tags', + help='Run only tests with the specified tag. Can be used multiple times.') + parser.add_argument('--exclude-tag', action='append', dest='exclude_tags', + help='Do not run tests with the specified tag. Can be used multiple times.') def setup_test_environment(self, **kwargs): setup_test_environment() @@ -459,6 +464,8 @@ class DiscoverRunner(object): for test in extra_tests: suite.addTest(test) + if self.tags or self.exclude_tags: + suite = filter_tests_by_tags(suite, self.tags, self.exclude_tags) suite = reorder_suite(suite, self.reorder_by, self.reverse) if self.parallel > 1: @@ -747,3 +754,23 @@ def setup_databases(verbosity, interactive, keepdb=False, debug_sql=False, paral connections[alias].force_debug_cursor = True return old_names + + +def filter_tests_by_tags(suite, tags, exclude_tags): + suite_class = type(suite) + filtered_suite = suite_class() + + for test in suite: + if isinstance(test, suite_class): + filtered_suite.addTests(filter_tests_by_tags(test, tags, exclude_tags)) + else: + test_tags = set(getattr(test, 'tags', set())) + test_fn_name = getattr(test, '_testMethodName', str(test)) + test_fn = getattr(test, test_fn_name, test) + test_fn_tags = set(getattr(test_fn, 'tags', set())) + all_tags = test_tags.union(test_fn_tags) + matched_tags = all_tags.intersection(tags) + if (matched_tags or not tags) and not all_tags.intersection(exclude_tags): + filtered_suite.addTest(test) + + return filtered_suite diff --git a/django/test/utils.py b/django/test/utils.py index ee2837f497..28b02e2199 100644 --- a/django/test/utils.py +++ b/django/test/utils.py @@ -707,3 +707,13 @@ class isolate_apps(TestContextDecorator): def disable(self): setattr(Options, 'default_apps', self.old_apps) + + +def tag(*tags): + """ + Decorator to add tags to a test class or method. + """ + def decorator(obj): + setattr(obj, 'tags', set(tags)) + return obj + return decorator diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 0fbad35bcf..87753b28bd 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -1348,6 +1348,20 @@ don't. in order to exchange them between processes. See :ref:`python:pickle-picklable` for details. +.. option:: --tag TAGS + +.. versionadded:: 1.10 + +Runs only tests :ref:`marked with the specified tags `. +May be specified multiple times and combined with :option:`test --exclude-tag`. + +.. option:: --exclude-tag EXCLUDE_TAGS + +.. versionadded:: 1.10 + +Excludes tests :ref:`marked with the specified tags `. +May be specified multiple times and combined with :option:`test --tag`. + ``testserver`` -------------- diff --git a/docs/releases/1.10.txt b/docs/releases/1.10.txt index b3bd4b8afd..1154429f2b 100644 --- a/docs/releases/1.10.txt +++ b/docs/releases/1.10.txt @@ -336,6 +336,10 @@ Tests * To better catch bugs, :class:`~django.test.TestCase` now checks deferrable database constraints at the end of each test. +* Tests and test cases can be :ref:`marked with tags ` + and run selectively with the new :option:`test --tag` and :option:`test + --exclude-tag` options. + URLs ~~~~ diff --git a/docs/topics/testing/tools.txt b/docs/topics/testing/tools.txt index 2bb53454cf..de008bd325 100644 --- a/docs/topics/testing/tools.txt +++ b/docs/topics/testing/tools.txt @@ -1622,6 +1622,60 @@ your test suite. Person.objects.create(name="Aaron") Person.objects.create(name="Daniel") +.. _topics-tagging-tests: + +Tagging tests +------------- + +.. versionadded:: 1.10 + +You can tag your tests so you can easily run a particular subset. For example, +you might label fast or slow tests:: + + from django.test.utils import tag + + class SampleTestCase(TestCase): + + @tag('fast') + def test_fast(self): + ... + + @tag('slow') + def test_slow(self): + ... + + @tag('slow', 'core') + def test_slow_but_core(self): + ... + +You can also tag a test case:: + + @tag('slow', 'core') + class SampleTestCase(TestCase): + ... + +Then you can choose which tests to run. For example, to run only fast tests: + +.. code-block:: console + + $ ./manage.py test --tag=fast + +Or to run fast tests and the core one (even though it's slow): + +.. code-block:: console + + $ ./manage.py test --tag=fast --tag=core + +You can also exclude tests by tag. To run core tests if they are not slow: + +.. code-block:: console + + $ ./manage.py test --tag=core --exclude-tag=slow + +:option:`test --exclude-tag` has precedence over :option:`test --tag`, so if a +test has two tags and you select one of them and exclude the other, the test +won't be run. + .. _topics-testing-email: Email services diff --git a/tests/runtests.py b/tests/runtests.py index 4be186c123..6fca3dc6a6 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -234,7 +234,7 @@ def actual_test_processes(parallel): def django_tests(verbosity, interactive, failfast, keepdb, reverse, - test_labels, debug_sql, parallel): + test_labels, debug_sql, parallel, tags, exclude_tags): state = setup(verbosity, test_labels, parallel) extra_tests = [] @@ -251,6 +251,8 @@ def django_tests(verbosity, interactive, failfast, keepdb, reverse, reverse=reverse, debug_sql=debug_sql, parallel=actual_test_processes(parallel), + tags=tags, + exclude_tags=exclude_tags, ) failures = test_runner.run_tests( test_labels or get_installed(), @@ -270,6 +272,10 @@ def get_subprocess_args(options): subprocess_args.append('--verbosity=%s' % options.verbosity) if not options.interactive: subprocess_args.append('--noinput') + if options.tags: + subprocess_args.append('--tag=%s' % options.tags) + if options.exclude_tags: + subprocess_args.append('--exclude_tag=%s' % options.exclude_tags) return subprocess_args @@ -399,6 +405,12 @@ if __name__ == "__main__": '--parallel', dest='parallel', nargs='?', default=0, type=int, const=default_test_processes(), metavar='N', help='Run tests using up to N parallel processes.') + parser.add_argument( + '--tag', dest='tags', action='append', + help='Run only tests with the specified tags. Can be used multiple times.') + parser.add_argument( + '--exclude-tag', dest='exclude_tags', action='append', + help='Do not run tests with the specified tag. Can be used multiple times.') options = parser.parse_args() @@ -433,9 +445,11 @@ if __name__ == "__main__": elif options.pair: paired_tests(options.pair, options, options.modules, options.parallel) else: - failures = django_tests(options.verbosity, options.interactive, - options.failfast, options.keepdb, - options.reverse, options.modules, - options.debug_sql, options.parallel) + failures = django_tests( + options.verbosity, options.interactive, options.failfast, + options.keepdb, options.reverse, options.modules, + options.debug_sql, options.parallel, options.tags, + options.exclude_tags, + ) if failures: sys.exit(bool(failures)) diff --git a/tests/test_discovery_sample/tests_sample.py b/tests/test_discovery_sample/tests_sample.py index eb977e44fb..53588709ea 100644 --- a/tests/test_discovery_sample/tests_sample.py +++ b/tests/test_discovery_sample/tests_sample.py @@ -2,6 +2,7 @@ import doctest from unittest import TestCase from django.test import SimpleTestCase, TestCase as DjangoTestCase +from django.test.utils import tag from . import doctests @@ -29,6 +30,18 @@ class EmptyTestCase(TestCase): pass +@tag('slow') +class TaggedTestCase(TestCase): + + @tag('fast') + def test_single_tag(self): + self.assertEqual(1, 1) + + @tag('fast', 'core') + def test_multiple_tags(self): + self.assertEqual(1, 1) + + def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite(doctests)) return tests diff --git a/tests/test_runner/test_discover_runner.py b/tests/test_runner/test_discover_runner.py index 833cca96ea..37c18b2f8f 100644 --- a/tests/test_runner/test_discover_runner.py +++ b/tests/test_runner/test_discover_runner.py @@ -25,7 +25,7 @@ class DiscoverRunnerTest(TestCase): ["test_discovery_sample.tests_sample"], ).countTestCases() - self.assertEqual(count, 4) + self.assertEqual(count, 6) def test_dotted_test_class_vanilla_unittest(self): count = DiscoverRunner().build_suite( @@ -61,7 +61,7 @@ class DiscoverRunnerTest(TestCase): ["test_discovery_sample/"], ).countTestCases() - self.assertEqual(count, 5) + self.assertEqual(count, 7) def test_empty_label(self): """ @@ -165,3 +165,19 @@ class DiscoverRunnerTest(TestCase): def test_overridable_test_loader(self): self.assertEqual(DiscoverRunner().test_loader, defaultTestLoader) + + def test_tags(self): + runner = DiscoverRunner(tags=['core']) + self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 1) + runner = DiscoverRunner(tags=['fast']) + self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 2) + runner = DiscoverRunner(tags=['slow']) + self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 2) + + def test_exclude_tags(self): + runner = DiscoverRunner(tags=['fast'], exclude_tags=['core']) + self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 1) + runner = DiscoverRunner(tags=['fast'], exclude_tags=['slow']) + self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 0) + runner = DiscoverRunner(exclude_tags=['slow']) + self.assertEqual(runner.build_suite(['test_discovery_sample.tests_sample']).countTestCases(), 4)