diff --git a/AUTHORS b/AUTHORS index 1f842780f..2cdf587c0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -94,6 +94,7 @@ Punyashloka Biswal Quentin Pradet Ralf Schmitt Raphael Pierzina +Romain Dorgueil Roman Bolshakov Ronny Pfannschmidt Ross Lawley diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 522d63aa2..5f8ba271d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -70,6 +70,10 @@ time or change existing behaviors in order to make them less surprising/more use namespace in which doctests run. Thanks `@milliams`_ for the complete PR (`#1428`_). +* New ``--doctest-report`` option available to change the output format of diffs + when running (failing) doctests (implements `#1749`_). + Thanks `@hartym`_ for the PR. + * New ``name`` argument to ``pytest.fixture`` decorator which allows a custom name for a fixture (to solve the funcarg-shadowing-fixture problem). Thanks `@novas0x2a`_ for the complete PR (`#1444`_). @@ -314,6 +318,7 @@ time or change existing behaviors in order to make them less surprising/more use .. _#1664: https://github.com/pytest-dev/pytest/pull/1664 .. _#1684: https://github.com/pytest-dev/pytest/pull/1684 .. _#1723: https://github.com/pytest-dev/pytest/pull/1723 +.. _#1749: https://github.com/pytest-dev/pytest/issues/1749 .. _@DRMacIver: https://github.com/DRMacIver .. _@RedBeardCode: https://github.com/RedBeardCode @@ -328,6 +333,7 @@ time or change existing behaviors in order to make them less surprising/more use .. _@fengxx: https://github.com/fengxx .. _@flub: https://github.com/flub .. _@graingert: https://github.com/graingert +.. _@hartym: https://github.com/hartym .. _@kalekundert: https://github.com/kalekundert .. _@kvas-it: https://github.com/kvas-it .. _@marscher: https://github.com/marscher diff --git a/_pytest/doctest.py b/_pytest/doctest.py index 649077533..144aa2a6d 100644 --- a/_pytest/doctest.py +++ b/_pytest/doctest.py @@ -4,10 +4,23 @@ from __future__ import absolute_import import traceback import pytest -from _pytest._code.code import TerminalRepr, ReprFileLocation, ExceptionInfo +from _pytest._code.code import ExceptionInfo, ReprFileLocation, TerminalRepr from _pytest.fixtures import FixtureRequest +DOCTEST_REPORT_CHOICE_NONE = 'none' +DOCTEST_REPORT_CHOICE_CDIFF = 'cdiff' +DOCTEST_REPORT_CHOICE_NDIFF = 'ndiff' +DOCTEST_REPORT_CHOICE_UDIFF = 'udiff' +DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE = 'only_first_failure' + +DOCTEST_REPORT_CHOICES = ( + DOCTEST_REPORT_CHOICE_NONE, + DOCTEST_REPORT_CHOICE_CDIFF, + DOCTEST_REPORT_CHOICE_NDIFF, + DOCTEST_REPORT_CHOICE_UDIFF, + DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE, +) def pytest_addoption(parser): parser.addini('doctest_optionflags', 'option flags for doctests', @@ -17,6 +30,11 @@ def pytest_addoption(parser): action="store_true", default=False, help="run doctests in all .py modules", dest="doctestmodules") + group.addoption("--doctest-report", + type=str.lower, default="udiff", + help="choose another output format for diffs on doctest failure", + choices=DOCTEST_REPORT_CHOICES, + dest="doctestreport") group.addoption("--doctest-glob", action="append", default=[], metavar="pat", help="doctests file matching pattern, default: test*.txt", @@ -59,7 +77,6 @@ class ReprFailDoctest(TerminalRepr): class DoctestItem(pytest.Item): - def __init__(self, name, parent, runner=None, dtest=None): super(DoctestItem, self).__init__(name, parent) self.runner = runner @@ -94,7 +111,7 @@ class DoctestItem(pytest.Item): message = excinfo.type.__name__ reprlocation = ReprFileLocation(filename, lineno, message) checker = _get_checker() - REPORT_UDIFF = doctest.REPORT_UDIFF + report_choice = _get_report_choice(self.config.getoption("doctestreport")) if lineno is not None: lines = doctestfailure.test.docstring.splitlines(False) # add line numbers to the left of the error message @@ -110,7 +127,7 @@ class DoctestItem(pytest.Item): indent = '...' if excinfo.errisinstance(doctest.DocTestFailure): lines += checker.output_difference(example, - doctestfailure.got, REPORT_UDIFF).split("\n") + doctestfailure.got, report_choice).split("\n") else: inner_excinfo = ExceptionInfo(excinfo.value.exc_info) lines += ["UNEXPECTED EXCEPTION: %s" % @@ -291,6 +308,21 @@ def _get_allow_bytes_flag(): return doctest.register_optionflag('ALLOW_BYTES') +def _get_report_choice(key): + """ + This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid + importing `doctest` and all its dependencies when parsing options, as it adds overhead and breaks tests. + """ + import doctest + + return { + DOCTEST_REPORT_CHOICE_UDIFF: doctest.REPORT_UDIFF, + DOCTEST_REPORT_CHOICE_CDIFF: doctest.REPORT_CDIFF, + DOCTEST_REPORT_CHOICE_NDIFF: doctest.REPORT_NDIFF, + DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE: doctest.REPORT_ONLY_FIRST_FAILURE, + DOCTEST_REPORT_CHOICE_NONE: 0, + }[key] + @pytest.fixture(scope='session') def doctest_namespace(): """ diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index b9b19cb85..7875c25e0 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -16,7 +16,6 @@ from docstrings in all python modules (including regular python test modules):: pytest --doctest-modules - You can make these changes permanent in your project by putting them into a pytest.ini file like this: @@ -102,6 +101,7 @@ itself:: >>> get_unicode_greeting() # doctest: +ALLOW_UNICODE 'Hello' + The 'doctest_namespace' fixture ------------------------------- @@ -130,3 +130,22 @@ which can then be used in your doctests directly:: 10 """ pass + + +Output format +------------- + +.. versionadded:: 3.0 + +You can change the diff output format on failure for your doctests +by using one of standard doctest modules format in options +(see :data:`python:doctest.REPORT_UDIFF`, :data:`python:doctest.REPORT_CDIFF`, +:data:`python:doctest.REPORT_NDIFF`, :data:`python:doctest.REPORT_ONLY_FIRST_FAILURE`):: + + pytest --doctest-modules --doctest-report none + pytest --doctest-modules --doctest-report udiff + pytest --doctest-modules --doctest-report cdiff + pytest --doctest-modules --doctest-report ndiff + pytest --doctest-modules --doctest-report only_first_failure + + diff --git a/testing/test_doctest.py b/testing/test_doctest.py index d0bcb1425..4ea2cc58e 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -784,3 +784,81 @@ class TestDoctestNamespaceFixture: """) reprec = testdir.inline_run(p, "--doctest-modules") reprec.assertoutcome(passed=1) + + +class TestDoctestReportingOption: + def _run_doctest_report(self, testdir, format): + testdir.makepyfile(""" + def foo(): + ''' + >>> foo() + a b + 0 1 4 + 1 2 4 + 2 3 6 + ''' + print(' a b\\n' + '0 1 4\\n' + '1 2 5\\n' + '2 3 6') + """) + return testdir.runpytest("--doctest-modules", "--doctest-report", format) + + @pytest.mark.parametrize('format', ['udiff', 'UDIFF', 'uDiFf']) + def test_doctest_report_udiff(self, testdir, format): + result = self._run_doctest_report(testdir, format) + result.stdout.fnmatch_lines([ + ' 0 1 4', + ' -1 2 4', + ' +1 2 5', + ' 2 3 6', + ]) + + def test_doctest_report_cdiff(self, testdir): + result = self._run_doctest_report(testdir, 'cdiff') + result.stdout.fnmatch_lines([ + ' a b', + ' 0 1 4', + ' ! 1 2 4', + ' 2 3 6', + ' --- 1,4 ----', + ' a b', + ' 0 1 4', + ' ! 1 2 5', + ' 2 3 6', + ]) + + def test_doctest_report_ndiff(self, testdir): + result = self._run_doctest_report(testdir, 'ndiff') + result.stdout.fnmatch_lines([ + ' a b', + ' 0 1 4', + ' - 1 2 4', + ' ? ^', + ' + 1 2 5', + ' ? ^', + ' 2 3 6', + ]) + + @pytest.mark.parametrize('format', ['none', 'only_first_failure']) + def test_doctest_report_none_or_only_first_failure(self, testdir, format): + result = self._run_doctest_report(testdir, format) + result.stdout.fnmatch_lines([ + 'Expected:', + ' a b', + ' 0 1 4', + ' 1 2 4', + ' 2 3 6', + 'Got:', + ' a b', + ' 0 1 4', + ' 1 2 5', + ' 2 3 6', + ]) + + def test_doctest_report_invalid(self, testdir): + result = self._run_doctest_report(testdir, 'obviously_invalid_format') + result.stderr.fnmatch_lines([ + "*error: argument --doctest-report: invalid choice: 'obviously_invalid_format' (choose from*" + ]) +