Merge pull request #1754 from hartym/1749_doctest_report_format

Doctest report format option (#1749)

AUTHORS

@@ -94,6 +94,7 @@ Punyashloka Biswal
 Quentin Pradet
 Ralf Schmitt
 Raphael Pierzina
+Romain Dorgueil
 Roman Bolshakov
 Ronny Pfannschmidt
 Ross Lawley

CHANGELOG.rst

@@ -70,6 +70,10 @@ time or change existing behaviors in order to make them less surprising/more use
   namespace in which doctests run.
   Thanks `@milliams`_ for the complete PR (`#1428`_).
 
+* New ``--doctest-report`` option available to change the output format of diffs
+  when running (failing) doctests (implements `#1749`_).
+  Thanks `@hartym`_ for the PR.
+
 * New ``name`` argument to ``pytest.fixture`` decorator which allows a custom name
   for a fixture (to solve the funcarg-shadowing-fixture problem).
   Thanks `@novas0x2a`_ for the complete PR (`#1444`_).
@@ -314,6 +318,7 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _#1664: https://github.com/pytest-dev/pytest/pull/1664
 .. _#1684: https://github.com/pytest-dev/pytest/pull/1684
 .. _#1723: https://github.com/pytest-dev/pytest/pull/1723
+.. _#1749: https://github.com/pytest-dev/pytest/issues/1749
 
 .. _@DRMacIver: https://github.com/DRMacIver
 .. _@RedBeardCode: https://github.com/RedBeardCode
@@ -328,6 +333,7 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _@fengxx: https://github.com/fengxx
 .. _@flub: https://github.com/flub
 .. _@graingert: https://github.com/graingert
+.. _@hartym: https://github.com/hartym
 .. _@kalekundert: https://github.com/kalekundert
 .. _@kvas-it: https://github.com/kvas-it
 .. _@marscher: https://github.com/marscher

_pytest/doctest.py

@@ -4,10 +4,23 @@ from __future__ import absolute_import
 import traceback
 
 import pytest
-from _pytest._code.code import TerminalRepr, ReprFileLocation, ExceptionInfo
+from _pytest._code.code import ExceptionInfo, ReprFileLocation, TerminalRepr
 from _pytest.fixtures import FixtureRequest
 
 
+DOCTEST_REPORT_CHOICE_NONE = 'none'
+DOCTEST_REPORT_CHOICE_CDIFF = 'cdiff'
+DOCTEST_REPORT_CHOICE_NDIFF = 'ndiff'
+DOCTEST_REPORT_CHOICE_UDIFF = 'udiff'
+DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE = 'only_first_failure'
+
+DOCTEST_REPORT_CHOICES = (
+    DOCTEST_REPORT_CHOICE_NONE,
+    DOCTEST_REPORT_CHOICE_CDIFF,
+    DOCTEST_REPORT_CHOICE_NDIFF,
+    DOCTEST_REPORT_CHOICE_UDIFF,
+    DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE,
+)
 
 def pytest_addoption(parser):
     parser.addini('doctest_optionflags', 'option flags for doctests',
@@ -17,6 +30,11 @@ def pytest_addoption(parser):
         action="store_true", default=False,
         help="run doctests in all .py modules",
         dest="doctestmodules")
+    group.addoption("--doctest-report",
+        type=str.lower, default="udiff",
+        help="choose another output format for diffs on doctest failure",
+        choices=DOCTEST_REPORT_CHOICES,
+        dest="doctestreport")
     group.addoption("--doctest-glob",
         action="append", default=[], metavar="pat",
         help="doctests file matching pattern, default: test*.txt",
@@ -59,7 +77,6 @@ class ReprFailDoctest(TerminalRepr):
 
 
 class DoctestItem(pytest.Item):
-
     def __init__(self, name, parent, runner=None, dtest=None):
         super(DoctestItem, self).__init__(name, parent)
         self.runner = runner
@@ -94,7 +111,7 @@ class DoctestItem(pytest.Item):
             message = excinfo.type.__name__
             reprlocation = ReprFileLocation(filename, lineno, message)
             checker = _get_checker()
-            REPORT_UDIFF = doctest.REPORT_UDIFF
+            report_choice = _get_report_choice(self.config.getoption("doctestreport"))
             if lineno is not None:
                 lines = doctestfailure.test.docstring.splitlines(False)
                 # add line numbers to the left of the error message
@@ -110,7 +127,7 @@ class DoctestItem(pytest.Item):
                     indent = '...'
             if excinfo.errisinstance(doctest.DocTestFailure):
                 lines += checker.output_difference(example,
-                        doctestfailure.got, REPORT_UDIFF).split("\n")
+                        doctestfailure.got, report_choice).split("\n")
             else:
                 inner_excinfo = ExceptionInfo(excinfo.value.exc_info)
                 lines += ["UNEXPECTED EXCEPTION: %s" %
@@ -291,6 +308,21 @@ def _get_allow_bytes_flag():
     return doctest.register_optionflag('ALLOW_BYTES')
 
 
+def _get_report_choice(key):
+    """
+    This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid
+    importing `doctest` and all its dependencies when parsing options, as it adds overhead and breaks tests.
+    """
+    import doctest
+
+    return {
+        DOCTEST_REPORT_CHOICE_UDIFF: doctest.REPORT_UDIFF,
+        DOCTEST_REPORT_CHOICE_CDIFF: doctest.REPORT_CDIFF,
+        DOCTEST_REPORT_CHOICE_NDIFF: doctest.REPORT_NDIFF,
+        DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE: doctest.REPORT_ONLY_FIRST_FAILURE,
+        DOCTEST_REPORT_CHOICE_NONE: 0,
+    }[key]
+
 @pytest.fixture(scope='session')
 def doctest_namespace():
     """

doc/en/doctest.rst

@@ -16,7 +16,6 @@ from docstrings in all python modules (including regular
 python test modules)::
 
     pytest --doctest-modules
-
 You can make these changes permanent in your project by
 putting them into a pytest.ini file like this:
 
@@ -102,6 +101,7 @@ itself::
     >>> get_unicode_greeting()  # doctest: +ALLOW_UNICODE
     'Hello'
 
+
 The 'doctest_namespace' fixture
 -------------------------------
 
@@ -130,3 +130,22 @@ which can then be used in your doctests directly::
         10
         """
         pass
+
+
+Output format
+-------------
+
+.. versionadded:: 3.0
+
+You can change the diff output format on failure for your doctests
+by using one of standard doctest modules format in options
+(see :data:`python:doctest.REPORT_UDIFF`, :data:`python:doctest.REPORT_CDIFF`,
+:data:`python:doctest.REPORT_NDIFF`, :data:`python:doctest.REPORT_ONLY_FIRST_FAILURE`)::
+
+    pytest --doctest-modules --doctest-report none
+    pytest --doctest-modules --doctest-report udiff
+    pytest --doctest-modules --doctest-report cdiff
+    pytest --doctest-modules --doctest-report ndiff
+    pytest --doctest-modules --doctest-report only_first_failure
+
+

testing/test_doctest.py

@@ -784,3 +784,81 @@ class TestDoctestNamespaceFixture:
         """)
         reprec = testdir.inline_run(p, "--doctest-modules")
         reprec.assertoutcome(passed=1)
+
+
+class TestDoctestReportingOption:
+    def _run_doctest_report(self, testdir, format):
+        testdir.makepyfile("""
+            def foo():
+                '''
+                >>> foo()
+                   a  b
+                0  1  4
+                1  2  4
+                2  3  6
+                '''
+                print('   a  b\\n'
+                      '0  1  4\\n'
+                      '1  2  5\\n'
+                      '2  3  6')
+            """)
+        return testdir.runpytest("--doctest-modules", "--doctest-report", format)
+
+    @pytest.mark.parametrize('format', ['udiff', 'UDIFF', 'uDiFf'])
+    def test_doctest_report_udiff(self, testdir, format):
+        result = self._run_doctest_report(testdir, format)
+        result.stdout.fnmatch_lines([
+            '     0  1  4',
+            '    -1  2  4',
+            '    +1  2  5',
+            '     2  3  6',
+        ])
+
+    def test_doctest_report_cdiff(self, testdir):
+        result = self._run_doctest_report(testdir, 'cdiff')
+        result.stdout.fnmatch_lines([
+            '         a  b',
+            '      0  1  4',
+            '    ! 1  2  4',
+            '      2  3  6',
+            '    --- 1,4 ----',
+            '         a  b',
+            '      0  1  4',
+            '    ! 1  2  5',
+            '      2  3  6',
+        ])
+
+    def test_doctest_report_ndiff(self, testdir):
+        result = self._run_doctest_report(testdir, 'ndiff')
+        result.stdout.fnmatch_lines([
+            '         a  b',
+            '      0  1  4',
+            '    - 1  2  4',
+            '    ?       ^',
+            '    + 1  2  5',
+            '    ?       ^',
+            '      2  3  6',
+        ])
+
+    @pytest.mark.parametrize('format', ['none', 'only_first_failure'])
+    def test_doctest_report_none_or_only_first_failure(self, testdir, format):
+        result = self._run_doctest_report(testdir, format)
+        result.stdout.fnmatch_lines([
+            'Expected:',
+            '       a  b',
+            '    0  1  4',
+            '    1  2  4',
+            '    2  3  6',
+            'Got:',
+            '       a  b',
+            '    0  1  4',
+            '    1  2  5',
+            '    2  3  6',
+        ])
+
+    def test_doctest_report_invalid(self, testdir):
+        result = self._run_doctest_report(testdir, 'obviously_invalid_format')
+        result.stderr.fnmatch_lines([
+            "*error: argument --doctest-report: invalid choice: 'obviously_invalid_format' (choose from*"
+        ])
+