Fixed #3149 where doctest does not continue to run when there is a failure

AUTHORS

@@ -195,6 +195,7 @@ Victor Uriarte
 Vidar T. Fauske
 Vitaly Lashmanov
 Vlad Dragos
+William Lee
 Wouter van Ackooy
 Xuan Luong
 Xuecong Liao

_pytest/doctest.py

@@ -24,6 +24,9 @@ DOCTEST_REPORT_CHOICES = (
     DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE,
 )
 
+# Lazy definiton of runner class
+RUNNER_CLASS = None
+
 
 def pytest_addoption(parser):
     parser.addini('doctest_optionflags', 'option flags for doctests',
@@ -77,14 +80,91 @@ def _is_doctest(config, path, parent):
 
 class ReprFailDoctest(TerminalRepr):
 
-    def __init__(self, reprlocation, lines):
+    def __init__(self, reprlocation_lines):
-        self.reprlocation = reprlocation
+        # List of (reprlocation, lines) tuples
-        self.lines = lines
+        self.reprlocation_lines = reprlocation_lines
 
     def toterminal(self, tw):
-        for line in self.lines:
+        for reprlocation, lines in self.reprlocation_lines:
-            tw.line(line)
+            for line in lines:
-        self.reprlocation.toterminal(tw)
+                tw.line(line)
+            reprlocation.toterminal(tw)
+
+
+# class DoctestFailureContainer(object):
+#
+#     NAME = 'DocTestFailure'
+#
+#     def __init__(self, test, example, got):
+#         self.test = test
+#         self.example = example
+#         self.got = got
+#
+#
+# class DoctestUnexpectedExceptionContainer(object):
+#
+#     NAME = 'DoctestUnexpectedException'
+#
+#     def __init__(self, test, example, exc_info):
+#         self.test = test
+#         self.example = example
+#         self.exc_info = exc_info
+
+
+class MultipleDoctestFailures(Exception):
+    def __init__(self, failures):
+        super(MultipleDoctestFailures, self).__init__()
+        self.failures = failures
+
+
+def _init_runner_class():
+    import doctest
+
+    class PytestDoctestRunner(doctest.DocTestRunner):
+        """
+        Runner to collect failures.  Note that the out variable in this case is
+        a list instead of a stdout-like object
+        """
+        def __init__(self, checker=None, verbose=None, optionflags=0,
+                     continue_on_failure=True):
+            doctest.DocTestRunner.__init__(
+                self, checker=checker, verbose=verbose, optionflags=optionflags)
+            self.continue_on_failure = continue_on_failure
+
+        def report_start(self, out, test, example):
+            pass
+
+        def report_success(self, out, test, example, got):
+            pass
+
+        def report_failure(self, out, test, example, got):
+            # failure = DoctestFailureContainer(test, example, got)
+            failure = doctest.DocTestFailure(test, example, got)
+            if self.continue_on_failure:
+                out.append(failure)
+            else:
+                raise failure
+
+        def report_unexpected_exception(self, out, test, example, exc_info):
+            # failure = DoctestUnexpectedExceptionContainer(test, example, exc_info)
+            failure = doctest.UnexpectedException(test, example, exc_info)
+            if self.continue_on_failure:
+                out.append(failure)
+            else:
+                raise failure
+
+    return PytestDoctestRunner
+
+
+def _get_runner(checker=None, verbose=None, optionflags=0,
+                continue_on_failure=True):
+    # We need this in order to do a lazy import on doctest
+    global RUNNER_CLASS
+    if RUNNER_CLASS is None:
+        RUNNER_CLASS = _init_runner_class()
+    return RUNNER_CLASS(
+        checker=checker, verbose=verbose, optionflags=optionflags,
+        continue_on_failure=continue_on_failure)
 
 
 class DoctestItem(pytest.Item):
@@ -106,7 +186,10 @@ class DoctestItem(pytest.Item):
     def runtest(self):
         _check_all_skipped(self.dtest)
         self._disable_output_capturing_for_darwin()
-        self.runner.run(self.dtest)
+        failures = []
+        self.runner.run(self.dtest, out=failures)
+        if failures:
+            raise MultipleDoctestFailures(failures)
 
     def _disable_output_capturing_for_darwin(self):
         """
@@ -122,42 +205,51 @@ class DoctestItem(pytest.Item):
 
     def repr_failure(self, excinfo):
         import doctest
+        failures = None
         if excinfo.errisinstance((doctest.DocTestFailure,
                                   doctest.UnexpectedException)):
-            doctestfailure = excinfo.value
+            failures = [excinfo.value]
-            example = doctestfailure.example
+        elif excinfo.errisinstance(MultipleDoctestFailures):
-            test = doctestfailure.test
+            failures = excinfo.value.failures
-            filename = test.filename
+
-            if test.lineno is None:
+        if failures is not None:
-                lineno = None
+            reprlocation_lines = []
-            else:
+            for failure in failures:
-                lineno = test.lineno + example.lineno + 1
+                example = failure.example
-            message = excinfo.type.__name__
+                test = failure.test
-            reprlocation = ReprFileLocation(filename, lineno, message)
+                filename = test.filename
-            checker = _get_checker()
+                if test.lineno is None:
-            report_choice = _get_report_choice(self.config.getoption("doctestreport"))
+                    lineno = None
-            if lineno is not None:
+                else:
-                lines = doctestfailure.test.docstring.splitlines(False)
+                    lineno = test.lineno + example.lineno + 1
-                # add line numbers to the left of the error message
+                message = type(failure).__name__
-                lines = ["%03d %s" % (i + test.lineno + 1, x)
+                reprlocation = ReprFileLocation(filename, lineno, message)
-                         for (i, x) in enumerate(lines)]
+                checker = _get_checker()
-                # trim docstring error lines to 10
+                report_choice = _get_report_choice(self.config.getoption("doctestreport"))
-                lines = lines[max(example.lineno - 9, 0):example.lineno + 1]
+                if lineno is not None:
-            else:
+                    lines = failure.test.docstring.splitlines(False)
-                lines = ['EXAMPLE LOCATION UNKNOWN, not showing all tests of that example']
+                    # add line numbers to the left of the error message
-                indent = '>>>'
+                    lines = ["%03d %s" % (i + test.lineno + 1, x)
-                for line in example.source.splitlines():
+                             for (i, x) in enumerate(lines)]
-                    lines.append('??? %s %s' % (indent, line))
+                    # trim docstring error lines to 10
-                    indent = '...'
+                    lines = lines[max(example.lineno - 9, 0):example.lineno + 1]
-            if excinfo.errisinstance(doctest.DocTestFailure):
+                else:
-                lines += checker.output_difference(example,
+                    lines = ['EXAMPLE LOCATION UNKNOWN, not showing all tests of that example']
-                                                   doctestfailure.got, report_choice).split("\n")
+                    indent = '>>>'
-            else:
+                    for line in example.source.splitlines():
-                inner_excinfo = ExceptionInfo(excinfo.value.exc_info)
+                        lines.append('??? %s %s' % (indent, line))
-                lines += ["UNEXPECTED EXCEPTION: %s" %
+                        indent = '...'
-                          repr(inner_excinfo.value)]
+                if isinstance(failure, doctest.DocTestFailure):
-                lines += traceback.format_exception(*excinfo.value.exc_info)
+                    lines += checker.output_difference(example,
-            return ReprFailDoctest(reprlocation, lines)
+                                                       failure.got,
+                                                       report_choice).split("\n")
+                else:
+                    inner_excinfo = ExceptionInfo(failure.exc_info)
+                    lines += ["UNEXPECTED EXCEPTION: %s" %
+                              repr(inner_excinfo.value)]
+                    lines += traceback.format_exception(*failure.exc_info)
+                reprlocation_lines.append((reprlocation, lines))
+            return ReprFailDoctest(reprlocation_lines)
         else:
             return super(DoctestItem, self).repr_failure(excinfo)
 
@@ -202,8 +294,10 @@ class DoctestTextfile(pytest.Module):
         globs = {'__name__': '__main__'}
 
         optionflags = get_optionflags(self)
-        runner = doctest.DebugRunner(verbose=0, optionflags=optionflags,
+        continue_on_failure = not self.config.getvalue("usepdb")
-                                     checker=_get_checker())
+        runner = _get_runner(verbose=0, optionflags=optionflags,
+                             checker=_get_checker(),
+                             continue_on_failure=continue_on_failure)
         _fix_spoof_python2(runner, encoding)
 
         parser = doctest.DocTestParser()
@@ -238,8 +332,10 @@ class DoctestModule(pytest.Module):
         # uses internal doctest module parsing mechanism
         finder = doctest.DocTestFinder()
         optionflags = get_optionflags(self)
-        runner = doctest.DebugRunner(verbose=0, optionflags=optionflags,
+        continue_on_failure = not self.config.getvalue("usepdb")
-                                     checker=_get_checker())
+        runner = _get_runner(verbose=0, optionflags=optionflags,
+                             checker=_get_checker(),
+                             continue_on_failure=continue_on_failure)
 
         for test in finder.find(module, module.__name__):
             if test.examples:  # skip empty doctests

testing/test_doctest.py

@@ -756,6 +756,25 @@ class TestDoctestSkips(object):
         reprec = testdir.inline_run("--doctest-modules")
         reprec.assertoutcome(passed=0, skipped=0)
 
+    def test_continue_on_failure(self, testdir):
+        testdir.maketxtfile(test_something="""
+            >>> i = 5
+            >>> def foo():
+            ...     raise ValueError('error1')
+            >>> foo()
+            >>> i
+            >>> i + 2
+            7
+            >>> i + 1
+        """)
+        result = testdir.runpytest("--doctest-modules")
+        result.assert_outcomes(passed=0, failed=1)
+        # We need to make sure we have two failure lines (4, 5, and 8) instead of
+        # one.
+        result.stdout.fnmatch_lines("*test_something.txt:4: DoctestUnexpectedException*")
+        result.stdout.fnmatch_lines("*test_something.txt:5: DocTestFailure*")
+        result.stdout.fnmatch_lines("*test_something.txt:8: DocTestFailure*")
+
 
 class TestDoctestAutoUseFixtures(object):