p53209761
/
test_ok2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed #3149 where doctest does not continue to run when there is a failure

This commit is contained in:
William Lee 2018-02-22 21:00:54 -06:00
parent 7336dbb979
commit fbc45be83f
3 changed files with 161 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -195,6 +195,7 @@ Victor Uriarte
Vidar T. Fauske
Vitaly Lashmanov
Vlad Dragos
William Lee
Wouter van Ackooy
Xuan Luong
Xuecong Liao

136
_pytest/doctest.py
View File

@ -24,6 +24,9 @@ DOCTEST_REPORT_CHOICES = (
DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE,
)
# Lazy definiton of runner class
RUNNER_CLASS = None
def pytest_addoption(parser):
parser.addini('doctest_optionflags', 'option flags for doctests',
@ -77,14 +80,91 @@ def _is_doctest(config, path, parent):
class ReprFailDoctest(TerminalRepr):
def __init__(self, reprlocation, lines):
self.reprlocation = reprlocation
self.lines = lines
def __init__(self, reprlocation_lines):
# List of (reprlocation, lines) tuples
self.reprlocation_lines = reprlocation_lines
def toterminal(self, tw):
for line in self.lines:
for reprlocation, lines in self.reprlocation_lines:
for line in lines:
tw.line(line)
self.reprlocation.toterminal(tw)
reprlocation.toterminal(tw)
# class DoctestFailureContainer(object):
#
# NAME = 'DocTestFailure'
#
# def __init__(self, test, example, got):
# self.test = test
# self.example = example
# self.got = got
#
#
# class DoctestUnexpectedExceptionContainer(object):
#
# NAME = 'DoctestUnexpectedException'
#
# def __init__(self, test, example, exc_info):
# self.test = test
# self.example = example
# self.exc_info = exc_info
class MultipleDoctestFailures(Exception):
def __init__(self, failures):
super(MultipleDoctestFailures, self).__init__()
self.failures = failures
def _init_runner_class():
import doctest
class PytestDoctestRunner(doctest.DocTestRunner):
"""
Runner to collect failures. Note that the out variable in this case is
a list instead of a stdout-like object
"""
def __init__(self, checker=None, verbose=None, optionflags=0,
continue_on_failure=True):
doctest.DocTestRunner.__init__(
self, checker=checker, verbose=verbose, optionflags=optionflags)
self.continue_on_failure = continue_on_failure
def report_start(self, out, test, example):
pass
def report_success(self, out, test, example, got):
pass
def report_failure(self, out, test, example, got):
# failure = DoctestFailureContainer(test, example, got)
failure = doctest.DocTestFailure(test, example, got)
if self.continue_on_failure:
out.append(failure)
else:
raise failure
def report_unexpected_exception(self, out, test, example, exc_info):
# failure = DoctestUnexpectedExceptionContainer(test, example, exc_info)
failure = doctest.UnexpectedException(test, example, exc_info)
if self.continue_on_failure:
out.append(failure)
else:
raise failure
return PytestDoctestRunner
def _get_runner(checker=None, verbose=None, optionflags=0,
continue_on_failure=True):
# We need this in order to do a lazy import on doctest
global RUNNER_CLASS
if RUNNER_CLASS is None:
RUNNER_CLASS = _init_runner_class()
return RUNNER_CLASS(
checker=checker, verbose=verbose, optionflags=optionflags,
continue_on_failure=continue_on_failure)
class DoctestItem(pytest.Item):
@ -106,7 +186,10 @@ class DoctestItem(pytest.Item):
def runtest(self):
_check_all_skipped(self.dtest)
self._disable_output_capturing_for_darwin()
self.runner.run(self.dtest)
failures = []
self.runner.run(self.dtest, out=failures)
if failures:
raise MultipleDoctestFailures(failures)
def _disable_output_capturing_for_darwin(self):
"""
@ -122,22 +205,29 @@ class DoctestItem(pytest.Item):
def repr_failure(self, excinfo):
import doctest
failures = None
if excinfo.errisinstance((doctest.DocTestFailure,
doctest.UnexpectedException)):
doctestfailure = excinfo.value
example = doctestfailure.example
test = doctestfailure.test
failures = [excinfo.value]
elif excinfo.errisinstance(MultipleDoctestFailures):
failures = excinfo.value.failures
if failures is not None:
reprlocation_lines = []
for failure in failures:
example = failure.example
test = failure.test
filename = test.filename
if test.lineno is None:
lineno = None
else:
lineno = test.lineno + example.lineno + 1
message = excinfo.type.__name__
message = type(failure).__name__
reprlocation = ReprFileLocation(filename, lineno, message)
checker = _get_checker()
report_choice = _get_report_choice(self.config.getoption("doctestreport"))
if lineno is not None:
lines = doctestfailure.test.docstring.splitlines(False)
lines = failure.test.docstring.splitlines(False)
# add line numbers to the left of the error message
lines = ["%03d %s" % (i + test.lineno + 1, x)
for (i, x) in enumerate(lines)]
@ -149,15 +239,17 @@ class DoctestItem(pytest.Item):
for line in example.source.splitlines():
lines.append('??? %s %s' % (indent, line))
indent = '...'
if excinfo.errisinstance(doctest.DocTestFailure):
if isinstance(failure, doctest.DocTestFailure):
lines += checker.output_difference(example,
doctestfailure.got, report_choice).split("\n")
failure.got,
report_choice).split("\n")
else:
inner_excinfo = ExceptionInfo(excinfo.value.exc_info)
inner_excinfo = ExceptionInfo(failure.exc_info)
lines += ["UNEXPECTED EXCEPTION: %s" %
repr(inner_excinfo.value)]
lines += traceback.format_exception(*excinfo.value.exc_info)
return ReprFailDoctest(reprlocation, lines)
lines += traceback.format_exception(*failure.exc_info)
reprlocation_lines.append((reprlocation, lines))
return ReprFailDoctest(reprlocation_lines)
else:
return super(DoctestItem, self).repr_failure(excinfo)
@ -202,8 +294,10 @@ class DoctestTextfile(pytest.Module):
globs = {'__name__': '__main__'}
optionflags = get_optionflags(self)
runner = doctest.DebugRunner(verbose=0, optionflags=optionflags,
checker=_get_checker())
continue_on_failure = not self.config.getvalue("usepdb")
runner = _get_runner(verbose=0, optionflags=optionflags,
checker=_get_checker(),
continue_on_failure=continue_on_failure)
_fix_spoof_python2(runner, encoding)
parser = doctest.DocTestParser()
@ -238,8 +332,10 @@ class DoctestModule(pytest.Module):
# uses internal doctest module parsing mechanism
finder = doctest.DocTestFinder()
optionflags = get_optionflags(self)
runner = doctest.DebugRunner(verbose=0, optionflags=optionflags,
checker=_get_checker())
continue_on_failure = not self.config.getvalue("usepdb")
runner = _get_runner(verbose=0, optionflags=optionflags,
checker=_get_checker(),
continue_on_failure=continue_on_failure)
for test in finder.find(module, module.__name__):
if test.examples: # skip empty doctests

19
testing/test_doctest.py
View File

@ -756,6 +756,25 @@ class TestDoctestSkips(object):
reprec = testdir.inline_run("--doctest-modules")
reprec.assertoutcome(passed=0, skipped=0)
def test_continue_on_failure(self, testdir):
testdir.maketxtfile(test_something="""
>>> i = 5
>>> def foo():
... raise ValueError('error1')
>>> foo()
>>> i
>>> i + 2
7
>>> i + 1
""")
result = testdir.runpytest("--doctest-modules")
result.assert_outcomes(passed=0, failed=1)
# We need to make sure we have two failure lines (4, 5, and 8) instead of
# one.
result.stdout.fnmatch_lines("*test_something.txt:4: DoctestUnexpectedException*")
result.stdout.fnmatch_lines("*test_something.txt:5: DocTestFailure*")
result.stdout.fnmatch_lines("*test_something.txt:8: DocTestFailure*")
class TestDoctestAutoUseFixtures(object):