doctest: Add +NUMBER option to ignore irrelevant floating-point differences
When enabled, floating-point numbers only need to match as far as the
precision you have written in the expected doctest output. This avoids
false positives caused by limited floating-point precision, like this:
Expected:
0.233
Got:
0.23300000000000001
This is inspired by Sébastien Boisgérault's [numtest] but the
implementation is a bit different:
* This implementation edits the literals that are in the "got"
string (the actual output from the expression being tested), and then
proceeds to compare the strings literally. This is similar to pytest's
existing ALLOW_UNICODE and ALLOW_BYTES implementation.
* This implementation only compares floats against floats, not ints
against floats. That is, the following doctest will fail with pytest
whereas it would pass with numtest:
>>> math.py # doctest: +NUMBER
3
This behaviour should be less surprising (less false negatives) when
you enable NUMBER globally in pytest.ini.
Advantages of this implementation compared to numtest:
* Doesn't require `import numtest` at the top level of the file.
* Works with pytest (if you try to use pytest & numtest together, pytest
raises "TypeError: unbound method check_output() must be called with
NumTestOutputChecker instance as first argument (got
LiteralsOutputChecker instance instead)").
* Works with Python 3.
[numtest]: https://github.com/boisgera/numtest
This commit is contained in:
David Röthlisberger
parent
60a358fa2d
commit
aaa7e837cc
|
|
@ -115,15 +115,36 @@ lengthy exception stack traces you can just write:
|
|||
[pytest]
|
||||
doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL
|
||||
|
||||
pytest also introduces new options to allow doctests to run in Python 2 and
|
||||
Python 3 unchanged:
|
||||
pytest also introduces new options:
|
||||
|
||||
* ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode
|
||||
strings in expected doctest output.
|
||||
strings in expected doctest output. This allows doctests to run in Python 2
|
||||
and Python 3 unchanged.
|
||||
|
||||
* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings
|
||||
* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings
|
||||
in expected doctest output.
|
||||
|
||||
* ``NUMBER``: when enabled, floating-point numbers only need to match as far as
|
||||
the precision you have written in the expected doctest output. For example,
|
||||
the following output would only need to match to 2 decimal places::
|
||||
|
||||
>>> math.pi
|
||||
3.14
|
||||
|
||||
If you wrote ``3.1416`` then the actual output would need to match to 4
|
||||
decimal places; and so on.
|
||||
|
||||
This avoids false positives caused by limited floating-point precision, like
|
||||
this::
|
||||
|
||||
Expected:
|
||||
0.233
|
||||
Got:
|
||||
0.23300000000000001
|
||||
|
||||
``NUMBER`` also supports lists of floating-point numbers -- in fact, it
|
||||
supports floating-point numbers appearing anywhere in the output.
|
||||
|
||||
Alternatively, options can be enabled by an inline comment in the doc test
|
||||
itself:
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from _pytest._code.code import TerminalRepr
|
|||
from _pytest.compat import safe_getattr
|
||||
from _pytest.fixtures import FixtureRequest
|
||||
from _pytest.outcomes import Skipped
|
||||
from _pytest.python_api import approx
|
||||
from _pytest.warning_types import PytestWarning
|
||||
|
||||
DOCTEST_REPORT_CHOICE_NONE = "none"
|
||||
|
|
@ -286,6 +287,7 @@ def _get_flag_lookup():
|
|||
COMPARISON_FLAGS=doctest.COMPARISON_FLAGS,
|
||||
ALLOW_UNICODE=_get_allow_unicode_flag(),
|
||||
ALLOW_BYTES=_get_allow_bytes_flag(),
|
||||
NUMBER=_get_number_flag(),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -453,10 +455,15 @@ def _setup_fixtures(doctest_item):
|
|||
|
||||
def _get_checker():
|
||||
"""
|
||||
Returns a doctest.OutputChecker subclass that takes in account the
|
||||
ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES
|
||||
to strip b'' prefixes.
|
||||
Useful when the same doctest should run in Python 2 and Python 3.
|
||||
Returns a doctest.OutputChecker subclass that supports some
|
||||
additional options:
|
||||
|
||||
* ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b''
|
||||
prefixes (respectively) in string literals. Useful when the same
|
||||
doctest should run in Python 2 and Python 3.
|
||||
|
||||
* NUMBER to ignore floating-point differences smaller than the
|
||||
precision of the literal number in the doctest.
|
||||
|
||||
An inner class is used to avoid importing "doctest" at the module
|
||||
level.
|
||||
|
|
@ -469,38 +476,89 @@ def _get_checker():
|
|||
|
||||
class LiteralsOutputChecker(doctest.OutputChecker):
|
||||
"""
|
||||
Copied from doctest_nose_plugin.py from the nltk project:
|
||||
https://github.com/nltk/nltk
|
||||
|
||||
Further extended to also support byte literals.
|
||||
Based on doctest_nose_plugin.py from the nltk project
|
||||
(https://github.com/nltk/nltk) and on the "numtest" doctest extension
|
||||
by Sebastien Boisgerault (https://github.com/boisgera/numtest).
|
||||
"""
|
||||
|
||||
_unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)
|
||||
_bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE)
|
||||
_number_re = re.compile(
|
||||
r"""
|
||||
(?P<number>
|
||||
(?P<mantissa>
|
||||
(?P<integer1> [+-]?\d*)\.(?P<fraction>\d+)
|
||||
|
|
||||
(?P<integer2> [+-]?\d+)\.
|
||||
)
|
||||
(?:
|
||||
[Ee]
|
||||
(?P<exponent1> [+-]?\d+)
|
||||
)?
|
||||
|
|
||||
(?P<integer3> [+-]?\d+)
|
||||
(?:
|
||||
[Ee]
|
||||
(?P<exponent2> [+-]?\d+)
|
||||
)
|
||||
)
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
def check_output(self, want, got, optionflags):
|
||||
res = doctest.OutputChecker.check_output(self, want, got, optionflags)
|
||||
if res:
|
||||
if doctest.OutputChecker.check_output(self, want, got, optionflags):
|
||||
return True
|
||||
|
||||
allow_unicode = optionflags & _get_allow_unicode_flag()
|
||||
allow_bytes = optionflags & _get_allow_bytes_flag()
|
||||
if not allow_unicode and not allow_bytes:
|
||||
allow_number = optionflags & _get_number_flag()
|
||||
|
||||
if not allow_unicode and not allow_bytes and not allow_number:
|
||||
return False
|
||||
|
||||
else: # pragma: no cover
|
||||
def remove_prefixes(regex, txt):
|
||||
return re.sub(regex, r"\1\2", txt)
|
||||
|
||||
def remove_prefixes(regex, txt):
|
||||
return re.sub(regex, r"\1\2", txt)
|
||||
if allow_unicode:
|
||||
want = remove_prefixes(self._unicode_literal_re, want)
|
||||
got = remove_prefixes(self._unicode_literal_re, got)
|
||||
|
||||
if allow_unicode:
|
||||
want = remove_prefixes(self._unicode_literal_re, want)
|
||||
got = remove_prefixes(self._unicode_literal_re, got)
|
||||
if allow_bytes:
|
||||
want = remove_prefixes(self._bytes_literal_re, want)
|
||||
got = remove_prefixes(self._bytes_literal_re, got)
|
||||
res = doctest.OutputChecker.check_output(self, want, got, optionflags)
|
||||
return res
|
||||
if allow_bytes:
|
||||
want = remove_prefixes(self._bytes_literal_re, want)
|
||||
got = remove_prefixes(self._bytes_literal_re, got)
|
||||
|
||||
if allow_number:
|
||||
got = self._remove_unwanted_precision(want, got)
|
||||
|
||||
return doctest.OutputChecker.check_output(self, want, got, optionflags)
|
||||
|
||||
def _remove_unwanted_precision(self, want, got):
|
||||
wants = list(self._number_re.finditer(want))
|
||||
gots = list(self._number_re.finditer(got))
|
||||
if len(wants) != len(gots):
|
||||
return got
|
||||
offset = 0
|
||||
for w, g in zip(wants, gots):
|
||||
fraction = w.group("fraction")
|
||||
exponent = w.group("exponent1")
|
||||
if exponent is None:
|
||||
exponent = w.group("exponent2")
|
||||
if fraction is None:
|
||||
precision = 0
|
||||
else:
|
||||
precision = len(fraction)
|
||||
if exponent is not None:
|
||||
precision -= int(exponent)
|
||||
if float(w.group()) == approx(float(g.group()), abs=10 ** -precision):
|
||||
# They're close enough. Replace the text we actually
|
||||
# got with the text we want, so that it will match when we
|
||||
# check the string literally.
|
||||
got = (
|
||||
got[: g.start() + offset] + w.group() + got[g.end() + offset :]
|
||||
)
|
||||
offset += w.end() - w.start() - (g.end() - g.start())
|
||||
return got
|
||||
|
||||
_get_checker.LiteralsOutputChecker = LiteralsOutputChecker
|
||||
return _get_checker.LiteralsOutputChecker()
|
||||
|
|
@ -524,6 +582,15 @@ def _get_allow_bytes_flag():
|
|||
return doctest.register_optionflag("ALLOW_BYTES")
|
||||
|
||||
|
||||
def _get_number_flag():
|
||||
"""
|
||||
Registers and returns the NUMBER flag.
|
||||
"""
|
||||
import doctest
|
||||
|
||||
return doctest.register_optionflag("NUMBER")
|
||||
|
||||
|
||||
def _get_report_choice(key):
|
||||
"""
|
||||
This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import textwrap
|
|||
|
||||
import pytest
|
||||
from _pytest.compat import MODULE_NOT_FOUND_ERROR
|
||||
from _pytest.doctest import _get_checker
|
||||
from _pytest.doctest import _is_mocked
|
||||
from _pytest.doctest import _patch_unwrap_mock_aware
|
||||
from _pytest.doctest import DoctestItem
|
||||
|
|
@ -838,6 +839,149 @@ class TestLiterals:
|
|||
reprec = testdir.inline_run()
|
||||
reprec.assertoutcome(failed=1)
|
||||
|
||||
def test_number_re(self):
|
||||
for s in [
|
||||
"1.",
|
||||
"+1.",
|
||||
"-1.",
|
||||
".1",
|
||||
"+.1",
|
||||
"-.1",
|
||||
"0.1",
|
||||
"+0.1",
|
||||
"-0.1",
|
||||
"1e5",
|
||||
"+1e5",
|
||||
"1e+5",
|
||||
"+1e+5",
|
||||
"1e-5",
|
||||
"+1e-5",
|
||||
"-1e-5",
|
||||
"1.2e3",
|
||||
"-1.2e-3",
|
||||
]:
|
||||
print(s)
|
||||
m = _get_checker()._number_re.match(s)
|
||||
assert m is not None
|
||||
assert float(m.group()) == pytest.approx(float(s))
|
||||
for s in ["1", "abc"]:
|
||||
print(s)
|
||||
assert _get_checker()._number_re.match(s) is None
|
||||
|
||||
@pytest.mark.parametrize("config_mode", ["ini", "comment"])
|
||||
def test_number_precision(self, testdir, config_mode):
|
||||
"""Test the NUMBER option."""
|
||||
if config_mode == "ini":
|
||||
testdir.makeini(
|
||||
"""
|
||||
[pytest]
|
||||
doctest_optionflags = NUMBER
|
||||
"""
|
||||
)
|
||||
comment = ""
|
||||
else:
|
||||
comment = "#doctest: +NUMBER"
|
||||
|
||||
testdir.maketxtfile(
|
||||
test_doc="""
|
||||
|
||||
Scalars:
|
||||
|
||||
>>> import math
|
||||
>>> math.pi {comment}
|
||||
3.141592653589793
|
||||
>>> math.pi {comment}
|
||||
3.1416
|
||||
>>> math.pi {comment}
|
||||
3.14
|
||||
>>> -math.pi {comment}
|
||||
-3.14
|
||||
>>> math.pi {comment}
|
||||
3.
|
||||
>>> 3. {comment}
|
||||
3.0
|
||||
>>> 3. {comment}
|
||||
3.
|
||||
>>> 3. {comment}
|
||||
3.01
|
||||
>>> 3. {comment}
|
||||
2.99
|
||||
>>> .299 {comment}
|
||||
.3
|
||||
>>> .301 {comment}
|
||||
.3
|
||||
>>> 951. {comment}
|
||||
1e3
|
||||
>>> 1049. {comment}
|
||||
1e3
|
||||
>>> -1049. {comment}
|
||||
-1e3
|
||||
>>> 1e3 {comment}
|
||||
1e3
|
||||
>>> 1e3 {comment}
|
||||
1000.
|
||||
|
||||
Lists:
|
||||
|
||||
>>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment}
|
||||
[3.14, 0.1, 13., 7, 8.22e5, 6.0e-3]
|
||||
>>> [[0.333, 0.667], [0.999, 1.333]] {comment}
|
||||
[[0.33, 0.667], [0.999, 1.333]]
|
||||
>>> [[[0.101]]] {comment}
|
||||
[[[0.1]]]
|
||||
|
||||
Doesn't barf on non-numbers:
|
||||
|
||||
>>> 'abc' {comment}
|
||||
'abc'
|
||||
>>> None {comment}
|
||||
""".format(
|
||||
comment=comment
|
||||
)
|
||||
)
|
||||
reprec = testdir.inline_run()
|
||||
reprec.assertoutcome(passed=1)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,output",
|
||||
[
|
||||
# ints shouldn't match floats:
|
||||
("3.0", "3"),
|
||||
("3e0", "3"),
|
||||
("1e3", "1000"),
|
||||
("3", "3.0"),
|
||||
# Rounding:
|
||||
("3.1", "3.0"),
|
||||
("3.1", "3.2"),
|
||||
("3.1", "4.0"),
|
||||
("8.22e5", "810000.0"),
|
||||
# Only the actual output is rounded up, not the expected output:
|
||||
("3.0", "2.99"),
|
||||
("1e3", "999"),
|
||||
],
|
||||
)
|
||||
def test_number_non_matches(self, testdir, expression, output):
|
||||
testdir.maketxtfile(
|
||||
test_doc="""
|
||||
>>> {expression} #doctest: +NUMBER
|
||||
{output}
|
||||
"""
|
||||
)
|
||||
reprec = testdir.inline_run()
|
||||
reprec.assertoutcome(passed=0, failed=1)
|
||||
|
||||
def test_number_and_allow_unicode(self, testdir):
|
||||
testdir.maketxtfile(
|
||||
test_doc="""
|
||||
>>> from collections import namedtuple
|
||||
>>> T = namedtuple('T', 'a b c')
|
||||
>>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER
|
||||
T(a=0.233, b=u'str', c='bytes')
|
||||
"""
|
||||
)
|
||||
reprec = testdir.inline_run()
|
||||
reprec.assertoutcome(passed=1)
|
||||
|
||||
|
||||
class TestDoctestSkips:
|
||||
"""
|
||||
|
|
|
|||
Loading…
Reference in New Issue