diff --git a/AUTHORS b/AUTHORS index 3f73b9272..140f4de3d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -71,6 +71,7 @@ Danielle Jenkins Dave Hunt David Díaz-Barquero David Mohr +David Paul Röthlisberger David Szotten David Vierra Daw-Ran Liou diff --git a/changelog/5576.feature.rst b/changelog/5576.feature.rst new file mode 100644 index 000000000..267a28292 --- /dev/null +++ b/changelog/5576.feature.rst @@ -0,0 +1,4 @@ +New `NUMBER `__ +option for doctests to ignore irrelevant differences in floating-point numbers. +Inspired by Sébastien Boisgérault's `numtest `__ +extension for doctest. diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 3c40036ef..819266c57 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -103,7 +103,7 @@ that will be used for those doctest files using the Using 'doctest' options ----------------------- -The standard ``doctest`` module provides some `options `__ +Python's standard ``doctest`` module provides some `options `__ to configure the strictness of doctest tests. In pytest, you can enable those flags using the configuration file. @@ -115,23 +115,50 @@ lengthy exception stack traces you can just write: [pytest] doctest_optionflags= NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL -pytest also introduces new options to allow doctests to run in Python 2 and -Python 3 unchanged: - -* ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode - strings in expected doctest output. - -* ``ALLOW_BYTES``: when enabled, the ``b`` prefix is stripped from byte strings - in expected doctest output. - Alternatively, options can be enabled by an inline comment in the doc test itself: .. code-block:: rst - # content of example.rst - >>> get_unicode_greeting() # doctest: +ALLOW_UNICODE - 'Hello' + >>> something_that_raises() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + +pytest also introduces new options: + +* ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode + strings in expected doctest output. This allows doctests to run in Python 2 + and Python 3 unchanged. + +* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings + in expected doctest output. + +* ``NUMBER``: when enabled, floating-point numbers only need to match as far as + the precision you have written in the expected doctest output. For example, + the following output would only need to match to 2 decimal places:: + + >>> math.pi + 3.14 + + If you wrote ``3.1416`` then the actual output would need to match to 4 + decimal places; and so on. + + This avoids false positives caused by limited floating-point precision, like + this:: + + Expected: + 0.233 + Got: + 0.23300000000000001 + + ``NUMBER`` also supports lists of floating-point numbers -- in fact, it + matches floating-point numbers appearing anywhere in the output, even inside + a string! This means that it may not be appropriate to enable globally in + ``doctest_optionflags`` in your configuration file. + + +Continue on failure +------------------- By default, pytest would report only the first failure for a given doctest. If you want to continue the test even when you have failures, do: diff --git a/src/_pytest/doctest.py b/src/_pytest/doctest.py index ca6e4675f..cf886f906 100644 --- a/src/_pytest/doctest.py +++ b/src/_pytest/doctest.py @@ -13,6 +13,7 @@ from _pytest._code.code import TerminalRepr from _pytest.compat import safe_getattr from _pytest.fixtures import FixtureRequest from _pytest.outcomes import Skipped +from _pytest.python_api import approx from _pytest.warning_types import PytestWarning DOCTEST_REPORT_CHOICE_NONE = "none" @@ -286,6 +287,7 @@ def _get_flag_lookup(): COMPARISON_FLAGS=doctest.COMPARISON_FLAGS, ALLOW_UNICODE=_get_allow_unicode_flag(), ALLOW_BYTES=_get_allow_bytes_flag(), + NUMBER=_get_number_flag(), ) @@ -453,10 +455,15 @@ def _setup_fixtures(doctest_item): def _get_checker(): """ - Returns a doctest.OutputChecker subclass that takes in account the - ALLOW_UNICODE option to ignore u'' prefixes in strings and ALLOW_BYTES - to strip b'' prefixes. - Useful when the same doctest should run in Python 2 and Python 3. + Returns a doctest.OutputChecker subclass that supports some + additional options: + + * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b'' + prefixes (respectively) in string literals. Useful when the same + doctest should run in Python 2 and Python 3. + + * NUMBER to ignore floating-point differences smaller than the + precision of the literal number in the doctest. An inner class is used to avoid importing "doctest" at the module level. @@ -469,38 +476,89 @@ def _get_checker(): class LiteralsOutputChecker(doctest.OutputChecker): """ - Copied from doctest_nose_plugin.py from the nltk project: - https://github.com/nltk/nltk - - Further extended to also support byte literals. + Based on doctest_nose_plugin.py from the nltk project + (https://github.com/nltk/nltk) and on the "numtest" doctest extension + by Sebastien Boisgerault (https://github.com/boisgera/numtest). """ _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE) _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE) + _number_re = re.compile( + r""" + (?P + (?P + (?P [+-]?\d*)\.(?P\d+) + | + (?P [+-]?\d+)\. + ) + (?: + [Ee] + (?P [+-]?\d+) + )? + | + (?P [+-]?\d+) + (?: + [Ee] + (?P [+-]?\d+) + ) + ) + """, + re.VERBOSE, + ) def check_output(self, want, got, optionflags): - res = doctest.OutputChecker.check_output(self, want, got, optionflags) - if res: + if doctest.OutputChecker.check_output(self, want, got, optionflags): return True allow_unicode = optionflags & _get_allow_unicode_flag() allow_bytes = optionflags & _get_allow_bytes_flag() - if not allow_unicode and not allow_bytes: + allow_number = optionflags & _get_number_flag() + + if not allow_unicode and not allow_bytes and not allow_number: return False - else: # pragma: no cover + def remove_prefixes(regex, txt): + return re.sub(regex, r"\1\2", txt) - def remove_prefixes(regex, txt): - return re.sub(regex, r"\1\2", txt) + if allow_unicode: + want = remove_prefixes(self._unicode_literal_re, want) + got = remove_prefixes(self._unicode_literal_re, got) - if allow_unicode: - want = remove_prefixes(self._unicode_literal_re, want) - got = remove_prefixes(self._unicode_literal_re, got) - if allow_bytes: - want = remove_prefixes(self._bytes_literal_re, want) - got = remove_prefixes(self._bytes_literal_re, got) - res = doctest.OutputChecker.check_output(self, want, got, optionflags) - return res + if allow_bytes: + want = remove_prefixes(self._bytes_literal_re, want) + got = remove_prefixes(self._bytes_literal_re, got) + + if allow_number: + got = self._remove_unwanted_precision(want, got) + + return doctest.OutputChecker.check_output(self, want, got, optionflags) + + def _remove_unwanted_precision(self, want, got): + wants = list(self._number_re.finditer(want)) + gots = list(self._number_re.finditer(got)) + if len(wants) != len(gots): + return got + offset = 0 + for w, g in zip(wants, gots): + fraction = w.group("fraction") + exponent = w.group("exponent1") + if exponent is None: + exponent = w.group("exponent2") + if fraction is None: + precision = 0 + else: + precision = len(fraction) + if exponent is not None: + precision -= int(exponent) + if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): + # They're close enough. Replace the text we actually + # got with the text we want, so that it will match when we + # check the string literally. + got = ( + got[: g.start() + offset] + w.group() + got[g.end() + offset :] + ) + offset += w.end() - w.start() - (g.end() - g.start()) + return got _get_checker.LiteralsOutputChecker = LiteralsOutputChecker return _get_checker.LiteralsOutputChecker() @@ -524,6 +582,15 @@ def _get_allow_bytes_flag(): return doctest.register_optionflag("ALLOW_BYTES") +def _get_number_flag(): + """ + Registers and returns the NUMBER flag. + """ + import doctest + + return doctest.register_optionflag("NUMBER") + + def _get_report_choice(key): """ This function returns the actual `doctest` module flag value, we want to do it as late as possible to avoid diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 236066673..4aac5432d 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -3,6 +3,7 @@ import textwrap import pytest from _pytest.compat import MODULE_NOT_FOUND_ERROR +from _pytest.doctest import _get_checker from _pytest.doctest import _is_mocked from _pytest.doctest import _patch_unwrap_mock_aware from _pytest.doctest import DoctestItem @@ -838,6 +839,154 @@ class TestLiterals: reprec = testdir.inline_run() reprec.assertoutcome(failed=1) + def test_number_re(self): + for s in [ + "1.", + "+1.", + "-1.", + ".1", + "+.1", + "-.1", + "0.1", + "+0.1", + "-0.1", + "1e5", + "+1e5", + "1e+5", + "+1e+5", + "1e-5", + "+1e-5", + "-1e-5", + "1.2e3", + "-1.2e-3", + ]: + print(s) + m = _get_checker()._number_re.match(s) + assert m is not None + assert float(m.group()) == pytest.approx(float(s)) + for s in ["1", "abc"]: + print(s) + assert _get_checker()._number_re.match(s) is None + + @pytest.mark.parametrize("config_mode", ["ini", "comment"]) + def test_number_precision(self, testdir, config_mode): + """Test the NUMBER option.""" + if config_mode == "ini": + testdir.makeini( + """ + [pytest] + doctest_optionflags = NUMBER + """ + ) + comment = "" + else: + comment = "#doctest: +NUMBER" + + testdir.maketxtfile( + test_doc=""" + + Scalars: + + >>> import math + >>> math.pi {comment} + 3.141592653589793 + >>> math.pi {comment} + 3.1416 + >>> math.pi {comment} + 3.14 + >>> -math.pi {comment} + -3.14 + >>> math.pi {comment} + 3. + >>> 3. {comment} + 3.0 + >>> 3. {comment} + 3. + >>> 3. {comment} + 3.01 + >>> 3. {comment} + 2.99 + >>> .299 {comment} + .3 + >>> .301 {comment} + .3 + >>> 951. {comment} + 1e3 + >>> 1049. {comment} + 1e3 + >>> -1049. {comment} + -1e3 + >>> 1e3 {comment} + 1e3 + >>> 1e3 {comment} + 1000. + + Lists: + + >>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment} + [3.14, 0.1, 13., 7, 8.22e5, 6.0e-3] + >>> [[0.333, 0.667], [0.999, 1.333]] {comment} + [[0.33, 0.667], [0.999, 1.333]] + >>> [[[0.101]]] {comment} + [[[0.1]]] + + Doesn't barf on non-numbers: + + >>> 'abc' {comment} + 'abc' + >>> None {comment} + """.format( + comment=comment + ) + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize( + "expression,output", + [ + # ints shouldn't match floats: + ("3.0", "3"), + ("3e0", "3"), + ("1e3", "1000"), + ("3", "3.0"), + # Rounding: + ("3.1", "3.0"), + ("3.1", "3.2"), + ("3.1", "4.0"), + ("8.22e5", "810000.0"), + # Only the actual output is rounded up, not the expected output: + ("3.0", "2.98"), + ("1e3", "999"), + # The current implementation doesn't understand that numbers inside + # strings shouldn't be treated as numbers: + pytest.param("'3.1416'", "'3.14'", marks=pytest.mark.xfail), + ], + ) + def test_number_non_matches(self, testdir, expression, output): + testdir.maketxtfile( + test_doc=""" + >>> {expression} #doctest: +NUMBER + {output} + """.format( + expression=expression, output=output + ) + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=0, failed=1) + + def test_number_and_allow_unicode(self, testdir): + testdir.maketxtfile( + test_doc=""" + >>> from collections import namedtuple + >>> T = namedtuple('T', 'a b c') + >>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER + T(a=0.233, b=u'str', c='bytes') + """ + ) + reprec = testdir.inline_run() + reprec.assertoutcome(passed=1) + class TestDoctestSkips: """