From c8c5a416ef8b849868fa077918567f401be96b8d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 1 Mar 2016 18:48:13 -0300 Subject: [PATCH 01/23] Bump version to 2.10.0.dev1 --- _pytest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/__init__.py b/_pytest/__init__.py index 7ca204091..79942e374 100644 --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.9.0' +__version__ = '2.10.0.dev1' From 5a2500800d591babb822073b1ec9497a375be023 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 1 Mar 2016 18:54:08 -0300 Subject: [PATCH 02/23] Add 2.10.0.dev1 to CHANGELOG --- CHANGELOG.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c4dc1cb7..b5b400f9c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,33 @@ +2.10.0.dev1 +=========== + +**New Features** + +* + +* + +* + + +**Changes** + +* + +* + +* + + +**Bug Fixes** + +* + +* + +* + + 2.9.0 ===== From 891e0295183d6cd8ca8b6e58af65c397cb7e2197 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 2 Mar 2016 12:43:57 +0000 Subject: [PATCH 03/23] Add a new doctest_namespace fixture This fixture can be used to inject names into the namespace in which your doctests run. --- AUTHORS | 1 + CHANGELOG.rst | 3 ++- _pytest/doctest.py | 13 +++++++++++ doc/en/doctest.rst | 25 +++++++++++++++++++++ testing/test_doctest.py | 50 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b9868ef2a..47f137892 100644 --- a/AUTHORS +++ b/AUTHORS @@ -59,6 +59,7 @@ Marc Schlaich Mark Abramowitz Markus Unterwaditzer Martijn Faassen +Matt Williams Michael Aquilina Michael Birtwell Michael Droettboom diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b5b400f9c..a203ae2c8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,7 +3,8 @@ **New Features** -* +* New ``doctest_namespace`` fixture for injecting names into the + namespace in which your doctests run. * diff --git a/_pytest/doctest.py b/_pytest/doctest.py index a57f7a494..4050d1ba7 100644 --- a/_pytest/doctest.py +++ b/_pytest/doctest.py @@ -71,6 +71,8 @@ class DoctestItem(pytest.Item): if self.dtest is not None: self.fixture_request = _setup_fixtures(self) globs = dict(getfixture=self.fixture_request.getfuncargvalue) + for name, value in self.fixture_request.getfuncargvalue('doctest_namespace').items(): + globs[name] = value self.dtest.globs.update(globs) def runtest(self): @@ -159,6 +161,9 @@ class DoctestTextfile(DoctestItem, pytest.Module): if '__name__' not in globs: globs['__name__'] = '__main__' + for name, value in fixture_request.getfuncargvalue('doctest_namespace').items(): + globs[name] = value + optionflags = get_optionflags(self) runner = doctest.DebugRunner(verbose=0, optionflags=optionflags, checker=_get_checker()) @@ -288,3 +293,11 @@ def _get_allow_bytes_flag(): """ import doctest return doctest.register_optionflag('ALLOW_BYTES') + + +@pytest.fixture(scope='session') +def doctest_namespace(): + """ + Inject names into the doctest namespace. + """ + return dict() diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index f13752e66..4798d9714 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -102,4 +102,29 @@ itself:: >>> get_unicode_greeting() # doctest: +ALLOW_UNICODE 'Hello' +The 'doctest_namespace' fixture +------------------------------- +The ``doctest_namespace`` fixture can be used to inject items into the +namespace in which your doctests run. It is intended to be used within +your own fixtures to provide the tests that use them with context. + +``doctest_namespace`` is a standard ``dict`` object into which you +place the objects you want to appear in the doctest namespace:: + + # content of conftest.py + import numpy + @pytest.fixture(autouse=True) + def add_np(doctest_namespace): + doctest_namespace['np'] = numpy + +which can then be used in your doctests directly:: + + # content of numpy.py + def arange(): + """ + >>> a = np.arange(10) + >>> len(a) + 10 + """ + pass diff --git a/testing/test_doctest.py b/testing/test_doctest.py index a4821ee4c..d104d98d3 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -713,3 +713,53 @@ class TestDoctestAutoUseFixtures: result = testdir.runpytest('--doctest-modules') assert 'FAILURES' not in str(result.stdout.str()) result.stdout.fnmatch_lines(['*=== 1 passed in *']) + + +class TestDoctestNamespaceFixture: + + SCOPES = ['module', 'session', 'class', 'function'] + + @pytest.mark.parametrize('scope', SCOPES) + def test_namespace_doctestfile(self, testdir, scope): + """ + Check that inserting something into the namespace works in a + simple text file doctest + """ + testdir.makeconftest(""" + import pytest + import contextlib + + @pytest.fixture(autouse=True, scope="{scope}") + def add_contextlib(doctest_namespace): + doctest_namespace['cl'] = contextlib + """.format(scope=scope)) + p = testdir.maketxtfile(""" + >>> print(cl.__name__) + contextlib + """) + reprec = testdir.inline_run(p) + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize('scope', SCOPES) + def test_namespace_pyfile(self, testdir, scope): + """ + Check that inserting something into the namespace works in a + simple Python file docstring doctest + """ + testdir.makeconftest(""" + import pytest + import contextlib + + @pytest.fixture(autouse=True, scope="{scope}") + def add_contextlib(doctest_namespace): + doctest_namespace['cl'] = contextlib + """.format(scope=scope)) + p = testdir.makepyfile(""" + def foo(): + ''' + >>> print(cl.__name__) + contextlib + ''' + """) + reprec = testdir.inline_run(p, "--doctest-modules") + reprec.assertoutcome(passed=1) From 6dd2ff5332b735487743a92b280ad276164536b1 Mon Sep 17 00:00:00 2001 From: Matt Williams Date: Wed, 2 Mar 2016 13:02:15 +0000 Subject: [PATCH 04/23] Correct indentation in documentation --- doc/en/doctest.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 4798d9714..32e37008f 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -115,8 +115,8 @@ place the objects you want to appear in the doctest namespace:: # content of conftest.py import numpy @pytest.fixture(autouse=True) - def add_np(doctest_namespace): - doctest_namespace['np'] = numpy + def add_np(doctest_namespace): + doctest_namespace['np'] = numpy which can then be used in your doctests directly:: From 2e02a1c370f7b42baf3a2e2ab9ff0b314ef15cdd Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 2 Mar 2016 23:30:42 -0300 Subject: [PATCH 05/23] Give proper credit for PR #1428 --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a203ae2c8..b53379e17 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ * New ``doctest_namespace`` fixture for injecting names into the namespace in which your doctests run. + Thanks `@milliams`_ for the complete PR (`#1428`_). * @@ -29,6 +30,11 @@ * +.. _@milliams: https://github.com/milliams + +.. _#1428: https://github.com/pytest-dev/pytest/pull/1428 + + 2.9.0 ===== From 28937a5cd90c7d4faf65cf9933d5e8deff6c1fc6 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 2 Mar 2016 23:37:51 -0300 Subject: [PATCH 06/23] Add versionadded directive to doctest_namespace section --- doc/en/doctest.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/en/doctest.rst b/doc/en/doctest.rst index 32e37008f..64b5621d4 100644 --- a/doc/en/doctest.rst +++ b/doc/en/doctest.rst @@ -105,6 +105,8 @@ itself:: The 'doctest_namespace' fixture ------------------------------- +.. versionadded:: 2.10 + The ``doctest_namespace`` fixture can be used to inject items into the namespace in which your doctests run. It is intended to be used within your own fixtures to provide the tests that use them with context. From 6f5e1e386a685cc5975762f0a0457738a3522b13 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sun, 6 Mar 2016 18:53:48 -0800 Subject: [PATCH 07/23] Add a convenient and correct way to compare floats. --- _pytest/python.py | 102 ++++++++++++++++++++++++++++++++++++++- testing/python/approx.py | 27 +++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 testing/python/approx.py diff --git a/_pytest/python.py b/_pytest/python.py index ec346f587..3d458ed82 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -261,7 +261,8 @@ def pytest_namespace(): return { 'fixture': fixture, 'yield_fixture': yield_fixture, - 'raises' : raises, + 'raises': raises, + 'approx': approx, 'collect': { 'Module': Module, 'Class': Class, 'Instance': Instance, 'Function': Function, 'Generator': Generator, @@ -1336,6 +1337,105 @@ class RaisesContext(object): self.excinfo.__init__(tp) return issubclass(self.excinfo.type, self.expected_exception) +# builtin pytest.approx helper + +class approx: + """ assert that two numbers (or two sets of numbers) are equal to each + other within some margin. + + Due to the intricacies of floating-point arithmetic, numbers that we would + intuitively expect to be the same are not always so:: + + >>> 0.1 + 0.2 == 0.3 + False + + This problem is commonly encountered when writing tests, e.g. to make sure + that a floating-point function returns the expected values. The best way + to deal with this problem is to assert that two floating point numbers are + equal to within some appropriate margin:: + + >>> abs((0.1 + 0.2) - 0.3) < 1e-6 + True + + However, comparisons like this are tedious to write and difficult to + understand. Furthermore, absolute comparisons like the one above are + usually discouraged in favor of relative comparisons, which can't even be + easily written on one line. The ``approx`` class provides a way to make + floating point comparisons that solves both these problems:: + + >>> from pytest import approx + >>> 0.1 + 0.2 == approx(0.3) + True + + ``approx`` also makes is easy to compare ordered sets of numbers, which + would otherwise be very tedious:: + + >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) + True + + By default, ``approx`` considers two numbers to be equal if the relative + error between them is less than one part in a million (e.g. 1e-6). + Relative error is defined as ``abs(x - a) / x`` where ``x`` is the value + you're expecting and ``a`` is the value you're comparing to. This + definition breaks down when the numbers being compared get very close to + zero, so ``approx`` will also consider two numbers to be equal if the + absolute difference between them is less than 1e-100. + + Both the relative and absolute error thresholds can be changed by passing + arguments to the ``approx`` constructor:: + + >>> 1.0001 == approx(1) + False + >>> 1.0001 == approx(1, rel=1e-3) + True + >>> 1.0001 == approx(1, abs=1e-3) + True + + Note that if you specify ``abs`` but not ``rel``, the comparison will not + consider the relative error between the two values at all. In other words, + two number that are within the default relative error threshold of 1e-6 + will still be considered unequal if they exceed the specified absolute + error threshold:: + + >>> 0.1 + 0.2 == approx(0.3, abs=1e-100) + False + """ + + def __init__(self, expected, rel=None, abs=None): + self.expected = expected + self.max_relative_error = rel + self.max_absolute_error = abs + + def __repr__(self): + from collections import Iterable + plus_minus = lambda x: '{}\u00B1{}'.format(x, self._margin(x)) + + if isinstance(self.expected, Iterable): + return str([plus_minus(x) for x in self.expected]) + else: + plus_minus(self.expected) + + def __eq__(self, actual): + from collections import Iterable + expected = self.expected + almost_eq = lambda a, x: abs(x - a) < self._margin(x) + + if isinstance(actual, Iterable) and isinstance(expected, Iterable): + return all(almost_eq(a, x) for a, x in zip(actual, expected)) + else: + return almost_eq(actual, expected) + + def _margin(self, x): + margin = self.max_absolute_error or 1e-100 + + if self.max_relative_error is None: + if self.max_absolute_error is not None: + return margin + + return max(margin, x * (self.max_relative_error or 1e-6)) + + + # # the basic pytest Function item # diff --git a/testing/python/approx.py b/testing/python/approx.py new file mode 100644 index 000000000..76064114d --- /dev/null +++ b/testing/python/approx.py @@ -0,0 +1,27 @@ +import pytest +import doctest + +class MyDocTestRunner(doctest.DocTestRunner): + + def __init__(self): + doctest.DocTestRunner.__init__(self) + + def report_failure(self, out, test, example, got): + raise AssertionError("'{}' evaluates to '{}', not '{}'".format( + example.source.strip(), got.strip(), example.want.strip())) + + +class TestApprox: + + def test_approx(self): + parser = doctest.DocTestParser() + test = parser.get_doctest( + pytest.approx.__doc__, + {'approx': pytest.approx}, + pytest.approx.__name__, + None, None, + ) + runner = MyDocTestRunner() + runner.run(test) + + From dd28e28b34a2d37dc41219b73ce5462ff6d86e10 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 10:09:20 -0800 Subject: [PATCH 08/23] Make a few stylistic improvements. --- _pytest/python.py | 15 ++++++++------- testing/python/approx.py | 7 ++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 3d458ed82..f214aef94 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1349,8 +1349,8 @@ class approx: >>> 0.1 + 0.2 == 0.3 False - This problem is commonly encountered when writing tests, e.g. to make sure - that a floating-point function returns the expected values. The best way + This problem is commonly encountered when writing tests, e.g. when making + sure that a floating-point function returns the expected values. One way to deal with this problem is to assert that two floating point numbers are equal to within some appropriate margin:: @@ -1399,6 +1399,8 @@ class approx: >>> 0.1 + 0.2 == approx(0.3, abs=1e-100) False + >>> 0.1 + 0.2 == approx(0.3, rel=1e-6, abs=1e-100) + True """ def __init__(self, expected, rel=None, abs=None): @@ -1408,24 +1410,24 @@ class approx: def __repr__(self): from collections import Iterable - plus_minus = lambda x: '{}\u00B1{}'.format(x, self._margin(x)) + plus_minus = lambda x: '{} \u00B1 {:.1e}'.format(x, self._get_margin(x)) if isinstance(self.expected, Iterable): return str([plus_minus(x) for x in self.expected]) else: - plus_minus(self.expected) + return plus_minus(self.expected) def __eq__(self, actual): from collections import Iterable expected = self.expected - almost_eq = lambda a, x: abs(x - a) < self._margin(x) + almost_eq = lambda a, x: abs(x - a) < self._get_margin(x) if isinstance(actual, Iterable) and isinstance(expected, Iterable): return all(almost_eq(a, x) for a, x in zip(actual, expected)) else: return almost_eq(actual, expected) - def _margin(self, x): + def _get_margin(self, x): margin = self.max_absolute_error or 1e-100 if self.max_relative_error is None: @@ -1435,7 +1437,6 @@ class approx: return max(margin, x * (self.max_relative_error or 1e-6)) - # # the basic pytest Function item # diff --git a/testing/python/approx.py b/testing/python/approx.py index 76064114d..432ca4748 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + import pytest import doctest @@ -13,7 +15,7 @@ class MyDocTestRunner(doctest.DocTestRunner): class TestApprox: - def test_approx(self): + def test_approx_doctests(self): parser = doctest.DocTestParser() test = parser.get_doctest( pytest.approx.__doc__, @@ -24,4 +26,7 @@ class TestApprox: runner = MyDocTestRunner() runner.run(test) + def test_repr_string(self): + print(pytest.approx(1.0)) + assert repr(pytest.approx(1.0)) == '1.0 ± 1.0e-06' From bf97d5b81735c6bdf85b27deeac2f83f94d436ad Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 16:40:41 -0800 Subject: [PATCH 09/23] Use the plus/minus unicode symbol in the repr string. This was a challenge because it had to work in python2 and python3, which have almost opposite unicode models, and I couldn't use the six library. I'm also not sure the solution I found would work in python3 before python3.3, because I use the u'' string prefix which I think was initially not part of python3. --- _pytest/python.py | 7 ++++--- testing/python/approx.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index f214aef94..6189ef362 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1339,7 +1339,7 @@ class RaisesContext(object): # builtin pytest.approx helper -class approx: +class approx(object): """ assert that two numbers (or two sets of numbers) are equal to each other within some margin. @@ -1410,10 +1410,11 @@ class approx: def __repr__(self): from collections import Iterable - plus_minus = lambda x: '{} \u00B1 {:.1e}'.format(x, self._get_margin(x)) + utf_8 = lambda s: s.encode('utf-8') if sys.version_info.major == 2 else s + plus_minus = lambda x: utf_8(u'{} \u00b1 {:.1e}'.format(x, self._get_margin(x))) if isinstance(self.expected, Iterable): - return str([plus_minus(x) for x in self.expected]) + return ', '.join([plus_minus(x) for x in self.expected]) else: return plus_minus(self.expected) diff --git a/testing/python/approx.py b/testing/python/approx.py index 432ca4748..2f37c9b9b 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -27,6 +27,7 @@ class TestApprox: runner.run(test) def test_repr_string(self): + # Just make sure the Unicode handling doesn't raise any exceptions. print(pytest.approx(1.0)) - assert repr(pytest.approx(1.0)) == '1.0 ± 1.0e-06' + print(pytest.approx([1.0, 2.0, 3.0])) From b8a8382c2cb0c81cae6cf6b8744fe65e13331296 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 16:43:53 -0800 Subject: [PATCH 10/23] Reduce the default absolute error threshold to 1e-12. --- _pytest/python.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 6189ef362..2a89696f7 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1,4 +1,5 @@ """ Python test discovery, setup and run of test functions. """ + import fnmatch import functools import inspect @@ -1379,7 +1380,7 @@ class approx(object): you're expecting and ``a`` is the value you're comparing to. This definition breaks down when the numbers being compared get very close to zero, so ``approx`` will also consider two numbers to be equal if the - absolute difference between them is less than 1e-100. + absolute difference between them is less than 1e-12. Both the relative and absolute error thresholds can be changed by passing arguments to the ``approx`` constructor:: @@ -1393,13 +1394,15 @@ class approx(object): Note that if you specify ``abs`` but not ``rel``, the comparison will not consider the relative error between the two values at all. In other words, - two number that are within the default relative error threshold of 1e-6 + two numbers that are within the default relative error threshold of 1e-6 will still be considered unequal if they exceed the specified absolute error threshold:: - >>> 0.1 + 0.2 == approx(0.3, abs=1e-100) + >>> 1 + 1e-8 == approx(1) + True + >>> 1 + 1e-8 == approx(1, abs=1e-12) False - >>> 0.1 + 0.2 == approx(0.3, rel=1e-6, abs=1e-100) + >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) True """ @@ -1429,7 +1432,7 @@ class approx(object): return almost_eq(actual, expected) def _get_margin(self, x): - margin = self.max_absolute_error or 1e-100 + margin = self.max_absolute_error or 1e-12 if self.max_relative_error is None: if self.max_absolute_error is not None: From 5dab0954a097592226652c1dab762e7aa676759a Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 18:14:49 -0800 Subject: [PATCH 11/23] Add approx() to the Sphinx docs. --- _pytest/python.py | 20 ++++++++++++-------- doc/en/builtin.rst | 7 ++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 2a89696f7..86b5bd4a3 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1205,7 +1205,8 @@ def getlocation(function, curdir): # builtin pytest.raises helper def raises(expected_exception, *args, **kwargs): - """ assert that a code block/function call raises ``expected_exception`` + """ + Assert that a code block/function call raises ``expected_exception`` and raise a failure exception otherwise. This helper produces a ``ExceptionInfo()`` object (see below). @@ -1341,7 +1342,8 @@ class RaisesContext(object): # builtin pytest.approx helper class approx(object): - """ assert that two numbers (or two sets of numbers) are equal to each + """ + Assert that two numbers (or two sets of numbers) are equal to each other within some margin. Due to the intricacies of floating-point arithmetic, numbers that we would @@ -1351,8 +1353,8 @@ class approx(object): False This problem is commonly encountered when writing tests, e.g. when making - sure that a floating-point function returns the expected values. One way - to deal with this problem is to assert that two floating point numbers are + sure that floating-point values are what you expect them to be. One way to + deal with this problem is to assert that two floating-point numbers are equal to within some appropriate margin:: >>> abs((0.1 + 0.2) - 0.3) < 1e-6 @@ -1362,7 +1364,7 @@ class approx(object): understand. Furthermore, absolute comparisons like the one above are usually discouraged in favor of relative comparisons, which can't even be easily written on one line. The ``approx`` class provides a way to make - floating point comparisons that solves both these problems:: + floating-point comparisons that solves both these problems:: >>> from pytest import approx >>> 0.1 + 0.2 == approx(0.3) @@ -1375,12 +1377,13 @@ class approx(object): True By default, ``approx`` considers two numbers to be equal if the relative - error between them is less than one part in a million (e.g. 1e-6). + error between them is less than one part in a million (e.g. ``1e-6``). Relative error is defined as ``abs(x - a) / x`` where ``x`` is the value you're expecting and ``a`` is the value you're comparing to. This definition breaks down when the numbers being compared get very close to zero, so ``approx`` will also consider two numbers to be equal if the - absolute difference between them is less than 1e-12. + absolute difference between them is less than one part in a trillion (e.g. + ``1e-12``). Both the relative and absolute error thresholds can be changed by passing arguments to the ``approx`` constructor:: @@ -1396,7 +1399,8 @@ class approx(object): consider the relative error between the two values at all. In other words, two numbers that are within the default relative error threshold of 1e-6 will still be considered unequal if they exceed the specified absolute - error threshold:: + error threshold. If you specify both ``abs`` and ``rel``, the numbers will + be considered equal if either threshold is met:: >>> 1 + 1e-8 == approx(1) True diff --git a/doc/en/builtin.rst b/doc/en/builtin.rst index b18c3f828..d364dd56d 100644 --- a/doc/en/builtin.rst +++ b/doc/en/builtin.rst @@ -35,6 +35,11 @@ Examples at :ref:`assertraises`. .. autofunction:: deprecated_call +Comparing floating point numbers +-------------------------------- + +.. autoclass:: approx + Raising a specific test outcome -------------------------------------- @@ -48,7 +53,7 @@ you can rather use declarative marks, see :ref:`skipping`. .. autofunction:: _pytest.skipping.xfail .. autofunction:: _pytest.runner.exit -fixtures and requests +Fixtures and requests ----------------------------------------------------- To mark a fixture function: From 4d0f066db7a4d2884c1f70373013616994eac33b Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 18:29:22 -0800 Subject: [PATCH 12/23] Add approx() to the CHANGELOG. --- AUTHORS | 1 + CHANGELOG.rst | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 47f137892..40b455ff1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -50,6 +50,7 @@ Jason R. Coombs Joshua Bronson Jurko Gospodnetić Katarzyna Jachim +Kale Kundert Kevin Cox Lee Kamentsky Lukas Bednar diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 34fc1916d..7ee14abd0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,8 @@ namespace in which your doctests run. Thanks `@milliams`_ for the complete PR (`#1428`_). -* +* New ``approx()`` function for easily comparing floating-point numbers in + tests. * From c9c73b8d8e8e985864569d93e012145c870c0929 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 19:54:43 -0800 Subject: [PATCH 13/23] Fix zero-length field name error in python2.6 --- _pytest/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/python.py b/_pytest/python.py index 86b5bd4a3..1a3582d60 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1418,7 +1418,7 @@ class approx(object): def __repr__(self): from collections import Iterable utf_8 = lambda s: s.encode('utf-8') if sys.version_info.major == 2 else s - plus_minus = lambda x: utf_8(u'{} \u00b1 {:.1e}'.format(x, self._get_margin(x))) + plus_minus = lambda x: utf_8(u'{0} \u00b1 {1:.1e}'.format(x, self._get_margin(x))) if isinstance(self.expected, Iterable): return ', '.join([plus_minus(x) for x in self.expected]) From 6a902924f85c94002ea0dbe4bb0fb1201f7e1a70 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 7 Mar 2016 19:56:23 -0800 Subject: [PATCH 14/23] Fix trailing whitespace errors. --- _pytest/python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 1a3582d60..590b1afe1 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1382,7 +1382,7 @@ class approx(object): you're expecting and ``a`` is the value you're comparing to. This definition breaks down when the numbers being compared get very close to zero, so ``approx`` will also consider two numbers to be equal if the - absolute difference between them is less than one part in a trillion (e.g. + absolute difference between them is less than one part in a trillion (e.g. ``1e-12``). Both the relative and absolute error thresholds can be changed by passing @@ -1399,7 +1399,7 @@ class approx(object): consider the relative error between the two values at all. In other words, two numbers that are within the default relative error threshold of 1e-6 will still be considered unequal if they exceed the specified absolute - error threshold. If you specify both ``abs`` and ``rel``, the numbers will + error threshold. If you specify both ``abs`` and ``rel``, the numbers will be considered equal if either threshold is met:: >>> 1 + 1e-8 == approx(1) From 7d155bd3cff913dffbc5c813ee62ee05e2027eb1 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Tue, 8 Mar 2016 10:12:31 -0800 Subject: [PATCH 15/23] Fix sys.version_info errors. --- _pytest/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/python.py b/_pytest/python.py index 590b1afe1..1bed796d1 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1417,7 +1417,7 @@ class approx(object): def __repr__(self): from collections import Iterable - utf_8 = lambda s: s.encode('utf-8') if sys.version_info.major == 2 else s + utf_8 = lambda s: s.encode('utf-8') if sys.version_info[0] == 2 else s plus_minus = lambda x: utf_8(u'{0} \u00b1 {1:.1e}'.format(x, self._get_margin(x))) if isinstance(self.expected, Iterable): From 957712059206ea0994780719b98f38bc9e3dfcd4 Mon Sep 17 00:00:00 2001 From: Mike Lundy Date: Tue, 8 Mar 2016 18:16:57 -0800 Subject: [PATCH 16/23] Allow custom fixture names for fixtures When defining a fixture in the same module as where it is used, the function argument shadows the fixture name, which a) annoys pylint and b) can lead to bugs where you forget to request a fixture into a test method. This allows one to define fixtures with a different name than the name of the function, bypassing that problem. --- AUTHORS | 1 + CHANGELOG.rst | 8 +++++++- _pytest/python.py | 18 ++++++++++++++---- testing/python/fixture.py | 11 +++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 47f137892..a6021c311 100644 --- a/AUTHORS +++ b/AUTHORS @@ -63,6 +63,7 @@ Matt Williams Michael Aquilina Michael Birtwell Michael Droettboom +Mike Lundy Nicolas Delaby Pieter Mulder Piotr Banaszkiewicz diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 34fc1916d..0e7107337 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,11 @@ namespace in which your doctests run. Thanks `@milliams`_ for the complete PR (`#1428`_). -* +* New ``name`` argument to ``pytest.fixture`` mark, which allows a custom name + for a fixture (to solve the funcarg-shadowing-fixture problem). + Thanks `@novas0x2a`_ for the complete PR (`#1444`_). + +* * @@ -21,8 +25,10 @@ * .. _@milliams: https://github.com/milliams +.. _@novas0x2a: https://github.com/novas0x2a .. _#1428: https://github.com/pytest-dev/pytest/pull/1428 +.. _#1444: https://github.com/pytest-dev/pytest/pull/1444 2.9.1.dev1 diff --git a/_pytest/python.py b/_pytest/python.py index ec346f587..1d1d9e158 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -114,12 +114,13 @@ def safe_getattr(object, name, default): class FixtureFunctionMarker: def __init__(self, scope, params, - autouse=False, yieldctx=False, ids=None): + autouse=False, yieldctx=False, ids=None, name=None): self.scope = scope self.params = params self.autouse = autouse self.yieldctx = yieldctx self.ids = ids + self.name = name def __call__(self, function): if isclass(function): @@ -129,7 +130,7 @@ class FixtureFunctionMarker: return function -def fixture(scope="function", params=None, autouse=False, ids=None): +def fixture(scope="function", params=None, autouse=False, ids=None, name=None): """ (return a) decorator to mark a fixture factory function. This decorator can be used (with or or without parameters) to define @@ -155,14 +156,21 @@ def fixture(scope="function", params=None, autouse=False, ids=None): so that they are part of the test id. If no ids are provided they will be generated automatically from the params. + :arg name: the name of the fixture. This defaults to the name of the + decorated function. If a fixture is used in the same module in + which it is defined, the function name of the fixture will be + shadowed by the function arg that requests the fixture; one way + to resolve this is to name the decorated function + ``fixture_`` and then use + ``@pytest.fixture(name='')``. """ if callable(scope) and params is None and autouse == False: # direct decoration return FixtureFunctionMarker( - "function", params, autouse)(scope) + "function", params, autouse, name=name)(scope) if params is not None and not isinstance(params, (list, tuple)): params = list(params) - return FixtureFunctionMarker(scope, params, autouse, ids=ids) + return FixtureFunctionMarker(scope, params, autouse, ids=ids, name=name) def yield_fixture(scope="function", params=None, autouse=False, ids=None): """ (return a) decorator to mark a yield-fixture factory function @@ -1989,6 +1997,8 @@ class FixtureManager: # fixture attribute continue else: + if marker.name: + name = marker.name assert not name.startswith(self._argprefix) fixturedef = FixtureDef(self, nodeid, name, obj, marker.scope, marker.params, diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 506d8426e..eb8f9f34b 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -2691,3 +2691,14 @@ class TestContextManagerFixtureFuncs: *def arg1* """) + def test_custom_name(self, testdir): + testdir.makepyfile(""" + import pytest + @pytest.fixture(name='meow') + def arg1(): + return 'mew' + def test_1(meow): + print(meow) + """) + result = testdir.runpytest("-s") + result.stdout.fnmatch_lines("*mew*") From 42a7e0488da56f2cb0e5468af95bb1f124b05e65 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Fri, 11 Mar 2016 08:49:26 -0800 Subject: [PATCH 17/23] Properly handle inf, nan, and built-in numeric types. This commit also: - Dramatically increases the number of unit tests , mostly by borrowing from the standard library's unit tests for math.isclose(). - Refactors approx() into two classes, one of which handles comparing individual numbers (ApproxNonIterable) and another which uses the first to compare individual numbers or sequences of numbers. --- _pytest/python.py | 126 +++++++++++++++--- testing/python/approx.py | 268 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 364 insertions(+), 30 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 1bed796d1..fe1758204 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -6,6 +6,7 @@ import inspect import re import types import sys +import math import py import pytest @@ -1412,37 +1413,120 @@ class approx(object): def __init__(self, expected, rel=None, abs=None): self.expected = expected - self.max_relative_error = rel - self.max_absolute_error = abs + self.abs = abs + self.rel = rel def __repr__(self): - from collections import Iterable - utf_8 = lambda s: s.encode('utf-8') if sys.version_info[0] == 2 else s - plus_minus = lambda x: utf_8(u'{0} \u00b1 {1:.1e}'.format(x, self._get_margin(x))) - - if isinstance(self.expected, Iterable): - return ', '.join([plus_minus(x) for x in self.expected]) - else: - return plus_minus(self.expected) + return ', '.join(repr(x) for x in self.expected) def __eq__(self, actual): from collections import Iterable - expected = self.expected - almost_eq = lambda a, x: abs(x - a) < self._get_margin(x) + if not isinstance(actual, Iterable): actual = [actual] + if len(actual) != len(self.expected): return False + return all(a == x for a, x in zip(actual, self.expected)) - if isinstance(actual, Iterable) and isinstance(expected, Iterable): - return all(almost_eq(a, x) for a, x in zip(actual, expected)) + @property + def expected(self): + from collections import Iterable + approx_non_iter = lambda x: ApproxNonIterable(x, self.rel, self.abs) + if isinstance(self._expected, Iterable): + return [approx_non_iter(x) for x in self._expected] else: - return almost_eq(actual, expected) + return [approx_non_iter(self._expected)] - def _get_margin(self, x): - margin = self.max_absolute_error or 1e-12 + @expected.setter + def expected(self, expected): + self._expected = expected + - if self.max_relative_error is None: - if self.max_absolute_error is not None: - return margin - return max(margin, x * (self.max_relative_error or 1e-6)) +class ApproxNonIterable(object): + """ + Perform approximate comparisons for single numbers only. + + This class contains most of the + """ + + def __init__(self, expected, rel=None, abs=None): + self.expected = expected + self.abs = abs + self.rel = rel + + def __repr__(self): + # Infinities aren't compared using tolerances, so don't show a + # tolerance. + if math.isinf(self.expected): + return str(self.expected) + + # If a sensible tolerance can't be calculated, self.tolerance will + # raise a ValueError. In this case, display '???'. + try: + vetted_tolerance = '{:.1e}'.format(self.tolerance) + except ValueError: + vetted_tolerance = '???' + + repr = u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) + + # In python2, __repr__() must return a string (i.e. not a unicode + # object). In python3, __repr__() must return a unicode object + # (although now strings are unicode objects and bytes are what + # strings were). + if sys.version_info[0] == 2: + return repr.encode('utf-8') + else: + return repr + + def __eq__(self, actual): + # Short-circuit exact equality. + if actual == self.expected: + return True + + # Infinity shouldn't be approximately equal to anything but itself, but + # if there's a relative tolerance, it will be infinite and infinity + # will seem approximately equal to everything. The equal-to-itself + # case would have been short circuited above, so here we can just + # return false if the expected value is infinite. The abs() call is + # for compatibility with complex numbers. + if math.isinf(abs(self.expected)): + return False + + # Return true if the two numbers are within the tolerance. + return abs(self.expected - actual) <= self.tolerance + + @property + def tolerance(self): + set_default = lambda x, default: x if x is not None else default + + # Figure out what the absolute tolerance should be. ``self.abs`` is + # either None or a value specified by the user. + absolute_tolerance = set_default(self.abs, 1e-12) + + if absolute_tolerance < 0: + raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) + if math.isnan(absolute_tolerance): + raise ValueError("absolute tolerance can't be NaN.") + + # If the user specified an absolute tolerance but not a relative one, + # just return the absolute tolerance. + if self.rel is None: + if self.abs is not None: + return absolute_tolerance + + # Figure out what the absolute tolerance should be. ``self.rel`` is + # either None or a value specified by the user. This is done after + # we've made sure the user didn't ask for an absolute tolerance only, + # because we don't want to raise errors about the relative tolerance if + # it isn't even being used. + relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) + + if relative_tolerance < 0: + raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) + if math.isnan(relative_tolerance): + raise ValueError("relative tolerance can't be NaN.") + + # Return the larger of the relative and absolute tolerances. + return max(relative_tolerance, absolute_tolerance) + # diff --git a/testing/python/approx.py b/testing/python/approx.py index 2f37c9b9b..2d7eb6ea3 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -3,6 +3,12 @@ import pytest import doctest +from pytest import approx +from operator import eq, ne +from decimal import Decimal +from fractions import Fraction +inf, nan = float('inf'), float('nan') + class MyDocTestRunner(doctest.DocTestRunner): def __init__(self): @@ -15,19 +21,263 @@ class MyDocTestRunner(doctest.DocTestRunner): class TestApprox: - def test_approx_doctests(self): + def test_repr_string(self): + # Just make sure the Unicode handling doesn't raise any exceptions. + print(approx(1.0)) + print(approx([1.0, 2.0, 3.0])) + print(approx(inf)) + print(approx(1.0, rel=nan)) + print(approx(1.0, rel=inf)) + + def test_operator_overloading(self): + assert 1 == approx(1, rel=1e-6, abs=1e-12) + assert 10 != approx(1, rel=1e-6, abs=1e-12) + + def test_exactly_equal(self): + examples = [ + (2.0, 2.0), + (0.1e200, 0.1e200), + (1.123e-300, 1.123e-300), + (12345, 12345.0), + (0.0, -0.0), + (345678, 345678), + (Decimal(1.0001), Decimal(1.0001)), + ] + for a, x in examples: + assert a == approx(x) + + def test_opposite_sign(self): + examples = [ + (eq, 1e-100, -1e-100), + (ne, 1e100, -1e100), + ] + for op, a, x in examples: + assert op(a, approx(x)) + + def test_zero_tolerance(self): + within_1e10 = [ + (1.1e-100, 1e-100), + (-1.1e-100, -1e-100), + ] + for a, x in within_1e10: + assert x == approx(x, rel=0.0, abs=0.0) + assert a != approx(x, rel=0.0, abs=0.0) + assert a == approx(x, rel=0.0, abs=5e-101) + assert a != approx(x, rel=0.0, abs=5e-102) + assert a == approx(x, rel=5e-1, abs=0.0) + assert a != approx(x, rel=5e-2, abs=0.0) + + def test_negative_tolerance(self): + # Negative tolerances are not allowed. + illegal_kwargs = [ + dict(rel=-1e100), + dict(abs=-1e100), + dict(rel=1e100, abs=-1e100), + dict(rel=-1e100, abs=1e100), + dict(rel=-1e100, abs=-1e100), + ] + for kwargs in illegal_kwargs: + with pytest.raises(ValueError): + 1.1 == approx(1, **kwargs) + + def test_inf_tolerance(self): + # Everything should be equal if the tolerance is infinite. + large_diffs = [ + (1, 1000), + (1e-50, 1e50), + (-1.0, -1e300), + (0.0, 10), + ] + for a, x in large_diffs: + assert a != approx(x, rel=0.0, abs=0.0) + assert a == approx(x, rel=inf, abs=0.0) + assert a == approx(x, rel=0.0, abs=inf) + assert a == approx(x, rel=inf, abs=inf) + + def test_inf_tolerance_expecting_zero(self): + # If the relative tolerance is zero but the expected value is infinite, + # the actual tolerance is a NaN, which should be an error. + illegal_kwargs = [ + dict(rel=inf, abs=0.0), + dict(rel=inf, abs=inf), + ] + for kwargs in illegal_kwargs: + with pytest.raises(ValueError): + 1 == approx(0, **kwargs) + + def test_nan_tolerance(self): + illegal_kwargs = [ + dict(rel=nan), + dict(abs=nan), + dict(rel=nan, abs=nan), + ] + for kwargs in illegal_kwargs: + with pytest.raises(ValueError): + 1.1 == approx(1, **kwargs) + + def test_reasonable_defaults(self): + # Whatever the defaults are, they should work for numbers close to 1 + # than have a small amount of floating-point error. + assert 0.1 + 0.2 == approx(0.3) + + def test_default_tolerances(self): + # This tests the defaults as they are currently set. If you change the + # defaults, this test will fail but you should feel free to change it. + # None of the other tests (except the doctests) should be affected by + # the choice of defaults. + examples = [ + # Relative tolerance used. + (eq, 1e100 + 1e94, 1e100), + (ne, 1e100 + 2e94, 1e100), + (eq, 1e0 + 1e-6, 1e0), + (ne, 1e0 + 2e-6, 1e0), + # Absolute tolerance used. + (eq, 1e-100, + 1e-106), + (eq, 1e-100, + 2e-106), + (eq, 1e-100, 0), + ] + for op, a, x in examples: + assert op(a, approx(x)) + + def test_custom_tolerances(self): + assert 1e8 + 1e0 == approx(1e8, rel=5e-8, abs=5e0) + assert 1e8 + 1e0 == approx(1e8, rel=5e-9, abs=5e0) + assert 1e8 + 1e0 == approx(1e8, rel=5e-8, abs=5e-1) + assert 1e8 + 1e0 != approx(1e8, rel=5e-9, abs=5e-1) + + assert 1e0 + 1e-8 == approx(1e0, rel=5e-8, abs=5e-8) + assert 1e0 + 1e-8 == approx(1e0, rel=5e-9, abs=5e-8) + assert 1e0 + 1e-8 == approx(1e0, rel=5e-8, abs=5e-9) + assert 1e0 + 1e-8 != approx(1e0, rel=5e-9, abs=5e-9) + + assert 1e-8 + 1e-16 == approx(1e-8, rel=5e-8, abs=5e-16) + assert 1e-8 + 1e-16 == approx(1e-8, rel=5e-9, abs=5e-16) + assert 1e-8 + 1e-16 == approx(1e-8, rel=5e-8, abs=5e-17) + assert 1e-8 + 1e-16 != approx(1e-8, rel=5e-9, abs=5e-17) + + def test_relative_tolerance(self): + within_1e8_rel = [ + (1e8 + 1e0, 1e8), + (1e0 + 1e-8, 1e0), + (1e-8 + 1e-16, 1e-8), + ] + for a, x in within_1e8_rel: + assert a == approx(x, rel=5e-8, abs=0.0) + assert a != approx(x, rel=5e-9, abs=0.0) + + def test_absolute_tolerance(self): + within_1e8_abs = [ + (1e8 + 9e-9, 1e8), + (1e0 + 9e-9, 1e0), + (1e-8 + 9e-9, 1e-8), + ] + for a, x in within_1e8_abs: + assert a == approx(x, rel=0, abs=5e-8) + assert a != approx(x, rel=0, abs=5e-9) + + def test_expecting_zero(self): + examples = [ + (ne, 1e-6, 0.0), + (ne, -1e-6, 0.0), + (eq, 1e-12, 0.0), + (eq, -1e-12, 0.0), + (ne, 2e-12, 0.0), + (ne, -2e-12, 0.0), + (ne, inf, 0.0), + (ne, nan, 0.0), + ] + for op, a, x in examples: + assert op(a, approx(x, rel=0.0, abs=1e-12)) + assert op(a, approx(x, rel=1e-6, abs=1e-12)) + + def test_expecting_inf(self): + examples = [ + (eq, inf, inf), + (eq, -inf, -inf), + (ne, inf, -inf), + (ne, 0.0, inf), + (ne, nan, inf), + ] + for op, a, x in examples: + assert op(a, approx(x)) + + def test_expecting_nan(self): + examples = [ + (nan, nan), + (-nan, -nan), + (nan, -nan), + (0.0, nan), + (inf, nan), + ] + for a, x in examples: + # If there is a relative tolerance and the expected value is NaN, + # the actual tolerance is a NaN, which should be an error. + with pytest.raises(ValueError): + a != approx(x, rel=inf) + + # You can make comparisons against NaN by not specifying a relative + # tolerance, so only an absolute tolerance is calculated. + assert a != approx(x, abs=inf) + + def test_expecting_sequence(self): + within_1e8 = [ + (1e8 + 1e0, 1e8), + (1e0 + 1e-8, 1e0), + (1e-8 + 1e-16, 1e-8), + ] + actual, expected = zip(*within_1e8) + assert actual == approx(expected, rel=5e-8, abs=0.0) + + def test_expecting_sequence_wrong_len(self): + assert [1, 2] != approx([1]) + assert [1, 2] != approx([1,2,3]) + + def test_complex(self): + within_1e6 = [ + ( 1.000001 + 1.0j, 1.0 + 1.0j), + (1.0 + 1.000001j, 1.0 + 1.0j), + (-1.000001 + 1.0j, -1.0 + 1.0j), + (1.0 - 1.000001j, 1.0 - 1.0j), + ] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + + def test_int(self): + within_1e6 = [ + (1000001, 1000000), + (-1000001, -1000000), + ] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + + def test_decimal(self): + within_1e6 = [ + (Decimal('1.000001'), Decimal('1.0')), + (Decimal('-1.000001'), Decimal('-1.0')), + ] + for a, x in within_1e6: + assert a == approx(x, rel=Decimal(5e-6), abs=0) + assert a != approx(x, rel=Decimal(5e-7), abs=0) + + def test_fraction(self): + within_1e6 = [ + (1 + Fraction(1, 1000000), Fraction(1)), + (-1 - Fraction(-1, 1000000), Fraction(-1)), + ] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + + def test_doctests(self): parser = doctest.DocTestParser() test = parser.get_doctest( - pytest.approx.__doc__, - {'approx': pytest.approx}, - pytest.approx.__name__, + approx.__doc__, + {'approx': approx}, + approx.__name__, None, None, ) runner = MyDocTestRunner() runner.run(test) - def test_repr_string(self): - # Just make sure the Unicode handling doesn't raise any exceptions. - print(pytest.approx(1.0)) - print(pytest.approx([1.0, 2.0, 3.0])) - From 078448008c7394905acb1ec2e6a2c8ae8fd9f856 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Fri, 11 Mar 2016 15:59:48 -0800 Subject: [PATCH 18/23] Discuss alternative float comparison algorithms. --- _pytest/python.py | 152 +++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 50 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index fe1758204..e733de243 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1344,50 +1344,55 @@ class RaisesContext(object): class approx(object): """ - Assert that two numbers (or two sets of numbers) are equal to each - other within some margin. + Assert that two numbers (or two sets of numbers) are equal to each other + within some tolerance. - Due to the intricacies of floating-point arithmetic, numbers that we would - intuitively expect to be the same are not always so:: + Due to the `intricacies of floating-point arithmetic`__, numbers that we + would intuitively expect to be equal are not always so:: >>> 0.1 + 0.2 == 0.3 False + __ https://docs.python.org/3/tutorial/floatingpoint.html + This problem is commonly encountered when writing tests, e.g. when making sure that floating-point values are what you expect them to be. One way to deal with this problem is to assert that two floating-point numbers are - equal to within some appropriate margin:: - + equal to within some appropriate tolerance:: + >>> abs((0.1 + 0.2) - 0.3) < 1e-6 True - + However, comparisons like this are tedious to write and difficult to understand. Furthermore, absolute comparisons like the one above are - usually discouraged in favor of relative comparisons, which can't even be - easily written on one line. The ``approx`` class provides a way to make - floating-point comparisons that solves both these problems:: + usually discouraged because there's no tolerance that works well for all + situations. ``1e-6`` is good for numbers around ``1``, but too small for + very big numbers and too big for very small ones. It's better to express + the tolerance as a fraction of the expected value, but relative comparisons + like that are even more difficult to write correctly and concisely. + + The ``approx`` class performs floating-point comparisons using a syntax + that's as intuitive as possible:: >>> from pytest import approx >>> 0.1 + 0.2 == approx(0.3) True - ``approx`` also makes is easy to compare ordered sets of numbers, which - would otherwise be very tedious:: + The same syntax also works on sequences of numbers:: >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) True - By default, ``approx`` considers two numbers to be equal if the relative - error between them is less than one part in a million (e.g. ``1e-6``). - Relative error is defined as ``abs(x - a) / x`` where ``x`` is the value - you're expecting and ``a`` is the value you're comparing to. This - definition breaks down when the numbers being compared get very close to - zero, so ``approx`` will also consider two numbers to be equal if the - absolute difference between them is less than one part in a trillion (e.g. - ``1e-12``). - - Both the relative and absolute error thresholds can be changed by passing - arguments to the ``approx`` constructor:: + By default, ``approx`` considers numbers within a relative tolerance of + ``1e-6`` (i.e. one part in a million) of its expected value to be equal. + This treatment would lead to surprising results if the expected value was + ``0.0``, because nothing but ``0.0`` itself is relatively close to ``0.0``. + To handle this case less surprisingly, ``approx`` also considers numbers + within an absolute tolerance of ``1e-12`` of its expected value to be + equal. Infinite numbers are another special case. They are only + considered equal to themselves, regardless of the relative tolerance. Both + the relative and absolute tolerances can be changed by passing arguments to + the ``approx`` constructor:: >>> 1.0001 == approx(1) False @@ -1396,12 +1401,12 @@ class approx(object): >>> 1.0001 == approx(1, abs=1e-3) True - Note that if you specify ``abs`` but not ``rel``, the comparison will not - consider the relative error between the two values at all. In other words, - two numbers that are within the default relative error threshold of 1e-6 - will still be considered unequal if they exceed the specified absolute - error threshold. If you specify both ``abs`` and ``rel``, the numbers will - be considered equal if either threshold is met:: + If you specify ``abs`` but not ``rel``, the comparison will not consider + the relative tolerance at all. In other words, two numbers that are within + the default relative tolerance of ``1e-6`` will still be considered unequal + if they exceed the specified absolute tolerance. If you specify both + ``abs`` and ``rel``, the numbers will be considered equal if either + tolerance is met:: >>> 1 + 1e-8 == approx(1) True @@ -1409,6 +1414,46 @@ class approx(object): False >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) True + + If you're thinking about using ``approx``, then you might want to know how + it compares to other good ways of comparing floating-point numbers. All of + these algorithms are based on relative and absolute tolerances, but they do + have meaningful differences: + + - ``math.isclose(a, b, rel_tol=1e-9, abs_tol=0.0)``: True if the relative + tolerance is met w.r.t. either ``a`` or ``b`` or if the absolute + tolerance is met. Because the relative tolerance is calculated w.r.t. + both ``a`` and ``b``, this test is symmetric (i.e. neither ``a`` nor + ``b`` is a "reference value"). You have to specify an absolute tolerance + if you want to compare to ``0.0`` because there is no tolerance by + default. Only available in python>=3.5. `More information...`__ + + __ https://docs.python.org/3/library/math.html#math.isclose + + - ``numpy.isclose(a, b, rtol=1e-5, atol=1e-8)``: True if the difference + between ``a`` and ``b`` is less that the sum of the relative tolerance + w.r.t. ``b`` and the absolute tolerance. Because the relative tolerance + is only calculated w.r.t. ``b``, this test is asymmetric and you can + think of ``b`` as the reference value. Support for comparing sequences + is provided by ``numpy.allclose``. `More information...`__ + + __ http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.isclose.html + + - ``unittest.TestCase.assertAlmostEqual(a, b)``: True if ``a`` and ``b`` + are within an absolute tolerance of ``1e-7``. No relative tolerance is + considered and the absolute tolerance cannot be changed, so this function + is not appropriate for very large or very small numbers. Also, it's only + available in subclasses of ``unittest.TestCase`` and it's ugly because it + doesn't follow PEP8. `More information...`__ + + __ https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertAlmostEqual + + - ``a == pytest.approx(b, rel=1e-6, abs=1e-12)``: True if the relative + tolerance is met w.r.t. ``b`` or the if the absolute tolerance is met. + Because the relative tolerance is only calculated w.r.t. ``b``, this test + is asymmetric and you can think of ``b`` as the reference value. In the + special case that you explicitly specify an absolute tolerance but not a + relative tolerance, only the absolute tolerance is considered. """ def __init__(self, expected, rel=None, abs=None): @@ -1427,6 +1472,9 @@ class approx(object): @property def expected(self): + # Regardless of whether the user-specified expected value is a number + # or a sequence of numbers, return a list of ApproxNotIterable objects + # that can be compared against. from collections import Iterable approx_non_iter = lambda x: ApproxNonIterable(x, self.rel, self.abs) if isinstance(self._expected, Iterable): @@ -1437,14 +1485,19 @@ class approx(object): @expected.setter def expected(self, expected): self._expected = expected - class ApproxNonIterable(object): """ Perform approximate comparisons for single numbers only. - This class contains most of the + In other words, the ``expected`` attribute for objects of this class must + be some sort of number. This is in contrast to the ``approx`` class, where + the ``expected`` attribute can either be a number of a sequence of numbers. + This class is responsible for making comparisons, while ``approx`` is + responsible for abstracting the difference between numbers and sequences of + numbers. Although this class can stand on its own, it's only meant to be + used within ``approx``. """ def __init__(self, expected, rel=None, abs=None): @@ -1453,12 +1506,12 @@ class ApproxNonIterable(object): self.rel = rel def __repr__(self): - # Infinities aren't compared using tolerances, so don't show a + # Infinities aren't compared using tolerances, so don't show a # tolerance. if math.isinf(self.expected): return str(self.expected) - # If a sensible tolerance can't be calculated, self.tolerance will + # If a sensible tolerance can't be calculated, self.tolerance will # raise a ValueError. In this case, display '???'. try: vetted_tolerance = '{:.1e}'.format(self.tolerance) @@ -1467,9 +1520,9 @@ class ApproxNonIterable(object): repr = u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) - # In python2, __repr__() must return a string (i.e. not a unicode - # object). In python3, __repr__() must return a unicode object - # (although now strings are unicode objects and bytes are what + # In python2, __repr__() must return a string (i.e. not a unicode + # object). In python3, __repr__() must return a unicode object + # (although now strings are unicode objects and bytes are what # strings were). if sys.version_info[0] == 2: return repr.encode('utf-8') @@ -1481,11 +1534,11 @@ class ApproxNonIterable(object): if actual == self.expected: return True - # Infinity shouldn't be approximately equal to anything but itself, but - # if there's a relative tolerance, it will be infinite and infinity - # will seem approximately equal to everything. The equal-to-itself - # case would have been short circuited above, so here we can just - # return false if the expected value is infinite. The abs() call is + # Infinity shouldn't be approximately equal to anything but itself, but + # if there's a relative tolerance, it will be infinite and infinity + # will seem approximately equal to everything. The equal-to-itself + # case would have been short circuited above, so here we can just + # return false if the expected value is infinite. The abs() call is # for compatibility with complex numbers. if math.isinf(abs(self.expected)): return False @@ -1497,7 +1550,7 @@ class ApproxNonIterable(object): def tolerance(self): set_default = lambda x, default: x if x is not None else default - # Figure out what the absolute tolerance should be. ``self.abs`` is + # Figure out what the absolute tolerance should be. ``self.abs`` is # either None or a value specified by the user. absolute_tolerance = set_default(self.abs, 1e-12) @@ -1505,17 +1558,17 @@ class ApproxNonIterable(object): raise ValueError("absolute tolerance can't be negative: {}".format(absolute_tolerance)) if math.isnan(absolute_tolerance): raise ValueError("absolute tolerance can't be NaN.") - - # If the user specified an absolute tolerance but not a relative one, + + # If the user specified an absolute tolerance but not a relative one, # just return the absolute tolerance. if self.rel is None: if self.abs is not None: return absolute_tolerance - # Figure out what the absolute tolerance should be. ``self.rel`` is - # either None or a value specified by the user. This is done after - # we've made sure the user didn't ask for an absolute tolerance only, - # because we don't want to raise errors about the relative tolerance if + # Figure out what the absolute tolerance should be. ``self.rel`` is + # either None or a value specified by the user. This is done after + # we've made sure the user didn't ask for an absolute tolerance only, + # because we don't want to raise errors about the relative tolerance if # it isn't even being used. relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) @@ -1523,12 +1576,11 @@ class ApproxNonIterable(object): raise ValueError("relative tolerance can't be negative: {}".format(absolute_tolerance)) if math.isnan(relative_tolerance): raise ValueError("relative tolerance can't be NaN.") - + # Return the larger of the relative and absolute tolerances. return max(relative_tolerance, absolute_tolerance) - # # the basic pytest Function item # From 916c0a8b36dc7f9db3c7bef1b80ec85c61378e12 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Fri, 11 Mar 2016 16:29:18 -0800 Subject: [PATCH 19/23] Fix Decimal() and __ne__() errors. --- _pytest/python.py | 6 ++++++ testing/python/approx.py | 9 ++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index e733de243..594663eee 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1470,6 +1470,9 @@ class approx(object): if len(actual) != len(self.expected): return False return all(a == x for a, x in zip(actual, self.expected)) + def __ne__(self, actual): + return not (actual == self) + @property def expected(self): # Regardless of whether the user-specified expected value is a number @@ -1546,6 +1549,9 @@ class ApproxNonIterable(object): # Return true if the two numbers are within the tolerance. return abs(self.expected - actual) <= self.tolerance + def __ne__(self, actual): + return not (actual == self) + @property def tolerance(self): set_default = lambda x, default: x if x is not None else default diff --git a/testing/python/approx.py b/testing/python/approx.py index 2d7eb6ea3..2720c573f 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -31,7 +31,9 @@ class TestApprox: def test_operator_overloading(self): assert 1 == approx(1, rel=1e-6, abs=1e-12) + assert not (1 != approx(1, rel=1e-6, abs=1e-12)) assert 10 != approx(1, rel=1e-6, abs=1e-12) + assert not (10 == approx(1, rel=1e-6, abs=1e-12)) def test_exactly_equal(self): examples = [ @@ -41,7 +43,8 @@ class TestApprox: (12345, 12345.0), (0.0, -0.0), (345678, 345678), - (Decimal(1.0001), Decimal(1.0001)), + (Decimal('1.0001'), Decimal('1.0001')), + (Fraction(1, 3), Fraction(-1, -3)), ] for a, x in examples: assert a == approx(x) @@ -258,8 +261,8 @@ class TestApprox: (Decimal('-1.000001'), Decimal('-1.0')), ] for a, x in within_1e6: - assert a == approx(x, rel=Decimal(5e-6), abs=0) - assert a != approx(x, rel=Decimal(5e-7), abs=0) + assert a == approx(x, rel=Decimal('5e-6'), abs=0) + assert a != approx(x, rel=Decimal('5e-7'), abs=0) def test_fraction(self): within_1e6 = [ From 861265411f62d9b0fe4069efe4ae4ba5f127e270 Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Sat, 12 Mar 2016 22:15:19 -0800 Subject: [PATCH 20/23] Add "thanks" line to the CHANGELOG. --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7ee14abd0..cc46f145f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ * New ``approx()`` function for easily comparing floating-point numbers in tests. + Thanks `@kalekundert`_ for the complete PR (`#1441`_). * @@ -22,8 +23,10 @@ * .. _@milliams: https://github.com/milliams +.. _@kalekundert: https://github.com/kalekundert .. _#1428: https://github.com/pytest-dev/pytest/pull/1428 +.. _#1441: https://github.com/pytest-dev/pytest/pull/1441 2.9.1.dev1 From 9e7206a1cfa88c552c78d2b0d6128bcf4ce3122d Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 14 Mar 2016 11:29:45 -0700 Subject: [PATCH 21/23] Fix a few stylistic issues. --- _pytest/python.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index 594663eee..bf589ff0e 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1466,8 +1466,10 @@ class approx(object): def __eq__(self, actual): from collections import Iterable - if not isinstance(actual, Iterable): actual = [actual] - if len(actual) != len(self.expected): return False + if not isinstance(actual, Iterable): + actual = [actual] + if len(actual) != len(self.expected): + return False return all(a == x for a, x in zip(actual, self.expected)) def __ne__(self, actual): @@ -1521,16 +1523,16 @@ class ApproxNonIterable(object): except ValueError: vetted_tolerance = '???' - repr = u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) + plus_minus = u'{0} \u00b1 {1}'.format(self.expected, vetted_tolerance) # In python2, __repr__() must return a string (i.e. not a unicode # object). In python3, __repr__() must return a unicode object # (although now strings are unicode objects and bytes are what # strings were). if sys.version_info[0] == 2: - return repr.encode('utf-8') + return plus_minus.encode('utf-8') else: - return repr + return plus_minus def __eq__(self, actual): # Short-circuit exact equality. @@ -1571,11 +1573,11 @@ class ApproxNonIterable(object): if self.abs is not None: return absolute_tolerance - # Figure out what the absolute tolerance should be. ``self.rel`` is + # Figure out what the relative tolerance should be. ``self.rel`` is # either None or a value specified by the user. This is done after # we've made sure the user didn't ask for an absolute tolerance only, # because we don't want to raise errors about the relative tolerance if - # it isn't even being used. + # we aren't even going to use it. relative_tolerance = set_default(self.rel, 1e-6) * abs(self.expected) if relative_tolerance < 0: From 0dcc862a5602677966d89c67c074a90c93a9e67d Mon Sep 17 00:00:00 2001 From: Kale Kundert Date: Mon, 14 Mar 2016 11:38:00 -0700 Subject: [PATCH 22/23] Fix some typos in the documentation. --- _pytest/python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_pytest/python.py b/_pytest/python.py index bf589ff0e..812a2e870 100644 --- a/_pytest/python.py +++ b/_pytest/python.py @@ -1417,8 +1417,8 @@ class approx(object): If you're thinking about using ``approx``, then you might want to know how it compares to other good ways of comparing floating-point numbers. All of - these algorithms are based on relative and absolute tolerances, but they do - have meaningful differences: + these algorithms are based on relative and absolute tolerances and should + agree for the most part, but they do have meaningful differences: - ``math.isclose(a, b, rel_tol=1e-9, abs_tol=0.0)``: True if the relative tolerance is met w.r.t. either ``a`` or ``b`` or if the absolute @@ -1449,7 +1449,7 @@ class approx(object): __ https://docs.python.org/3/library/unittest.html#unittest.TestCase.assertAlmostEqual - ``a == pytest.approx(b, rel=1e-6, abs=1e-12)``: True if the relative - tolerance is met w.r.t. ``b`` or the if the absolute tolerance is met. + tolerance is met w.r.t. ``b`` or if the absolute tolerance is met. Because the relative tolerance is only calculated w.r.t. ``b``, this test is asymmetric and you can think of ``b`` as the reference value. In the special case that you explicitly specify an absolute tolerance but not a From fa6acdcfd49d84d5ffb85b23a63a4ed0459d81cc Mon Sep 17 00:00:00 2001 From: Tareq Alayan Date: Mon, 29 Feb 2016 16:14:23 +0200 Subject: [PATCH 23/23] junitxml: add properties node in testsuite level The commit allow users to add a properties node in testsuite level see example below: ') + logfile.write(Junit.testsuite( + self._get_global_properties_node(), [x.to_xml() for x in self.node_reporters_ordered], name="pytest", errors=self.stats['error'], @@ -374,3 +380,18 @@ class LogXML(object): def pytest_terminal_summary(self, terminalreporter): terminalreporter.write_sep("-", "generated xml file: %s" % (self.logfile)) + + def add_global_property(self, name, value): + self.global_properties.append((str(name), bin_xml_escape(value))) + + def _get_global_properties_node(self): + """Return a Junit node containing custom properties, if any. + """ + if self.global_properties: + return Junit.properties( + [ + Junit.property(name=name, value=value) + for name, value in self.global_properties + ] + ) + return '' diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 4b92fd1e1..4a44e26dc 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -193,6 +193,53 @@ This will add an extra property ``example_key="1"`` to the generated Also please note that using this feature will break any schema verification. This might be a problem when used with some CI servers. +LogXML: add_global_property +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.10 + +If you want to add a properties node in the testsuite level, which may contains properties that are relevant +to all testcases you can use ``LogXML.add_global_properties`` + +.. code-block:: python + + import pytest + + @pytest.fixture(scope="session") + def log_global_env_facts(f): + + if pytest.config.pluginmanager.hasplugin('junitxml'): + my_junit = getattr(pytest.config, '_xml', None) + + my_junit.add_global_property('ARCH', 'PPC') + my_junit.add_global_property('STORAGE_TYPE', 'CEPH') + + @pytest.mark.usefixtures(log_global_env_facts) + def start_and_prepare_env(): + pass + + class TestMe: + def test_foo(self): + assert True + +This will add a property node below the testsuite node to the generated xml: + +.. code-block:: xml + + + + + + + + + +.. warning:: + + This is an experimental feature, and its interface might be replaced + by something more powerful and general in future versions. The + functionality per-se will be kept. + Creating resultlog format files ---------------------------------------------------- diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index 99c59cb7a..8ff1028df 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -783,3 +783,38 @@ def test_fancy_items_regression(testdir): u'test_fancy_items_regression test_pass' u' test_fancy_items_regression.py', ] + + +def test_global_properties(testdir): + path = testdir.tmpdir.join("test_global_properties.xml") + log = LogXML(str(path), None) + from _pytest.runner import BaseReport + + class Report(BaseReport): + sections = [] + nodeid = "test_node_id" + + log.pytest_sessionstart() + log.add_global_property('foo', 1) + log.add_global_property('bar', 2) + log.pytest_sessionfinish() + + dom = minidom.parse(str(path)) + + properties = dom.getElementsByTagName('properties') + + assert (properties.length == 1), "There must be one node" + + property_list = dom.getElementsByTagName('property') + + assert (property_list.length == 2), "There most be only 2 property nodes" + + expected = {'foo': '1', 'bar': '2'} + actual = {} + + for p in property_list: + k = str(p.getAttribute('name')) + v = str(p.getAttribute('value')) + actual[k] = v + + assert actual == expected