p53209761
/
test_ok2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve pytest.approx error messages readability (Pull request) (#8429) 

Improve pytest.approx error messages readability (Pull request)
This commit is contained in:
Tarcísio Fischer 2021-04-30 07:36:56 -03:00 committed by GitHub
parent 992c403fc8
commit 9d9b84d175
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 428 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
changelog/8335.improvement.rst Normal file
View File

@ -0,0 +1,10 @@
Improved :func:`pytest.approx` assertion messages for sequences of numbers.
The assertion messages now dumps a table with the index and the error of each diff.
Example::
> assert [1, 2, 3, 4] == pytest.approx([1, 3, 3, 5])
E assert comparison failed for 2 values:
E Index | Obtained | Expected
E 1 | 2 | 3 +- 3.0e-06
E 3 | 4 | 5 +- 5.0e-06

12
src/_pytest/assertion/util.py
View File

@ -180,7 +180,15 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]:
if istext(left) and istext(right):
explanation = _diff_text(left, right, verbose)
else:
if type(left) == type(right) and (
from _pytest.python_api import ApproxBase
if isinstance(left, ApproxBase) or isinstance(right, ApproxBase):
# Although the common order should be obtained == expected, this ensures both ways
approx_side = left if isinstance(left, ApproxBase) else right
other_side = right if isinstance(left, ApproxBase) else left
explanation = approx_side._repr_compare(other_side)
elif type(left) == type(right) and (
isdatacls(left) or isattrs(left) or isnamedtuple(left)
):
# Note: unlike dataclasses/attrs, namedtuples compare only the
@ -196,9 +204,11 @@ def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]:
explanation = _compare_eq_dict(left, right, verbose)
elif verbose > 0:
explanation = _compare_eq_verbose(left, right)
if isiterable(left) and isiterable(right):
expl = _compare_eq_iterable(left, right, verbose)
explanation.extend(expl)
return explanation

180
src/_pytest/python_api.py
View File

@ -1,7 +1,5 @@
import math
import pprint
from collections.abc import Iterable
from collections.abc import Mapping
from collections.abc import Sized
from decimal import Decimal
from numbers import Complex
@ -10,9 +8,13 @@ from typing import Any
from typing import Callable
from typing import cast
from typing import Generic
from typing import Iterable
from typing import List
from typing import Mapping
from typing import Optional
from typing import overload
from typing import Pattern
from typing import Sequence
from typing import Tuple
from typing import Type
from typing import TYPE_CHECKING
@ -38,6 +40,32 @@ def _non_numeric_type_error(value, at: Optional[str]) -> TypeError:
)
def _compare_approx(
full_object: object,
message_data: Sequence[Tuple[str, str, str]],
number_of_elements: int,
different_ids: Sequence[object],
max_abs_diff: float,
max_rel_diff: float,
) -> List[str]:
message_list = list(message_data)
message_list.insert(0, ("Index", "Obtained", "Expected"))
max_sizes = [0, 0, 0]
for index, obtained, expected in message_list:
max_sizes[0] = max(max_sizes[0], len(index))
max_sizes[1] = max(max_sizes[1], len(obtained))
max_sizes[2] = max(max_sizes[2], len(expected))
explanation = [
f"comparison failed. Mismatched elements: {len(different_ids)} / {number_of_elements}:",
f"Max absolute difference: {max_abs_diff}",
f"Max relative difference: {max_rel_diff}",
] + [
f"{indexes:<{max_sizes[0]}} | {obtained:<{max_sizes[1]}} | {expected:<{max_sizes[2]}}"
for indexes, obtained, expected in message_list
]
return explanation
# builtin pytest.approx helper
@ -60,6 +88,13 @@ class ApproxBase:
def __repr__(self) -> str:
raise NotImplementedError
def _repr_compare(self, other_side: Any) -> List[str]:
return [
"comparison failed",
f"Obtained: {other_side}",
f"Expected: {self}",
]
def __eq__(self, actual) -> bool:
return all(
a == self._approx_scalar(x) for a, x in self._yield_comparisons(actual)
@ -107,6 +142,66 @@ class ApproxNumpy(ApproxBase):
list_scalars = _recursive_list_map(self._approx_scalar, self.expected.tolist())
return f"approx({list_scalars!r})"
def _repr_compare(self, other_side: "ndarray") -> List[str]:
import itertools
import math
def get_value_from_nested_list(
nested_list: List[Any], nd_index: Tuple[Any, ...]
) -> Any:
"""
Helper function to get the value out of a nested list, given an n-dimensional index.
This mimics numpy's indexing, but for raw nested python lists.
"""
value: Any = nested_list
for i in nd_index:
value = value[i]
return value
np_array_shape = self.expected.shape
approx_side_as_list = _recursive_list_map(
self._approx_scalar, self.expected.tolist()
)
if np_array_shape != other_side.shape:
return [
"Impossible to compare arrays with different shapes.",
f"Shapes: {np_array_shape} and {other_side.shape}",
]
number_of_elements = self.expected.size
max_abs_diff = -math.inf
max_rel_diff = -math.inf
different_ids = []
for index in itertools.product(*(range(i) for i in np_array_shape)):
approx_value = get_value_from_nested_list(approx_side_as_list, index)
other_value = get_value_from_nested_list(other_side, index)
if approx_value != other_value:
abs_diff = abs(approx_value.expected - other_value)
max_abs_diff = max(max_abs_diff, abs_diff)
if other_value == 0.0:
max_rel_diff = math.inf
else:
max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value))
different_ids.append(index)
message_data = [
(
str(index),
str(get_value_from_nested_list(other_side, index)),
str(get_value_from_nested_list(approx_side_as_list, index)),
)
for index in different_ids
]
return _compare_approx(
self.expected,
message_data,
number_of_elements,
different_ids,
max_abs_diff,
max_rel_diff,
)
def __eq__(self, actual) -> bool:
import numpy as np
@ -147,6 +242,44 @@ class ApproxMapping(ApproxBase):
{k: self._approx_scalar(v) for k, v in self.expected.items()}
)
def _repr_compare(self, other_side: Mapping[object, float]) -> List[str]:
import math
approx_side_as_map = {
k: self._approx_scalar(v) for k, v in self.expected.items()
}
number_of_elements = len(approx_side_as_map)
max_abs_diff = -math.inf
max_rel_diff = -math.inf
different_ids = []
for (approx_key, approx_value), other_value in zip(
approx_side_as_map.items(), other_side.values()
):
if approx_value != other_value:
max_abs_diff = max(
max_abs_diff, abs(approx_value.expected - other_value)
)
max_rel_diff = max(
max_rel_diff,
abs((approx_value.expected - other_value) / approx_value.expected),
)
different_ids.append(approx_key)
message_data = [
(str(key), str(other_side[key]), str(approx_side_as_map[key]))
for key in different_ids
]
return _compare_approx(
self.expected,
message_data,
number_of_elements,
different_ids,
max_abs_diff,
max_rel_diff,
)
def __eq__(self, actual) -> bool:
try:
if set(actual.keys()) != set(self.expected.keys()):
@ -179,6 +312,48 @@ class ApproxSequencelike(ApproxBase):
seq_type(self._approx_scalar(x) for x in self.expected)
)
def _repr_compare(self, other_side: Sequence[float]) -> List[str]:
import math
import numpy as np
if len(self.expected) != len(other_side):
return [
"Impossible to compare lists with different sizes.",
f"Lengths: {len(self.expected)} and {len(other_side)}",
]
approx_side_as_map = _recursive_list_map(self._approx_scalar, self.expected)
number_of_elements = len(approx_side_as_map)
max_abs_diff = -math.inf
max_rel_diff = -math.inf
different_ids = []
for i, (approx_value, other_value) in enumerate(
zip(approx_side_as_map, other_side)
):
if approx_value != other_value:
abs_diff = abs(approx_value.expected - other_value)
max_abs_diff = max(max_abs_diff, abs_diff)
if other_value == 0.0:
max_rel_diff = np.inf
else:
max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value))
different_ids.append(i)
message_data = [
(str(i), str(other_side[i]), str(approx_side_as_map[i]))
for i in different_ids
]
return _compare_approx(
self.expected,
message_data,
number_of_elements,
different_ids,
max_abs_diff,
max_rel_diff,
)
def __eq__(self, actual) -> bool:
try:
if len(actual) != len(self.expected):
@ -212,7 +387,6 @@ class ApproxScalar(ApproxBase):
For example, ``1.0 ± 1e-6``, ``(3+4j) ± 5e-6 ∠ ±180°``.
"""
# Don't show a tolerance for values that aren't compared using
# tolerances, i.e. non-numerics and infinities. Need to call abs to
# handle complex numbers, e.g. (inf + 1j).

230
testing/python/approx.py
View File

@ -1,5 +1,6 @@
import operator
import sys
from contextlib import contextmanager
from decimal import Decimal
from fractions import Fraction
from operator import eq
@ -43,7 +44,236 @@ def mocked_doctest_runner(monkeypatch):
return MyDocTestRunner()
@contextmanager
def temporary_verbosity(config, verbosity=0):
original_verbosity = config.getoption("verbose")
config.option.verbose = verbosity
try:
yield
finally:
config.option.verbose = original_verbosity
@pytest.fixture
def assert_approx_raises_regex(pytestconfig):
def do_assert(lhs, rhs, expected_message, verbosity_level=0):
import re
with temporary_verbosity(pytestconfig, verbosity_level):
with pytest.raises(AssertionError) as e:
assert lhs == approx(rhs)
nl = "\n"
obtained_message = str(e.value).splitlines()[1:]
assert len(obtained_message) == len(expected_message), (
"Regex message length doesn't match obtained.\n"
"Obtained:\n"
f"{nl.join(obtained_message)}\n\n"
"Expected regex:\n"
f"{nl.join(expected_message)}\n\n"
)
for i, (obtained_line, expected_line) in enumerate(
zip(obtained_message, expected_message)
):
regex = re.compile(expected_line)
assert regex.match(obtained_line) is not None, (
"Unexpected error message:\n"
f"{nl.join(obtained_message)}\n\n"
"Did not match regex:\n"
f"{nl.join(expected_message)}\n\n"
f"With verbosity level = {verbosity_level}, on line {i}"
)
return do_assert
SOME_FLOAT = r"[+-]?([0-9]*[.])?[0-9]+\s*"
SOME_INT = r"[0-9]+\s*"
class TestApprox:
def test_error_messages(self, assert_approx_raises_regex):
np = pytest.importorskip("numpy")
assert_approx_raises_regex(
2.0,
1.0,
[
" comparison failed",
f" Obtained: {SOME_FLOAT}",
f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
assert_approx_raises_regex(
{"a": 1.0, "b": 1000.0, "c": 1000000.0},
{
"a": 2.0,
"b": 1000.0,
"c": 3000000.0,
},
[
r" comparison failed. Mismatched elements: 2 / 3:",
rf" Max absolute difference: {SOME_FLOAT}",
rf" Max relative difference: {SOME_FLOAT}",
r" Index \| Obtained\s+\| Expected ",
rf" a \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
rf" c \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
assert_approx_raises_regex(
[1.0, 2.0, 3.0, 4.0],
[1.0, 3.0, 3.0, 5.0],
[
r" comparison failed. Mismatched elements: 2 / 4:",
rf" Max absolute difference: {SOME_FLOAT}",
rf" Max relative difference: {SOME_FLOAT}",
r" Index \| Obtained\s+\| Expected ",
rf" 1 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
rf" 3 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
a = np.linspace(0, 100, 20)
b = np.linspace(0, 100, 20)
a[10] += 0.5
assert_approx_raises_regex(
a,
b,
[
r" comparison failed. Mismatched elements: 1 / 20:",
rf" Max absolute difference: {SOME_FLOAT}",
rf" Max relative difference: {SOME_FLOAT}",
r" Index \| Obtained\s+\| Expected",
rf" \(10,\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
assert_approx_raises_regex(
np.array(
[
[[1.1987311, 12412342.3], [3.214143244, 1423412423415.677]],
[[1, 2], [3, 219371297321973]],
]
),
np.array(
[
[[1.12313, 12412342.3], [3.214143244, 534523542345.677]],
[[1, 2], [3, 7]],
]
),
[
r" comparison failed. Mismatched elements: 3 / 8:",
rf" Max absolute difference: {SOME_FLOAT}",
rf" Max relative difference: {SOME_FLOAT}",
r" Index\s+\| Obtained\s+\| Expected\s+",
rf" \(0, 0, 0\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
rf" \(0, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
rf" \(1, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
# Specific test for comparison with 0.0 (relative diff will be 'inf')
assert_approx_raises_regex(
[0.0],
[1.0],
[
r" comparison failed. Mismatched elements: 1 / 1:",
rf" Max absolute difference: {SOME_FLOAT}",
r" Max relative difference: inf",
r" Index \| Obtained\s+\| Expected ",
rf"\s*0\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
assert_approx_raises_regex(
np.array([0.0]),
np.array([1.0]),
[
r" comparison failed. Mismatched elements: 1 / 1:",
rf" Max absolute difference: {SOME_FLOAT}",
r" Max relative difference: inf",
r" Index \| Obtained\s+\| Expected ",
rf"\s*\(0,\)\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
],
)
def test_error_messages_invalid_args(self, assert_approx_raises_regex):
np = pytest.importorskip("numpy")
with pytest.raises(AssertionError) as e:
assert np.array([[1.2, 3.4], [4.0, 5.0]]) == pytest.approx(
np.array([[4.0], [5.0]])
)
message = "\n".join(str(e.value).split("\n")[1:])
assert message == "\n".join(
[
" Impossible to compare arrays with different shapes.",
" Shapes: (2, 1) and (2, 2)",
]
)
with pytest.raises(AssertionError) as e:
assert [1.0, 2.0, 3.0] == pytest.approx([4.0, 5.0])
message = "\n".join(str(e.value).split("\n")[1:])
assert message == "\n".join(
[
" Impossible to compare lists with different sizes.",
" Lengths: 2 and 3",
]
)
def test_error_messages_with_different_verbosity(self, assert_approx_raises_regex):
np = pytest.importorskip("numpy")
for v in [0, 1, 2]:
# Verbosity level doesn't affect the error message for scalars
assert_approx_raises_regex(
2.0,
1.0,
[
" comparison failed",
f" Obtained: {SOME_FLOAT}",
f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}",
],
verbosity_level=v,
)
a = np.linspace(1, 101, 20)
b = np.linspace(2, 102, 20)
assert_approx_raises_regex(
a,
b,
[
r" comparison failed. Mismatched elements: 20 / 20:",
rf" Max absolute difference: {SOME_FLOAT}",
rf" Max relative difference: {SOME_FLOAT}",
r" Index \| Obtained\s+\| Expected",
rf" \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
rf" \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}",
rf" \(2,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}...",
"",
rf"\s*...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show",
],
verbosity_level=0,
)
assert_approx_raises_regex(
a,
b,
[
r" comparison failed. Mismatched elements: 20 / 20:",
rf" Max absolute difference: {SOME_FLOAT}",
rf" Max relative difference: {SOME_FLOAT}",
r" Index \| Obtained\s+\| Expected",
]
+ [
rf" \({i},\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}"
for i in range(20)
],
verbosity_level=2,
)
def test_repr_string(self):
assert repr(approx(1.0)) == "1.0 ± 1.0e-06"
assert repr(approx([1.0, 2.0])) == "approx([1.0 ± 1.0e-06, 2.0 ± 2.0e-06])"