From a536f49d910eda84cbd0608e6edb1b70bb0868b2 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Wed, 6 Dec 2023 09:25:00 +0000 Subject: [PATCH] Separate the various parts of the error report with newlines (#11659) Previously the error report would have all sections glued together: - The assertion representation - The error explanation - The full diff This makes it hard to see at a glance where which one starts and ends. One of the representation (dataclasses, tuples, attrs) does display a newlines at the start already. Let's add a newlines before the error explanation and before the full diff, so we get an easier to read report. This has one disadvantage: we get one line less in the least verbose mode, where the output gets truncated. --- changelog/11520.improvement.rst | 2 ++ src/_pytest/assertion/util.py | 4 ++- testing/python/approx.py | 32 +++++++++++++----- testing/test_assertion.py | 59 +++++++++++++++++++++++++-------- 4 files changed, 74 insertions(+), 23 deletions(-) diff --git a/changelog/11520.improvement.rst b/changelog/11520.improvement.rst index 46e4992dd..d9b7b4933 100644 --- a/changelog/11520.improvement.rst +++ b/changelog/11520.improvement.rst @@ -1 +1,3 @@ Improved very verbose diff output to color it as a diff instead of only red. + +Improved the error reporting to better separate each section. diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 214c321f0..fe8904e15 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -230,6 +230,8 @@ def assertrepr_compare( if not explanation: return None + if explanation[0] != "": + explanation = [""] + explanation return [summary] + explanation @@ -332,7 +334,7 @@ def _compare_eq_iterable( left_formatting = PrettyPrinter().pformat(left).splitlines() right_formatting = PrettyPrinter().pformat(right).splitlines() - explanation = ["Full diff:"] + explanation = ["", "Full diff:"] # "right" is the expected base against which we compare "left", # see https://github.com/pytest-dev/pytest/issues/3333 explanation.extend( diff --git a/testing/python/approx.py b/testing/python/approx.py index 6ad411a3e..3b87e58f9 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -99,6 +99,7 @@ class TestApprox: 2.0, 1.0, [ + "", " comparison failed", f" Obtained: {SOME_FLOAT}", f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", @@ -113,6 +114,7 @@ class TestApprox: "c": 3000000.0, }, [ + r"", r" comparison failed. Mismatched elements: 2 / 3:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", @@ -130,6 +132,7 @@ class TestApprox: "c": None, }, [ + r"", r" comparison failed. Mismatched elements: 2 / 3:", r" Max absolute difference: -inf", r" Max relative difference: -inf", @@ -143,6 +146,7 @@ class TestApprox: [1.0, 2.0, 3.0, 4.0], [1.0, 3.0, 3.0, 5.0], [ + r"", r" comparison failed. Mismatched elements: 2 / 4:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", @@ -156,6 +160,7 @@ class TestApprox: (1, 2.2, 4), (1, 3.2, 4), [ + r"", r" comparison failed. Mismatched elements: 1 / 3:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", @@ -169,6 +174,7 @@ class TestApprox: [0.0], [1.0], [ + r"", r" comparison failed. Mismatched elements: 1 / 1:", rf" Max absolute difference: {SOME_FLOAT}", r" Max relative difference: inf", @@ -187,6 +193,7 @@ class TestApprox: a, b, [ + r"", r" comparison failed. Mismatched elements: 1 / 20:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", @@ -209,6 +216,7 @@ class TestApprox: ] ), [ + r"", r" comparison failed. Mismatched elements: 3 / 8:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", @@ -224,6 +232,7 @@ class TestApprox: np.array([0.0]), np.array([1.0]), [ + r"", r" comparison failed. Mismatched elements: 1 / 1:", rf" Max absolute difference: {SOME_FLOAT}", r" Max relative difference: inf", @@ -241,6 +250,7 @@ class TestApprox: message = "\n".join(str(e.value).split("\n")[1:]) assert message == "\n".join( [ + " ", " Impossible to compare arrays with different shapes.", " Shapes: (2, 1) and (2, 2)", ] @@ -251,6 +261,7 @@ class TestApprox: message = "\n".join(str(e.value).split("\n")[1:]) assert message == "\n".join( [ + " ", " Impossible to compare lists with different sizes.", " Lengths: 2 and 3", ] @@ -264,6 +275,7 @@ class TestApprox: 2.0, 1.0, [ + "", " comparison failed", f" Obtained: {SOME_FLOAT}", f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", @@ -277,15 +289,15 @@ class TestApprox: a, b, [ - r" comparison failed. Mismatched elements: 20 / 20:", - rf" Max absolute difference: {SOME_FLOAT}", - rf" Max relative difference: {SOME_FLOAT}", - r" Index \| Obtained\s+\| Expected", - rf" \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", - rf" \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", - rf" \(2,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}...", - "", - rf"\s*...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show", + r"^ $", + r"^ comparison failed. Mismatched elements: 20 / 20:$", + rf"^ Max absolute difference: {SOME_FLOAT}$", + rf"^ Max relative difference: {SOME_FLOAT}$", + r"^ Index \| Obtained\s+\| Expected\s+$", + rf"^ \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}e-{SOME_INT}$", + rf"^ \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}e-{SOME_INT}\.\.\.$", + "^ $", + rf"^ ...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show$", ], verbosity_level=0, ) @@ -294,6 +306,7 @@ class TestApprox: a, b, [ + r" ", r" comparison failed. Mismatched elements: 20 / 20:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", @@ -652,6 +665,7 @@ class TestApprox: {"foo": 42.0}, {"foo": 0.0}, [ + r"", r" comparison failed. Mismatched elements: 1 / 1:", rf" Max absolute difference: {SOME_FLOAT}", r" Max relative difference: inf", diff --git a/testing/test_assertion.py b/testing/test_assertion.py index ce10ed8c4..4d751f8db 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -392,6 +392,7 @@ class TestAssert_reprcompare: def test_text_diff(self) -> None: assert callequal("spam", "eggs") == [ "'spam' == 'eggs'", + "", "- eggs", "+ spam", ] @@ -399,7 +400,7 @@ class TestAssert_reprcompare: def test_text_skipping(self) -> None: lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs") assert lines is not None - assert "Skipping" in lines[1] + assert "Skipping" in lines[2] for line in lines: assert "a" * 50 not in line @@ -423,6 +424,7 @@ class TestAssert_reprcompare: assert diff == [ "b'spam' == b'eggs'", + "", "At index 0 diff: b's' != b'e'", "Use -v to get more diff", ] @@ -432,7 +434,9 @@ class TestAssert_reprcompare: diff = callequal(b"spam", b"eggs", verbose=1) assert diff == [ "b'spam' == b'eggs'", + "", "At index 0 diff: b's' != b'e'", + "", "Full diff:", "- b'eggs'", "+ b'spam'", @@ -509,6 +513,7 @@ class TestAssert_reprcompare: expl = callequal([1, 2], [10, 2], verbose=-1) assert expl == [ "[1, 2] == [10, 2]", + "", "At index 0 diff: 1 != 10", "Use -v to get more diff", ] @@ -547,7 +552,9 @@ class TestAssert_reprcompare: diff = callequal(l1, l2, verbose=True) assert diff == [ "['a', 'b', 'c'] == ['a', 'b', 'c...dddddddddddd']", + "", "Right contains one more item: '" + long_d + "'", + "", "Full diff:", " [", " 'a',", @@ -560,7 +567,9 @@ class TestAssert_reprcompare: diff = callequal(l2, l1, verbose=True) assert diff == [ "['a', 'b', 'c...dddddddddddd'] == ['a', 'b', 'c']", + "", "Left contains one more item: '" + long_d + "'", + "", "Full diff:", " [", " 'a',", @@ -579,7 +588,9 @@ class TestAssert_reprcompare: diff = callequal(l1, l2, verbose=True) assert diff == [ "['aaaaaaaaaaa...cccccccccccc'] == ['bbbbbbbbbbb...aaaaaaaaaaaa']", + "", "At index 0 diff: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' != 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'", + "", "Full diff:", " [", "+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", @@ -596,8 +607,10 @@ class TestAssert_reprcompare: diff = callequal(l1, l2, verbose=True) assert diff == [ "['a', 'aaaaaa...aaaaaaa', ...] == ['should not get wrapped']", + "", "At index 0 diff: 'a' != 'should not get wrapped'", "Left contains 7 more items, first extra item: 'aaaaaaaaaa'", + "", "Full diff:", " [", "- 'should not get wrapped',", @@ -619,9 +632,11 @@ class TestAssert_reprcompare: diff = callequal(d1, d2, verbose=True) assert diff == [ "{'common': 1,...1, 'env2': 2}} == {'common': 1,...: {'env1': 1}}", + "", "Omitting 1 identical items, use -vv to show", "Differing items:", "{'env': {'env1': 1, 'env2': 2}} != {'env': {'env1': 1}}", + "", "Full diff:", " {", " 'common': 1,", @@ -639,9 +654,11 @@ class TestAssert_reprcompare: diff = callequal(d1, d2, verbose=True) assert diff == [ "{'env': {'sub... wrapped '}}}} == {'env': {'sub...}}}, 'new': 1}", + "", "Omitting 1 identical items, use -vv to show", "Right contains 1 more item:", "{'new': 1}", + "", "Full diff:", " {", " 'env': {", @@ -665,7 +682,7 @@ class TestAssert_reprcompare: def test_dict_omitting(self) -> None: lines = callequal({"a": 0, "b": 1}, {"a": 1, "b": 1}) assert lines is not None - assert lines[1].startswith("Omitting 1 identical item") + assert lines[2].startswith("Omitting 1 identical item") assert "Common items" not in lines for line in lines[1:]: assert "b" not in line @@ -674,26 +691,29 @@ class TestAssert_reprcompare: """Ensure differing items are visible for verbosity=1 (#1512).""" lines = callequal({"a": 0, "b": 1}, {"a": 1, "b": 1}, verbose=1) assert lines is not None - assert lines[1].startswith("Omitting 1 identical item") - assert lines[2].startswith("Differing items") - assert lines[3] == "{'a': 0} != {'a': 1}" + assert lines[1] == "" + assert lines[2].startswith("Omitting 1 identical item") + assert lines[3].startswith("Differing items") + assert lines[4] == "{'a': 0} != {'a': 1}" assert "Common items" not in lines def test_dict_omitting_with_verbosity_2(self) -> None: lines = callequal({"a": 0, "b": 1}, {"a": 1, "b": 1}, verbose=2) assert lines is not None - assert lines[1].startswith("Common items:") - assert "Omitting" not in lines[1] - assert lines[2] == "{'b': 1}" + assert lines[2].startswith("Common items:") + assert "Omitting" not in lines[2] + assert lines[3] == "{'b': 1}" def test_dict_different_items(self) -> None: lines = callequal({"a": 0}, {"b": 1, "c": 2}, verbose=2) assert lines == [ "{'a': 0} == {'b': 1, 'c': 2}", + "", "Left contains 1 more item:", "{'a': 0}", "Right contains 2 more items:", "{'b': 1, 'c': 2}", + "", "Full diff:", " {", "- 'b': 1,", @@ -706,10 +726,12 @@ class TestAssert_reprcompare: lines = callequal({"b": 1, "c": 2}, {"a": 0}, verbose=2) assert lines == [ "{'b': 1, 'c': 2} == {'a': 0}", + "", "Left contains 2 more items:", "{'b': 1, 'c': 2}", "Right contains 1 more item:", "{'a': 0}", + "", "Full diff:", " {", "- 'a': 0,", @@ -724,8 +746,10 @@ class TestAssert_reprcompare: lines = callequal((1, 2), (3, 4, 5), verbose=2) assert lines == [ "(1, 2) == (3, 4, 5)", + "", "At index 0 diff: 1 != 3", "Right contains one more item: 5", + "", "Full diff:", " (", "- 3,", @@ -742,8 +766,10 @@ class TestAssert_reprcompare: lines = callequal((1, 2, 3), (4,), verbose=2) assert lines == [ "(1, 2, 3) == (4,)", + "", "At index 0 diff: 1 != 4", "Left contains 2 more items, first extra item: 2", + "", "Full diff:", " (", "- 4,", @@ -757,7 +783,9 @@ class TestAssert_reprcompare: lines = callequal((1, 2, 3), (1, 20, 3), verbose=2) assert lines == [ "(1, 2, 3) == (1, 20, 3)", + "", "At index 1 diff: 2 != 20", + "", "Full diff:", " (", " 1,", @@ -823,7 +851,7 @@ class TestAssert_reprcompare: assert expl is not None assert expl[0].startswith("{} == <[ValueError") assert "raised in repr" in expl[0] - assert expl[1:] == [ + assert expl[2:] == [ "(pytest_assertion plugin: representation of details failed:" " {}:{}: ValueError: 42.".format( __file__, A.__repr__.__code__.co_firstlineno + 1 @@ -849,6 +877,7 @@ class TestAssert_reprcompare: def test_unicode(self) -> None: assert callequal("£€", "£") == [ "'£€' == '£'", + "", "- £", "+ £€", ] @@ -864,7 +893,7 @@ class TestAssert_reprcompare: return "\xff" expl = callequal(A(), "1") - assert expl == ["ÿ == '1'", "- 1"] + assert expl == ["ÿ == '1'", "", "- 1"] def test_format_nonascii_explanation(self) -> None: assert util.format_explanation("λ") @@ -887,6 +916,7 @@ class TestAssert_reprcompare: expl = callequal(left, right) assert expl == [ r"'hyv\xe4' == 'hyva\u0308'", + "", f"- {str(right)}", f"+ {str(left)}", ] @@ -894,6 +924,7 @@ class TestAssert_reprcompare: expl = callequal(left, right, verbose=2) assert expl == [ r"'hyv\xe4' == 'hyva\u0308'", + "", f"- {str(right)}", f"+ {str(left)}", ] @@ -1182,6 +1213,7 @@ class TestAssert_reprcompare_namedtuple: # Because the types are different, uses the generic sequence matcher. assert lines == [ "NT1(a=1, b='b') == NT2(a=2, b='b')", + "", "At index 0 diff: 1 != 2", "Use -v to get more diff", ] @@ -1369,7 +1401,7 @@ class TestTruncateExplanation: line_count = 7 line_len = 100 - expected_truncated_lines = 1 + expected_truncated_lines = 2 pytester.makepyfile( r""" def test_many_lines(): @@ -1389,8 +1421,7 @@ class TestTruncateExplanation: [ "*+ 1*", "*+ 3*", - "*+ 5*", - "*truncated (%d line hidden)*use*-vv*" % expected_truncated_lines, + "*truncated (%d lines hidden)*use*-vv*" % expected_truncated_lines, ] ) @@ -1433,6 +1464,7 @@ def test_rewritten(pytester: Pytester) -> None: def test_reprcompare_notin() -> None: assert callop("not in", "foo", "aaafoobbb") == [ "'foo' not in 'aaafoobbb'", + "", "'foo' is contained here:", " aaafoobbb", "? +++", @@ -1442,6 +1474,7 @@ def test_reprcompare_notin() -> None: def test_reprcompare_whitespaces() -> None: assert callequal("\r\n", "\n") == [ r"'\r\n' == '\n'", + "", r"Strings contain only whitespace, escaping them using repr()", r"- '\n'", r"+ '\r\n'",