p53209761
/
test_ok2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reverse / fix meaning of "+/-" in error diffs 

The convention is "assert result is expected".  Pytest's error diffs now
reflect this. "-" means that sth. expected is missing in the result and
"+" means that there are unexpected extras in the result.

Fixes: #3333
This commit is contained in:
Stefan Scherfke 2020-02-04 14:38:18 +01:00
parent 4038d6c773
commit d59adc61f9
8 changed files with 360 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@ -246,6 +246,7 @@ Simon Gomizelj
Skylar Downes
Srinivas Reddy Thatiparthy
Stefan Farmbauer
Stefan Scherfke
Stefan Zimmermann
Stefano Taschini
Steffen Allner

1
changelog/6673.breaking.rst Normal file
View File

@ -0,0 +1 @@
Reversed / fix meaning of "+/-" in error diffs. "-" means that sth. expected is missing in the result and "+" means that there are unexpected extras in the result.

16
doc/en/example/reportingdemo.rst
View File

@ -81,8 +81,8 @@ Here is a nice run of several failures and how ``pytest`` presents things:
def test_eq_text(self):
> assert "spam" == "eggs"
E AssertionError: assert 'spam' == 'eggs'
E - spam
E + eggs
E - eggs
E + spam
failure_demo.py:45: AssertionError
_____________ TestSpecialisedExplanations.test_eq_similar_text _____________
@ -92,9 +92,9 @@ Here is a nice run of several failures and how ``pytest`` presents things:
def test_eq_similar_text(self):
> assert "foo 1 bar" == "foo 2 bar"
E AssertionError: assert 'foo 1 bar' == 'foo 2 bar'
E - foo 1 bar
E - foo 2 bar
E ? ^
E + foo 2 bar
E + foo 1 bar
E ? ^
failure_demo.py:48: AssertionError
@ -106,8 +106,8 @@ Here is a nice run of several failures and how ``pytest`` presents things:
> assert "foo\nspam\nbar" == "foo\neggs\nbar"
E AssertionError: assert 'foo\nspam\nbar' == 'foo\neggs\nbar'
E foo
E - spam
E + eggs
E - eggs
E + spam
E bar
failure_demo.py:51: AssertionError
@ -122,9 +122,9 @@ Here is a nice run of several failures and how ``pytest`` presents things:
E AssertionError: assert '111111111111...2222222222222' == '111111111111...2222222222222'
E Skipping 90 identical leading characters in diff, use -v to show
E Skipping 91 identical trailing characters in diff, use -v to show
E - 1111111111a222222222
E - 1111111111b222222222
E ? ^
E + 1111111111b222222222
E + 1111111111a222222222
E ? ^
failure_demo.py:56: AssertionError

19
src/_pytest/assertion/util.py
View File

@ -225,9 +225,11 @@ def _diff_text(left: str, right: str, verbose: int = 0) -> List[str]:
left = repr(str(left))
right = repr(str(right))
explanation += ["Strings contain only whitespace, escaping them using repr()"]
# "right" is the expected base against which we compare "left",
# see https://github.com/pytest-dev/pytest/issues/3333
explanation += [
line.strip("\n")
for line in ndiff(left.splitlines(keepends), right.splitlines(keepends))
for line in ndiff(right.splitlines(keepends), left.splitlines(keepends))
]
return explanation
@ -238,8 +240,8 @@ def _compare_eq_verbose(left: Any, right: Any) -> List[str]:
right_lines = repr(right).splitlines(keepends)
explanation = [] # type: List[str]
explanation += ["-" + line for line in left_lines]
explanation += ["+" + line for line in right_lines]
explanation += ["+" + line for line in left_lines]
explanation += ["-" + line for line in right_lines]
return explanation
@ -279,8 +281,10 @@ def _compare_eq_iterable(
_surrounding_parens_on_own_lines(right_formatting)
explanation = ["Full diff:"]
# "right" is the expected base against which we compare "left",
# see https://github.com/pytest-dev/pytest/issues/3333
explanation.extend(
line.rstrip() for line in difflib.ndiff(left_formatting, right_formatting)
line.rstrip() for line in difflib.ndiff(right_formatting, left_formatting)
)
return explanation
@ -315,8 +319,9 @@ def _compare_eq_sequence(
break
if comparing_bytes:
# when comparing bytes, it doesn't help to show the "sides contain one or more items"
# longer explanation, so skip it
# when comparing bytes, it doesn't help to show the "sides contain one or more
# items" longer explanation, so skip it
return explanation
len_diff = len_left - len_right
@ -443,7 +448,7 @@ def _notin_text(term: str, text: str, verbose: int = 0) -> List[str]:
head = text[:index]
tail = text[index + len(term) :]
correct_text = head + tail
diff = _diff_text(correct_text, text, verbose)
diff = _diff_text(text, correct_text, verbose)
newdiff = ["%s is contained here:" % saferepr(term, maxsize=42)]
for line in diff:
if line.startswith("Skipping"):

4
testing/acceptance_test.py
View File

@ -1277,8 +1277,8 @@ def test_pdb_can_be_rewritten(testdir):
" def check():",
"> assert 1 == 2",
"E assert 1 == 2",
"E -1",
"E +2",
"E +1",
"E -2",
"",
"pdb.py:2: AssertionError",
"*= 1 failed in *",

108
testing/test_assertion.py
View File

@ -316,8 +316,8 @@ class TestAssert_reprcompare:
def test_text_diff(self):
diff = callequal("spam", "eggs")[1:]
assert "- spam" in diff
assert "+ eggs" in diff
assert "- eggs" in diff
assert "+ spam" in diff
def test_text_skipping(self):
lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs")
@ -327,15 +327,15 @@ class TestAssert_reprcompare:
def test_text_skipping_verbose(self):
lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs", verbose=1)
assert "- " + "a" * 50 + "spam" in lines
assert "+ " + "a" * 50 + "eggs" in lines
assert "- " + "a" * 50 + "eggs" in lines
assert "+ " + "a" * 50 + "spam" in lines
def test_multiline_text_diff(self):
left = "foo\nspam\nbar"
right = "foo\neggs\nbar"
diff = callequal(left, right)
assert "- spam" in diff
assert "+ eggs" in diff
assert "- eggs" in diff
assert "+ spam" in diff
def test_bytes_diff_normal(self):
"""Check special handling for bytes diff (#5260)"""
@ -354,8 +354,8 @@ class TestAssert_reprcompare:
"b'spam' == b'eggs'",
"At index 0 diff: b's' != b'e'",
"Full diff:",
"- b'spam'",
"+ b'eggs'",
"- b'eggs'",
"+ b'spam'",
]
def test_list(self):
@ -370,9 +370,9 @@ class TestAssert_reprcompare:
[0, 2],
"""
Full diff:
- [0, 1]
- [0, 2]
? ^
+ [0, 2]
+ [0, 1]
? ^
""",
id="lists",
@ -382,9 +382,9 @@ class TestAssert_reprcompare:
{0: 2},
"""
Full diff:
- {0: 1}
- {0: 2}
? ^
+ {0: 2}
+ {0: 1}
? ^
""",
id="dicts",
@ -394,9 +394,9 @@ class TestAssert_reprcompare:
{0, 2},
"""
Full diff:
- {0, 1}
- {0, 2}
? ^
+ {0, 2}
+ {0, 1}
? ^
""",
id="sets",
@ -433,7 +433,7 @@ class TestAssert_reprcompare:
" 'a',",
" 'b',",
" 'c',",
"+ '" + long_d + "',",
"- '" + long_d + "',",
" ]",
]
@ -446,7 +446,7 @@ class TestAssert_reprcompare:
" 'a',",
" 'b',",
" 'c',",
"- '" + long_d + "',",
"+ '" + long_d + "',",
" ]",
]
@ -462,10 +462,10 @@ class TestAssert_reprcompare:
"At index 0 diff: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' != 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'",
"Full diff:",
" [",
"- 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
"+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
" 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',",
" 'cccccccccccccccccccccccccccccc',",
"+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
"- 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',",
" ]",
]
@ -480,28 +480,28 @@ class TestAssert_reprcompare:
"Left contains 7 more items, first extra item: 'aaaaaaaaaa'",
"Full diff:",
" [",
"+ 'should not get wrapped',",
"- 'a',",
"- 'aaaaaaaaaa',",
"- 'aaaaaaaaaa',",
"- 'aaaaaaaaaa',",
"- 'aaaaaaaaaa',",
"- 'aaaaaaaaaa',",
"- 'aaaaaaaaaa',",
"- 'aaaaaaaaaa',",
"- 'should not get wrapped',",
"+ 'a',",
"+ 'aaaaaaaaaa',",
"+ 'aaaaaaaaaa',",
"+ 'aaaaaaaaaa',",
"+ 'aaaaaaaaaa',",
"+ 'aaaaaaaaaa',",
"+ 'aaaaaaaaaa',",
"+ 'aaaaaaaaaa',",
" ]",
]
def test_dict_wrap(self):
d1 = {"common": 1, "env": {"env1": 1}}
d2 = {"common": 1, "env": {"env1": 1, "env2": 2}}
d1 = {"common": 1, "env": {"env1": 1, "env2": 2}}
d2 = {"common": 1, "env": {"env1": 1}}
diff = callequal(d1, d2, verbose=True)
assert diff == [
"{'common': 1,...: {'env1': 1}} == {'common': 1,...1, 'env2': 2}}",
"{'common': 1,...1, 'env2': 2}} == {'common': 1,...: {'env1': 1}}",
"Omitting 1 identical items, use -vv to show",
"Differing items:",
"{'env': {'env1': 1}} != {'env': {'env1': 1, 'env2': 2}}",
"{'env': {'env1': 1, 'env2': 2}} != {'env': {'env1': 1}}",
"Full diff:",
"- {'common': 1, 'env': {'env1': 1}}",
"+ {'common': 1, 'env': {'env1': 1, 'env2': 2}}",
@ -523,7 +523,7 @@ class TestAssert_reprcompare:
" 'env': {'sub': {'long_a': '" + long_a + "',",
" 'sub1': {'long_a': 'substring that gets wrapped substring '",
" 'that gets wrapped '}}},",
"+ 'new': 1,",
"- 'new': 1,",
" }",
]
@ -561,8 +561,8 @@ class TestAssert_reprcompare:
"Right contains 2 more items:",
"{'b': 1, 'c': 2}",
"Full diff:",
"- {'a': 0}",
"+ {'b': 1, 'c': 2}",
"- {'b': 1, 'c': 2}",
"+ {'a': 0}",
]
lines = callequal({"b": 1, "c": 2}, {"a": 0}, verbose=2)
assert lines == [
@ -572,8 +572,8 @@ class TestAssert_reprcompare:
"Right contains 1 more item:",
"{'a': 0}",
"Full diff:",
"- {'b': 1, 'c': 2}",
"+ {'a': 0}",
"- {'a': 0}",
"+ {'b': 1, 'c': 2}",
]
def test_sequence_different_items(self):
@ -583,8 +583,8 @@ class TestAssert_reprcompare:
"At index 0 diff: 1 != 3",
"Right contains one more item: 5",
"Full diff:",
"- (1, 2)",
"+ (3, 4, 5)",
"- (3, 4, 5)",
"+ (1, 2)",
]
lines = callequal((1, 2, 3), (4,), verbose=2)
assert lines == [
@ -592,8 +592,8 @@ class TestAssert_reprcompare:
"At index 0 diff: 1 != 4",
"Left contains 2 more items, first extra item: 2",
"Full diff:",
"- (1, 2, 3)",
"+ (4,)",
"- (4,)",
"+ (1, 2, 3)",
]
def test_set(self):
@ -654,12 +654,12 @@ class TestAssert_reprcompare:
assert callequal(nums_x, nums_y) is None
expl = callequal(nums_x, nums_y, verbose=1)
assert "-" + repr(nums_x) in expl
assert "+" + repr(nums_y) in expl
assert "+" + repr(nums_x) in expl
assert "-" + repr(nums_y) in expl
expl = callequal(nums_x, nums_y, verbose=2)
assert "-" + repr(nums_x) in expl
assert "+" + repr(nums_y) in expl
assert "+" + repr(nums_x) in expl
assert "-" + repr(nums_y) in expl
def test_list_bad_repr(self):
class A:
@ -693,8 +693,8 @@ class TestAssert_reprcompare:
right = "£"
expl = callequal(left, right)
assert expl[0] == "'£€' == '£'"
assert expl[1] == "- £€"
assert expl[2] == "+ £"
assert expl[1] == "- £"
assert expl[2] == "+ £€"
def test_nonascii_text(self):
"""
@ -707,7 +707,7 @@ class TestAssert_reprcompare:
return "\xff"
expl = callequal(A(), "1")
assert expl == ["ÿ == '1'", "+ 1"]
assert expl == ["ÿ == '1'", "- 1"]
def test_format_nonascii_explanation(self):
assert util.format_explanation("λ")
@ -1007,9 +1007,9 @@ class TestTruncateExplanation:
# without -vv, truncate the message showing a few diff lines only
result.stdout.fnmatch_lines(
[
"*- 1*",
"*- 3*",
"*- 5*",
"*+ 1*",
"*+ 3*",
"*+ 5*",
"*truncated (%d lines hidden)*use*-vv*" % expected_truncated_lines,
]
)
@ -1062,9 +1062,9 @@ def test_reprcompare_whitespaces():
assert detail == [
r"'\r\n' == '\n'",
r"Strings contain only whitespace, escaping them using repr()",
r"- '\r\n'",
r"? --",
r"+ '\n'",
r"- '\n'",
r"+ '\r\n'",
r"? ++",
]
@ -1312,8 +1312,8 @@ def test_diff_newline_at_end(testdir):
r"""
*assert 'asdf' == 'asdf\n'
* - asdf
* ? -
* + asdf
* ? +
"""
)

16
testing/test_assertrewrite.py
View File

@ -180,8 +180,8 @@ class TestAssertionRewrite:
if verbose > 0:
assert msg == (
"assert <module 'sys' (built-in)> == 42\n"
" -<module 'sys' (built-in)>\n"
" +42"
" +<module 'sys' (built-in)>\n"
" -42"
)
else:
assert msg == "assert sys == 42"
@ -194,12 +194,12 @@ class TestAssertionRewrite:
msg = getmsg(f, {"cls": X}).splitlines()
if verbose > 1:
assert msg == ["assert {!r} == 42".format(X), " -{!r}".format(X), " +42"]
assert msg == ["assert {!r} == 42".format(X), " +{!r}".format(X), " -42"]
elif verbose > 0:
assert msg == [
"assert <class 'test_...e.<locals>.X'> == 42",
" -{!r}".format(X),
" +42",
" +{!r}".format(X),
" -42",
]
else:
assert msg == ["assert cls == 42"]
@ -241,7 +241,7 @@ class TestAssertionRewrite:
# XXX: looks like the "where" should also be there in verbose mode?!
message = getmsg(f, {"cls": Y}).splitlines()
if request.config.getoption("verbose") > 0:
assert message == ["assert 3 == 2", " -3", " +2"]
assert message == ["assert 3 == 2", " +3", " -2"]
else:
assert message == [
"assert 3 == 2",
@ -625,7 +625,7 @@ class TestAssertionRewrite:
msg = getmsg(f)
if request.config.getoption("verbose") > 0:
assert msg == "assert 10 == 11\n -10\n +11"
assert msg == "assert 10 == 11\n +10\n -11"
else:
assert msg == "assert 10 == 11\n + where 10 = len([0, 1, 2, 3, 4, 5, ...])"
@ -688,7 +688,7 @@ class TestAssertionRewrite:
lines = util._format_lines([getmsg(f)])
if request.config.getoption("verbose") > 0:
assert lines == ["assert 0 == 1\n -0\n +1"]
assert lines == ["assert 0 == 1\n +0\n -1"]
else:
assert lines == ["assert 0 == 1\n + where 1 = \\n{ \\n~ \\n}.a"]

274
testing/test_error_diffs.py Normal file
View File

@ -0,0 +1,274 @@
"""
Tests and examples for correct "+/-" usage in error diffs.
See https://github.com/pytest-dev/pytest/issues/3333 for details.
"""
import sys
import pytest
TESTCASES = [
pytest.param(
"""
def test_this():
result = [1, 4, 3]
expected = [1, 2, 3]
assert result == expected
""",
"""
> assert result == expected
E assert [1, 4, 3] == [1, 2, 3]
E At index 1 diff: 4 != 2
E Full diff:
E - [1, 2, 3]
E ? ^
E + [1, 4, 3]
E ? ^
""",
id="Compare lists, one item differs",
),
pytest.param(
"""
def test_this():
result = [1, 2, 3]
expected = [1, 2]
assert result == expected
""",
"""
> assert result == expected
E assert [1, 2, 3] == [1, 2]
E Left contains one more item: 3
E Full diff:
E - [1, 2]
E + [1, 2, 3]
E ? +++
""",
id="Compare lists, one extra item",
),
pytest.param(
"""
def test_this():
result = [1, 3]
expected = [1, 2, 3]
assert result == expected
""",
"""
> assert result == expected
E assert [1, 3] == [1, 2, 3]
E At index 1 diff: 3 != 2
E Right contains one more item: 3
E Full diff:
E - [1, 2, 3]
E ? ---
E + [1, 3]
""",
id="Compare lists, one item missing",
),
pytest.param(
"""
def test_this():
result = (1, 4, 3)
expected = (1, 2, 3)
assert result == expected
""",
"""
> assert result == expected
E assert (1, 4, 3) == (1, 2, 3)
E At index 1 diff: 4 != 2
E Full diff:
E - (1, 2, 3)
E ? ^
E + (1, 4, 3)
E ? ^
""",
id="Compare tuples",
),
pytest.param(
"""
def test_this():
result = {1, 3, 4}
expected = {1, 2, 3}
assert result == expected
""",
"""
> assert result == expected
E assert {1, 3, 4} == {1, 2, 3}
E Extra items in the left set:
E 4
E Extra items in the right set:
E 2
E Full diff:
E - {1, 2, 3}
E ? ^ ^
E + {1, 3, 4}
E ? ^ ^
""",
id="Compare sets",
),
pytest.param(
"""
def test_this():
result = {1: 'spam', 3: 'eggs'}
expected = {1: 'spam', 2: 'eggs'}
assert result == expected
""",
"""
> assert result == expected
E AssertionError: assert {1: 'spam', 3: 'eggs'} == {1: 'spam', 2: 'eggs'}
E Common items:
E {1: 'spam'}
E Left contains 1 more item:
E {3: 'eggs'}
E Right contains 1 more item:
E {2: 'eggs'}
E Full diff:
E - {1: 'spam', 2: 'eggs'}
E ? ^
E + {1: 'spam', 3: 'eggs'}
E ? ^
""",
id="Compare dicts with differing keys",
),
pytest.param(
"""
def test_this():
result = {1: 'spam', 2: 'eggs'}
expected = {1: 'spam', 2: 'bacon'}
assert result == expected
""",
"""
> assert result == expected
E AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 2: 'bacon'}
E Common items:
E {1: 'spam'}
E Differing items:
E {2: 'eggs'} != {2: 'bacon'}
E Full diff:
E - {1: 'spam', 2: 'bacon'}
E ? ^^^^^
E + {1: 'spam', 2: 'eggs'}
E ? ^^^^
""",
id="Compare dicts with differing values",
),
pytest.param(
"""
def test_this():
result = {1: 'spam', 2: 'eggs'}
expected = {1: 'spam', 3: 'bacon'}
assert result == expected
""",
"""
> assert result == expected
E AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 3: 'bacon'}
E Common items:
E {1: 'spam'}
E Left contains 1 more item:
E {2: 'eggs'}
E Right contains 1 more item:
E {3: 'bacon'}
E Full diff:
E - {1: 'spam', 3: 'bacon'}
E ? ^ ^^^^^
E + {1: 'spam', 2: 'eggs'}
E ? ^ ^^^^
""",
id="Compare dicts with differing items",
),
pytest.param(
"""
def test_this():
result = "spmaeggs"
expected = "spameggs"
assert result == expected
""",
"""
> assert result == expected
E AssertionError: assert 'spmaeggs' == 'spameggs'
E - spameggs
E ? -
E + spmaeggs
E ? +
""",
id="Compare strings",
),
pytest.param(
"""
def test_this():
result = "spam bacon eggs"
assert "bacon" not in result
""",
"""
> assert "bacon" not in result
E AssertionError: assert 'bacon' not in 'spam bacon eggs'
E 'bacon' is contained here:
E spam bacon eggs
E ? +++++
""",
id='Test "not in" string',
),
]
if sys.version_info[:2] >= (3, 7):
TESTCASES.extend(
[
pytest.param(
"""
from dataclasses import dataclass
@dataclass
class A:
a: int
b: str
def test_this():
result = A(1, 'spam')
expected = A(2, 'spam')
assert result == expected
""",
"""
> assert result == expected
E AssertionError: assert A(a=1, b='spam') == A(a=2, b='spam')
E Matching attributes:
E ['b']
E Differing attributes:
E a: 1 != 2
""",
id="Compare data classes",
),
pytest.param(
"""
import attr
@attr.s(auto_attribs=True)
class A:
a: int
b: str
def test_this():
result = A(1, 'spam')
expected = A(1, 'eggs')
assert result == expected
""",
"""
> assert result == expected
E AssertionError: assert A(a=1, b='spam') == A(a=1, b='eggs')
E Matching attributes:
E ['a']
E Differing attributes:
E b: 'spam' != 'eggs'
""",
id="Compare attrs classes",
),
]
)
@pytest.mark.parametrize("code, expected", TESTCASES)
def test_error_diff(code, expected, testdir):
expected = [l.lstrip() for l in expected.splitlines()]
p = testdir.makepyfile(code)
result = testdir.runpytest(p, "-vv")
result.stdout.fnmatch_lines(expected)
assert result.ret == 1