From 8efbefb2d7968e44ea3e220ab102872c0f96ccff Mon Sep 17 00:00:00 2001 From: Brian Okken <1568356+okken@users.noreply.github.com> Date: Tue, 7 May 2024 10:37:00 -0700 Subject: [PATCH] New --xfail-tb flag (#12280) Fix #12231 --- changelog/12231.feature.rst | 11 +++++ src/_pytest/terminal.py | 25 +++++++++--- testing/test_terminal.py | 81 ++++++++++++++++++++++++------------- 3 files changed, 83 insertions(+), 34 deletions(-) create mode 100644 changelog/12231.feature.rst diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst new file mode 100644 index 000000000..dad04bc20 --- /dev/null +++ b/changelog/12231.feature.rst @@ -0,0 +1,11 @@ +Added `--xfail-tb` flag, which turns on traceback output for XFAIL results. + +* If the `--xfail-tb` flag is not sent, tracebacks for XFAIL results are NOT shown. +* The style of traceback for XFAIL is set with `--tb`, and can be `auto|long|short|line|native|no`. +* Note: Even if you have `--xfail-tb` set, you won't see them if `--tb=no`. + +Some history: + +With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. + +This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. With this, the default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. While this is a behavior change, it brings default behavior back to pre-8.0.0 behavior, which ultimately was considered the better course of action. diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index d05eb7e3d..2eef8d1f2 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -216,6 +216,13 @@ def pytest_addoption(parser: Parser) -> None: choices=["auto", "long", "short", "no", "line", "native"], help="Traceback print mode (auto/long/short/line/native/no)", ) + group._addoption( + "--xfail-tb", + action="store_true", + dest="xfail_tb", + default=False, + help="Show tracebacks for xfail (as long as --tb != no)", + ) group._addoption( "--show-capture", action="store", @@ -1086,21 +1093,29 @@ class TerminalReporter: self._tw.line(content) def summary_failures(self) -> None: - self.summary_failures_combined("failed", "FAILURES") + style = self.config.option.tbstyle + self.summary_failures_combined("failed", "FAILURES", style=style) def summary_xfailures(self) -> None: - self.summary_failures_combined("xfailed", "XFAILURES", "x") + show_tb = self.config.option.xfail_tb + style = self.config.option.tbstyle if show_tb else "no" + self.summary_failures_combined("xfailed", "XFAILURES", style=style) def summary_failures_combined( - self, which_reports: str, sep_title: str, needed_opt: Optional[str] = None + self, + which_reports: str, + sep_title: str, + *, + style: str, + needed_opt: Optional[str] = None, ) -> None: - if self.config.option.tbstyle != "no": + if style != "no": if not needed_opt or self.hasopt(needed_opt): reports: List[BaseReport] = self.getreports(which_reports) if not reports: return self.write_sep("=", sep_title) - if self.config.option.tbstyle == "line": + if style == "line": for rep in reports: line = self._getcrashline(rep) self.write_line(line) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index dd5fa4bf6..19848e7cb 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -2909,54 +2909,77 @@ def test_summary_xfail_reason(pytester: Pytester) -> None: assert result.stdout.lines.count(expect2) == 1 -def test_summary_xfail_tb(pytester: Pytester) -> None: - pytester.makepyfile( +@pytest.fixture() +def xfail_testfile(pytester: Pytester) -> Path: + return pytester.makepyfile( """ import pytest - @pytest.mark.xfail - def test_xfail(): + def test_fail(): a, b = 1, 2 assert a == b + + @pytest.mark.xfail + def test_xfail(): + c, d = 3, 4 + assert c == d """ ) - result = pytester.runpytest("-rx") + + +def test_xfail_tb_default(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile) + + # test_fail, show traceback result.stdout.fnmatch_lines( [ + "*= FAILURES =*", + "*_ test_fail _*", + "*def test_fail():*", + "* a, b = 1, 2*", + "*> assert a == b*", + "*E assert 1 == 2*", + ] + ) + + # test_xfail, don't show traceback + result.stdout.no_fnmatch_line("*= XFAILURES =*") + + +def test_xfail_tb_true(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile, "--xfail-tb") + + # both test_fail and test_xfail, show traceback + result.stdout.fnmatch_lines( + [ + "*= FAILURES =*", + "*_ test_fail _*", + "*def test_fail():*", + "* a, b = 1, 2*", + "*> assert a == b*", + "*E assert 1 == 2*", "*= XFAILURES =*", "*_ test_xfail _*", - "* @pytest.mark.xfail*", - "* def test_xfail():*", - "* a, b = 1, 2*", - "> *assert a == b*", - "E *assert 1 == 2*", - "test_summary_xfail_tb.py:6: AssertionError*", - "*= short test summary info =*", - "XFAIL test_summary_xfail_tb.py::test_xfail", - "*= 1 xfailed in * =*", + "*def test_xfail():*", + "* c, d = 3, 4*", + "*> assert c == d*", + "*E assert 3 == 4*", + "*short test summary info*", ] ) -def test_xfail_tb_line(pytester: Pytester) -> None: - pytester.makepyfile( - """ - import pytest +def test_xfail_tb_line(xfail_testfile, pytester: Pytester) -> None: + result = pytester.runpytest(xfail_testfile, "--xfail-tb", "--tb=line") - @pytest.mark.xfail - def test_xfail(): - a, b = 1, 2 - assert a == b - """ - ) - result = pytester.runpytest("-rx", "--tb=line") + # both test_fail and test_xfail, show line result.stdout.fnmatch_lines( [ + "*= FAILURES =*", + "*test_xfail_tb_line.py:5: assert 1 == 2", "*= XFAILURES =*", - "*test_xfail_tb_line.py:6: assert 1 == 2", - "*= short test summary info =*", - "XFAIL test_xfail_tb_line.py::test_xfail", - "*= 1 xfailed in * =*", + "*test_xfail_tb_line.py:10: assert 3 == 4", + "*short test summary info*", ] )