Fix teardown error reporting when `--maxfail=1` (#11721)

Co-authored-by: Ran Benita <ran@unusedvar.com>
This commit is contained in:
Ben Brown 2024-01-03 12:39:24 -05:00 committed by GitHub
parent f017df443a
commit 12b9bd5801
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 156 additions and 2 deletions

View File

@ -54,6 +54,7 @@ Aviral Verma
Aviv Palivoda
Babak Keyvani
Barney Gale
Ben Brown
Ben Gartner
Ben Webb
Benjamin Peterson

View File

@ -0,0 +1 @@
Fix reporting of teardown errors in higher-scoped fixtures when using `--maxfail` or `--stepwise`.

View File

@ -6,6 +6,7 @@ import functools
import importlib
import os
import sys
import warnings
from pathlib import Path
from typing import AbstractSet
from typing import Callable
@ -44,6 +45,7 @@ from _pytest.reports import CollectReport
from _pytest.reports import TestReport
from _pytest.runner import collect_one_node
from _pytest.runner import SetupState
from _pytest.warning_types import PytestWarning
def pytest_addoption(parser: Parser) -> None:
@ -548,8 +550,8 @@ class Session(nodes.Collector):
)
self.testsfailed = 0
self.testscollected = 0
self.shouldstop: Union[bool, str] = False
self.shouldfail: Union[bool, str] = False
self._shouldstop: Union[bool, str] = False
self._shouldfail: Union[bool, str] = False
self.trace = config.trace.root.get("collection")
self._initialpaths: FrozenSet[Path] = frozenset()
self._initialpaths_with_parents: FrozenSet[Path] = frozenset()
@ -576,6 +578,42 @@ class Session(nodes.Collector):
self.testscollected,
)
@property
def shouldstop(self) -> Union[bool, str]:
return self._shouldstop
@shouldstop.setter
def shouldstop(self, value: Union[bool, str]) -> None:
# The runner checks shouldfail and assumes that if it is set we are
# definitely stopping, so prevent unsetting it.
if value is False and self._shouldstop:
warnings.warn(
PytestWarning(
"session.shouldstop cannot be unset after it has been set; ignoring."
),
stacklevel=2,
)
return
self._shouldstop = value
@property
def shouldfail(self) -> Union[bool, str]:
return self._shouldfail
@shouldfail.setter
def shouldfail(self, value: Union[bool, str]) -> None:
# The runner checks shouldfail and assumes that if it is set we are
# definitely stopping, so prevent unsetting it.
if value is False and self._shouldfail:
warnings.warn(
PytestWarning(
"session.shouldfail cannot be unset after it has been set; ignoring."
),
stacklevel=2,
)
return
self._shouldfail = value
@property
def startpath(self) -> Path:
"""The path from which pytest was invoked.

View File

@ -131,6 +131,10 @@ def runtestprotocol(
show_test_item(item)
if not item.config.getoption("setuponly", False):
reports.append(call_and_report(item, "call", log))
# If the session is about to fail or stop, teardown everything - this is
# necessary to correctly report fixture teardown errors (see #11706)
if item.session.shouldfail or item.session.shouldstop:
nextitem = None
reports.append(call_and_report(item, "teardown", log, nextitem=nextitem))
# After all teardown hooks have been called
# want funcargs and request info to go away.

View File

@ -1087,3 +1087,53 @@ def test_outcome_exception_bad_msg() -> None:
with pytest.raises(TypeError) as excinfo:
OutcomeException(func) # type: ignore
assert str(excinfo.value) == expected
def test_teardown_session_failed(pytester: Pytester) -> None:
"""Test that higher-scoped fixture teardowns run in the context of the last
item after the test session bails early due to --maxfail.
Regression test for #11706.
"""
pytester.makepyfile(
"""
import pytest
@pytest.fixture(scope="module")
def baz():
yield
pytest.fail("This is a failing teardown")
def test_foo(baz):
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)
result = pytester.runpytest("--maxfail=1")
result.assert_outcomes(failed=1, errors=1)
def test_teardown_session_stopped(pytester: Pytester) -> None:
"""Test that higher-scoped fixture teardowns run in the context of the last
item after the test session bails early due to --stepwise.
Regression test for #11706.
"""
pytester.makepyfile(
"""
import pytest
@pytest.fixture(scope="module")
def baz():
yield
pytest.fail("This is a failing teardown")
def test_foo(baz):
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)
result = pytester.runpytest("--stepwise")
result.assert_outcomes(failed=1, errors=1)

View File

@ -418,3 +418,63 @@ def test_rootdir_wrong_option_arg(pytester: Pytester) -> None:
result.stderr.fnmatch_lines(
["*Directory *wrong_dir* not found. Check your '--rootdir' option.*"]
)
def test_shouldfail_is_sticky(pytester: Pytester) -> None:
"""Test that session.shouldfail cannot be reset to False after being set.
Issue #11706.
"""
pytester.makeconftest(
"""
def pytest_sessionfinish(session):
assert session.shouldfail
session.shouldfail = False
assert session.shouldfail
"""
)
pytester.makepyfile(
"""
import pytest
def test_foo():
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)
result = pytester.runpytest("--maxfail=1", "-Wall")
result.assert_outcomes(failed=1, warnings=1)
result.stdout.fnmatch_lines("*session.shouldfail cannot be unset*")
def test_shouldstop_is_sticky(pytester: Pytester) -> None:
"""Test that session.shouldstop cannot be reset to False after being set.
Issue #11706.
"""
pytester.makeconftest(
"""
def pytest_sessionfinish(session):
assert session.shouldstop
session.shouldstop = False
assert session.shouldstop
"""
)
pytester.makepyfile(
"""
import pytest
def test_foo():
pytest.fail("This is a failing test")
def test_bar(): pass
"""
)
result = pytester.runpytest("--stepwise", "-Wall")
result.assert_outcomes(failed=1, warnings=1)
result.stdout.fnmatch_lines("*session.shouldstop cannot be unset*")