p53209761
/
test_ok2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New verbosity_test_case ini option (#11653) 

Allow for the output of test case execution to be controlled independently from the application verbosity level. 

`verbosity_test_case` is the new ini setting to adjust this functionality.

Fix #11639
This commit is contained in:
Patrick Lannigan 2024-02-24 14:27:54 -05:00 committed by GitHub
parent a2a9aa6cde
commit 84bd31de64
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 273 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
changelog/11653.feature.rst Normal file
View File

@ -0,0 +1,2 @@
Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity.
See :ref:`Fine-grained verbosity <pytest.fine_grained_verbosity>` for more details.

4
doc/en/how-to/output.rst
View File

@ -325,7 +325,9 @@ This is done by setting a verbosity level in the configuration file for the spec
``pytest --no-header`` with a value of ``2`` would have the same output as the previous example, but each test inside
the file is shown by a single character in the output.
(Note: currently this is the only option available, but more might be added in the future).
:confval:`verbosity_test_cases`: Controls how verbose the test execution output should be when pytest is executed.
Running ``pytest --no-header`` with a value of ``2`` would have the same output as the first verbosity example, but each
test inside the file gets its own line in the output.
.. _`pytest.detailed_failed_tests_usage`:

13
doc/en/reference/reference.rst
View File

@ -1865,6 +1865,19 @@ passed multiple times. The expected format is ``name=value``. For example::
"auto" can be used to explicitly use the global verbosity level.
.. confval:: verbosity_test_cases
Set a verbosity level specifically for test case execution related output, overriding the application wide level.
.. code-block:: ini
[pytest]
verbosity_test_cases = 2
Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of
"auto" can be used to explicitly use the global verbosity level.
.. confval:: xfail_strict
If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the

2
src/_pytest/config/__init__.py
View File

@ -1657,6 +1657,8 @@ class Config:
#: Verbosity type for failed assertions (see :confval:`verbosity_assertions`).
VERBOSITY_ASSERTIONS: Final = "assertions"
#: Verbosity type for test case execution (see :confval:`verbosity_test_cases`).
VERBOSITY_TEST_CASES: Final = "test_cases"
_VERBOSITY_INI_DEFAULT: Final = "auto"
def get_verbosity(self, verbosity_type: Optional[str] = None) -> int:

28
src/_pytest/terminal.py
View File

@ -255,6 +255,14 @@ def pytest_addoption(parser: Parser) -> None:
"progress even when capture=no)",
default="progress",
)
Config._add_verbosity_ini(
parser,
Config.VERBOSITY_TEST_CASES,
help=(
"Specify a verbosity level for test case execution, overriding the main level. "
"Higher levels will provide more detailed information about each test case executed."
),
)
def pytest_configure(config: Config) -> None:
@ -408,7 +416,7 @@ class TerminalReporter:
@property
def showfspath(self) -> bool:
if self._showfspath is None:
return self.verbosity >= 0
return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) >= 0
return self._showfspath
@showfspath.setter
@ -417,7 +425,7 @@ class TerminalReporter:
@property
def showlongtestinfo(self) -> bool:
return self.verbosity > 0
return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) > 0
def hasopt(self, char: str) -> bool:
char = {"xfailed": "x", "skipped": "s"}.get(char, char)
@ -595,7 +603,7 @@ class TerminalReporter:
markup = {"yellow": True}
else:
markup = {}
if self.verbosity <= 0:
if self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0:
self._tw.write(letter, **markup)
else:
self._progress_nodeids_reported.add(rep.nodeid)
@ -604,7 +612,7 @@ class TerminalReporter:
self.write_ensure_prefix(line, word, **markup)
if rep.skipped or hasattr(report, "wasxfail"):
reason = _get_raw_skip_reason(rep)
if self.config.option.verbose < 2:
if self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) < 2:
available_width = (
(self._tw.fullwidth - self._tw.width_of_current_line)
- len(" [100%]")
@ -641,7 +649,10 @@ class TerminalReporter:
def pytest_runtest_logfinish(self, nodeid: str) -> None:
assert self._session
if self.verbosity <= 0 and self._show_progress_info:
if (
self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0
and self._show_progress_info
):
if self._show_progress_info == "count":
num_tests = self._session.testscollected
progress_length = len(f" [{num_tests}/{num_tests}]")
@ -819,8 +830,9 @@ class TerminalReporter:
rep.toterminal(self._tw)
def _printcollecteditems(self, items: Sequence[Item]) -> None:
if self.config.option.verbose < 0:
if self.config.option.verbose < -1:
test_cases_verbosity = self.config.get_verbosity(Config.VERBOSITY_TEST_CASES)
if test_cases_verbosity < 0:
if test_cases_verbosity < -1:
counts = Counter(item.nodeid.split("::", 1)[0] for item in items)
for name, count in sorted(counts.items()):
self._tw.line("%s: %d" % (name, count))
@ -840,7 +852,7 @@ class TerminalReporter:
stack.append(col)
indent = (len(stack) - 1) * " "
self._tw.line(f"{indent}{col}")
if self.config.option.verbose >= 1:
if test_cases_verbosity >= 1:
obj = getattr(col, "obj", None)
doc = inspect.getdoc(obj) if obj else None
if doc:

233
testing/test_terminal.py
View File

@ -2611,6 +2611,239 @@ def test_format_trimmed() -> None:
assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) "
class TestFineGrainedTestCase:
DEFAULT_FILE_CONTENTS = """
import pytest
@pytest.mark.parametrize("i", range(4))
def test_ok(i):
'''
some docstring
'''
pass
def test_fail():
assert False
"""
LONG_SKIP_FILE_CONTENTS = """
import pytest
@pytest.mark.skip(
"some long skip reason that will not fit on a single line with other content that goes"
" on and on and on and on and on"
)
def test_skip():
pass
"""
@pytest.mark.parametrize("verbosity", [1, 2])
def test_execute_positive(self, verbosity, pytester: Pytester) -> None:
# expected: one test case per line (with file name), word describing result
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity)
result = pytester.runpytest(p)
result.stdout.fnmatch_lines(
[
"collected 5 items",
"",
f"{p.name}::test_ok[0] PASSED [ 20%]",
f"{p.name}::test_ok[1] PASSED [ 40%]",
f"{p.name}::test_ok[2] PASSED [ 60%]",
f"{p.name}::test_ok[3] PASSED [ 80%]",
f"{p.name}::test_fail FAILED [100%]",
],
consecutive=True,
)
def test_execute_0_global_1(self, pytester: Pytester) -> None:
# expected: one file name per line, single character describing result
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0)
result = pytester.runpytest("-v", p)
result.stdout.fnmatch_lines(
[
"collecting ... collected 5 items",
"",
f"{p.name} ....F [100%]",
],
consecutive=True,
)
@pytest.mark.parametrize("verbosity", [-1, -2])
def test_execute_negative(self, verbosity, pytester: Pytester) -> None:
# expected: single character describing result
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity)
result = pytester.runpytest(p)
result.stdout.fnmatch_lines(
[
"collected 5 items",
"....F [100%]",
],
consecutive=True,
)
def test_execute_skipped_positive_2(self, pytester: Pytester) -> None:
# expected: one test case per line (with file name), word describing result, full reason
p = TestFineGrainedTestCase._initialize_files(
pytester,
verbosity=2,
file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
)
result = pytester.runpytest(p)
result.stdout.fnmatch_lines(
[
"collected 1 item",
"",
f"{p.name}::test_skip SKIPPED (some long skip",
"reason that will not fit on a single line with other content that goes",
"on and on and on and on and on) [100%]",
],
consecutive=True,
)
def test_execute_skipped_positive_1(self, pytester: Pytester) -> None:
# expected: one test case per line (with file name), word describing result, reason truncated
p = TestFineGrainedTestCase._initialize_files(
pytester,
verbosity=1,
file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
)
result = pytester.runpytest(p)
result.stdout.fnmatch_lines(
[
"collected 1 item",
"",
f"{p.name}::test_skip SKIPPED (some long ski...) [100%]",
],
consecutive=True,
)
def test_execute_skipped__0_global_1(self, pytester: Pytester) -> None:
# expected: one file name per line, single character describing result (no reason)
p = TestFineGrainedTestCase._initialize_files(
pytester,
verbosity=0,
file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
)
result = pytester.runpytest("-v", p)
result.stdout.fnmatch_lines(
[
"collecting ... collected 1 item",
"",
f"{p.name} s [100%]",
],
consecutive=True,
)
@pytest.mark.parametrize("verbosity", [-1, -2])
def test_execute_skipped_negative(self, verbosity, pytester: Pytester) -> None:
# expected: single character describing result (no reason)
p = TestFineGrainedTestCase._initialize_files(
pytester,
verbosity=verbosity,
file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
)
result = pytester.runpytest(p)
result.stdout.fnmatch_lines(
[
"collected 1 item",
"s [100%]",
],
consecutive=True,
)
@pytest.mark.parametrize("verbosity", [1, 2])
def test__collect_only_positive(self, verbosity, pytester: Pytester) -> None:
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity)
result = pytester.runpytest("--collect-only", p)
result.stdout.fnmatch_lines(
[
"collected 5 items",
"",
f"<Dir {p.parent.name}>",
f" <Module {p.name}>",
" <Function test_ok[0]>",
" some docstring",
" <Function test_ok[1]>",
" some docstring",
" <Function test_ok[2]>",
" some docstring",
" <Function test_ok[3]>",
" some docstring",
" <Function test_fail>",
],
consecutive=True,
)
def test_collect_only_0_global_1(self, pytester: Pytester) -> None:
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0)
result = pytester.runpytest("-v", "--collect-only", p)
result.stdout.fnmatch_lines(
[
"collecting ... collected 5 items",
"",
f"<Dir {p.parent.name}>",
f" <Module {p.name}>",
" <Function test_ok[0]>",
" <Function test_ok[1]>",
" <Function test_ok[2]>",
" <Function test_ok[3]>",
" <Function test_fail>",
],
consecutive=True,
)
def test_collect_only_negative_1(self, pytester: Pytester) -> None:
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-1)
result = pytester.runpytest("--collect-only", p)
result.stdout.fnmatch_lines(
[
"collected 5 items",
"",
f"{p.name}::test_ok[0]",
f"{p.name}::test_ok[1]",
f"{p.name}::test_ok[2]",
f"{p.name}::test_ok[3]",
f"{p.name}::test_fail",
],
consecutive=True,
)
def test_collect_only_negative_2(self, pytester: Pytester) -> None:
p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-2)
result = pytester.runpytest("--collect-only", p)
result.stdout.fnmatch_lines(
[
"collected 5 items",
"",
f"{p.name}: 5",
],
consecutive=True,
)
@staticmethod
def _initialize_files(
pytester: Pytester, verbosity: int, file_contents: str = DEFAULT_FILE_CONTENTS
) -> Path:
p = pytester.makepyfile(file_contents)
pytester.makeini(
f"""
[pytest]
verbosity_test_cases = {verbosity}
"""
)
return p
def test_summary_xfail_reason(pytester: Pytester) -> None:
pytester.makepyfile(
"""