From 84bd31de649eea0f7ea1cb5d9269fa184dc59a97 Mon Sep 17 00:00:00 2001
From: Patrick Lannigan
Date: Sat, 24 Feb 2024 14:27:54 -0500
Subject: [PATCH] New verbosity_test_case ini option (#11653)
Allow for the output of test case execution to be controlled independently from the application verbosity level.
`verbosity_test_case` is the new ini setting to adjust this functionality.
Fix #11639
---
changelog/11653.feature.rst | 2 +
doc/en/how-to/output.rst | 4 +-
doc/en/reference/reference.rst | 13 ++
src/_pytest/config/__init__.py | 2 +
src/_pytest/terminal.py | 28 ++--
testing/test_terminal.py | 233 +++++++++++++++++++++++++++++++++
6 files changed, 273 insertions(+), 9 deletions(-)
create mode 100644 changelog/11653.feature.rst
diff --git a/changelog/11653.feature.rst b/changelog/11653.feature.rst
new file mode 100644
index 000000000..f165c3f8e
--- /dev/null
+++ b/changelog/11653.feature.rst
@@ -0,0 +1,2 @@
+Added the new :confval:`verbosity_test_cases` configuration option for fine-grained control of test execution verbosity.
+See :ref:`Fine-grained verbosity ` for more details.
diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst
index 76b2a53dd..5b47a5c77 100644
--- a/doc/en/how-to/output.rst
+++ b/doc/en/how-to/output.rst
@@ -325,7 +325,9 @@ This is done by setting a verbosity level in the configuration file for the spec
``pytest --no-header`` with a value of ``2`` would have the same output as the previous example, but each test inside
the file is shown by a single character in the output.
-(Note: currently this is the only option available, but more might be added in the future).
+:confval:`verbosity_test_cases`: Controls how verbose the test execution output should be when pytest is executed.
+Running ``pytest --no-header`` with a value of ``2`` would have the same output as the first verbosity example, but each
+test inside the file gets its own line in the output.
.. _`pytest.detailed_failed_tests_usage`:
diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst
index 1076026c2..bba4a399c 100644
--- a/doc/en/reference/reference.rst
+++ b/doc/en/reference/reference.rst
@@ -1865,6 +1865,19 @@ passed multiple times. The expected format is ``name=value``. For example::
"auto" can be used to explicitly use the global verbosity level.
+.. confval:: verbosity_test_cases
+
+ Set a verbosity level specifically for test case execution related output, overriding the application wide level.
+
+ .. code-block:: ini
+
+ [pytest]
+ verbosity_test_cases = 2
+
+ Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of
+ "auto" can be used to explicitly use the global verbosity level.
+
+
.. confval:: xfail_strict
If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the
diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py
index f8eea936a..069e2196d 100644
--- a/src/_pytest/config/__init__.py
+++ b/src/_pytest/config/__init__.py
@@ -1657,6 +1657,8 @@ class Config:
#: Verbosity type for failed assertions (see :confval:`verbosity_assertions`).
VERBOSITY_ASSERTIONS: Final = "assertions"
+ #: Verbosity type for test case execution (see :confval:`verbosity_test_cases`).
+ VERBOSITY_TEST_CASES: Final = "test_cases"
_VERBOSITY_INI_DEFAULT: Final = "auto"
def get_verbosity(self, verbosity_type: Optional[str] = None) -> int:
diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py
index 8909d95ef..75d57197a 100644
--- a/src/_pytest/terminal.py
+++ b/src/_pytest/terminal.py
@@ -255,6 +255,14 @@ def pytest_addoption(parser: Parser) -> None:
"progress even when capture=no)",
default="progress",
)
+ Config._add_verbosity_ini(
+ parser,
+ Config.VERBOSITY_TEST_CASES,
+ help=(
+ "Specify a verbosity level for test case execution, overriding the main level. "
+ "Higher levels will provide more detailed information about each test case executed."
+ ),
+ )
def pytest_configure(config: Config) -> None:
@@ -408,7 +416,7 @@ class TerminalReporter:
@property
def showfspath(self) -> bool:
if self._showfspath is None:
- return self.verbosity >= 0
+ return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) >= 0
return self._showfspath
@showfspath.setter
@@ -417,7 +425,7 @@ class TerminalReporter:
@property
def showlongtestinfo(self) -> bool:
- return self.verbosity > 0
+ return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) > 0
def hasopt(self, char: str) -> bool:
char = {"xfailed": "x", "skipped": "s"}.get(char, char)
@@ -595,7 +603,7 @@ class TerminalReporter:
markup = {"yellow": True}
else:
markup = {}
- if self.verbosity <= 0:
+ if self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0:
self._tw.write(letter, **markup)
else:
self._progress_nodeids_reported.add(rep.nodeid)
@@ -604,7 +612,7 @@ class TerminalReporter:
self.write_ensure_prefix(line, word, **markup)
if rep.skipped or hasattr(report, "wasxfail"):
reason = _get_raw_skip_reason(rep)
- if self.config.option.verbose < 2:
+ if self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) < 2:
available_width = (
(self._tw.fullwidth - self._tw.width_of_current_line)
- len(" [100%]")
@@ -641,7 +649,10 @@ class TerminalReporter:
def pytest_runtest_logfinish(self, nodeid: str) -> None:
assert self._session
- if self.verbosity <= 0 and self._show_progress_info:
+ if (
+ self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0
+ and self._show_progress_info
+ ):
if self._show_progress_info == "count":
num_tests = self._session.testscollected
progress_length = len(f" [{num_tests}/{num_tests}]")
@@ -819,8 +830,9 @@ class TerminalReporter:
rep.toterminal(self._tw)
def _printcollecteditems(self, items: Sequence[Item]) -> None:
- if self.config.option.verbose < 0:
- if self.config.option.verbose < -1:
+ test_cases_verbosity = self.config.get_verbosity(Config.VERBOSITY_TEST_CASES)
+ if test_cases_verbosity < 0:
+ if test_cases_verbosity < -1:
counts = Counter(item.nodeid.split("::", 1)[0] for item in items)
for name, count in sorted(counts.items()):
self._tw.line("%s: %d" % (name, count))
@@ -840,7 +852,7 @@ class TerminalReporter:
stack.append(col)
indent = (len(stack) - 1) * " "
self._tw.line(f"{indent}{col}")
- if self.config.option.verbose >= 1:
+ if test_cases_verbosity >= 1:
obj = getattr(col, "obj", None)
doc = inspect.getdoc(obj) if obj else None
if doc:
diff --git a/testing/test_terminal.py b/testing/test_terminal.py
index bc457c398..b311d6c9b 100644
--- a/testing/test_terminal.py
+++ b/testing/test_terminal.py
@@ -2611,6 +2611,239 @@ def test_format_trimmed() -> None:
assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) "
+class TestFineGrainedTestCase:
+ DEFAULT_FILE_CONTENTS = """
+ import pytest
+
+ @pytest.mark.parametrize("i", range(4))
+ def test_ok(i):
+ '''
+ some docstring
+ '''
+ pass
+
+ def test_fail():
+ assert False
+ """
+ LONG_SKIP_FILE_CONTENTS = """
+ import pytest
+
+ @pytest.mark.skip(
+ "some long skip reason that will not fit on a single line with other content that goes"
+ " on and on and on and on and on"
+ )
+ def test_skip():
+ pass
+ """
+
+ @pytest.mark.parametrize("verbosity", [1, 2])
+ def test_execute_positive(self, verbosity, pytester: Pytester) -> None:
+ # expected: one test case per line (with file name), word describing result
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity)
+ result = pytester.runpytest(p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 5 items",
+ "",
+ f"{p.name}::test_ok[0] PASSED [ 20%]",
+ f"{p.name}::test_ok[1] PASSED [ 40%]",
+ f"{p.name}::test_ok[2] PASSED [ 60%]",
+ f"{p.name}::test_ok[3] PASSED [ 80%]",
+ f"{p.name}::test_fail FAILED [100%]",
+ ],
+ consecutive=True,
+ )
+
+ def test_execute_0_global_1(self, pytester: Pytester) -> None:
+ # expected: one file name per line, single character describing result
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0)
+ result = pytester.runpytest("-v", p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collecting ... collected 5 items",
+ "",
+ f"{p.name} ....F [100%]",
+ ],
+ consecutive=True,
+ )
+
+ @pytest.mark.parametrize("verbosity", [-1, -2])
+ def test_execute_negative(self, verbosity, pytester: Pytester) -> None:
+ # expected: single character describing result
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity)
+ result = pytester.runpytest(p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 5 items",
+ "....F [100%]",
+ ],
+ consecutive=True,
+ )
+
+ def test_execute_skipped_positive_2(self, pytester: Pytester) -> None:
+ # expected: one test case per line (with file name), word describing result, full reason
+ p = TestFineGrainedTestCase._initialize_files(
+ pytester,
+ verbosity=2,
+ file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
+ )
+ result = pytester.runpytest(p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 1 item",
+ "",
+ f"{p.name}::test_skip SKIPPED (some long skip",
+ "reason that will not fit on a single line with other content that goes",
+ "on and on and on and on and on) [100%]",
+ ],
+ consecutive=True,
+ )
+
+ def test_execute_skipped_positive_1(self, pytester: Pytester) -> None:
+ # expected: one test case per line (with file name), word describing result, reason truncated
+ p = TestFineGrainedTestCase._initialize_files(
+ pytester,
+ verbosity=1,
+ file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
+ )
+ result = pytester.runpytest(p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 1 item",
+ "",
+ f"{p.name}::test_skip SKIPPED (some long ski...) [100%]",
+ ],
+ consecutive=True,
+ )
+
+ def test_execute_skipped__0_global_1(self, pytester: Pytester) -> None:
+ # expected: one file name per line, single character describing result (no reason)
+ p = TestFineGrainedTestCase._initialize_files(
+ pytester,
+ verbosity=0,
+ file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
+ )
+ result = pytester.runpytest("-v", p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collecting ... collected 1 item",
+ "",
+ f"{p.name} s [100%]",
+ ],
+ consecutive=True,
+ )
+
+ @pytest.mark.parametrize("verbosity", [-1, -2])
+ def test_execute_skipped_negative(self, verbosity, pytester: Pytester) -> None:
+ # expected: single character describing result (no reason)
+ p = TestFineGrainedTestCase._initialize_files(
+ pytester,
+ verbosity=verbosity,
+ file_contents=TestFineGrainedTestCase.LONG_SKIP_FILE_CONTENTS,
+ )
+ result = pytester.runpytest(p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 1 item",
+ "s [100%]",
+ ],
+ consecutive=True,
+ )
+
+ @pytest.mark.parametrize("verbosity", [1, 2])
+ def test__collect_only_positive(self, verbosity, pytester: Pytester) -> None:
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=verbosity)
+ result = pytester.runpytest("--collect-only", p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 5 items",
+ "",
+ f"",
+ f" ",
+ " ",
+ " some docstring",
+ " ",
+ " some docstring",
+ " ",
+ " some docstring",
+ " ",
+ " some docstring",
+ " ",
+ ],
+ consecutive=True,
+ )
+
+ def test_collect_only_0_global_1(self, pytester: Pytester) -> None:
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=0)
+ result = pytester.runpytest("-v", "--collect-only", p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collecting ... collected 5 items",
+ "",
+ f"",
+ f" ",
+ " ",
+ " ",
+ " ",
+ " ",
+ " ",
+ ],
+ consecutive=True,
+ )
+
+ def test_collect_only_negative_1(self, pytester: Pytester) -> None:
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-1)
+ result = pytester.runpytest("--collect-only", p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 5 items",
+ "",
+ f"{p.name}::test_ok[0]",
+ f"{p.name}::test_ok[1]",
+ f"{p.name}::test_ok[2]",
+ f"{p.name}::test_ok[3]",
+ f"{p.name}::test_fail",
+ ],
+ consecutive=True,
+ )
+
+ def test_collect_only_negative_2(self, pytester: Pytester) -> None:
+ p = TestFineGrainedTestCase._initialize_files(pytester, verbosity=-2)
+ result = pytester.runpytest("--collect-only", p)
+
+ result.stdout.fnmatch_lines(
+ [
+ "collected 5 items",
+ "",
+ f"{p.name}: 5",
+ ],
+ consecutive=True,
+ )
+
+ @staticmethod
+ def _initialize_files(
+ pytester: Pytester, verbosity: int, file_contents: str = DEFAULT_FILE_CONTENTS
+ ) -> Path:
+ p = pytester.makepyfile(file_contents)
+ pytester.makeini(
+ f"""
+ [pytest]
+ verbosity_test_cases = {verbosity}
+ """
+ )
+ return p
+
+
def test_summary_xfail_reason(pytester: Pytester) -> None:
pytester.makepyfile(
"""