Add verbosity_assertions and config.get_verbosity

Fixes #11387

AUTHORS

@@ -293,6 +293,7 @@ Ondřej Súkup
 Oscar Benjamin
 Parth Patel
 Patrick Hayes
+Patrick Lannigan
 Paul Müller
 Paul Reece
 Pauli Virtanen

changelog/11387.feature.rst

@@ -0,0 +1,5 @@
+Added the new :confval:`verbosity_assertions` configuration option for fine-grained control of failed assertions verbosity.
+
+See :ref:`Fine-grained verbosity <pytest.fine_grained_verbosity>` for more details.
+
+For plugin authors, :attr:`config.get_verbosity <pytest.Config.get_verbosity>` can be used to retrieve the verbosity level for a specific verbosity type.

doc/en/how-to/output.rst

@@ -286,6 +286,20 @@ situations, for example you are shown even fixtures that start with ``_`` if you
 Using higher verbosity levels (``-vvv``, ``-vvvv``, ...) is supported, but has no effect in pytest itself at the moment,
 however some plugins might make use of higher verbosity.
 
+.. _`pytest.fine_grained_verbosity`:
+
+Fine-grained verbosity
+~~~~~~~~~~~~~~~~~~~~~~
+
+In addition to specifying the application wide verbosity level, it is possible to control specific aspects independently.
+This is done by setting a verbosity level in the configuration file for the specific aspect of the output.
+
+:confval:`verbosity_assertions`: Controls how verbose the assertion output should be when pytest is executed. Running
+``pytest --no-header`` with a value of ``2`` would have the same output as the previous example, but each test inside
+the file is shown by a single character in the output.
+
+(Note: currently this is the only option available, but more might be added in the future).
+
 .. _`pytest.detailed_failed_tests_usage`:
 
 Producing a detailed summary report

doc/en/reference/reference.rst

@@ -1822,6 +1822,19 @@ passed multiple times. The expected format is ``name=value``. For example::
             clean_db
 
 
+.. confval:: verbosity_assertions
+
+    Set a verbosity level specifically for assertion related output, overriding the application wide level.
+
+    .. code-block:: ini
+
+        [pytest]
+        verbosity_assertions = 2
+
+    Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of
+    "auto" can be used to explicitly use the global verbosity level.
+
+
 .. confval:: xfail_strict
 
     If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the

src/_pytest/assertion/__init__.py

@@ -42,6 +42,14 @@ def pytest_addoption(parser: Parser) -> None:
         help="Enables the pytest_assertion_pass hook. "
         "Make sure to delete any previously generated pyc cache files.",
     )
+    Config._add_verbosity_ini(
+        parser,
+        Config.VERBOSITY_ASSERTIONS,
+        help=(
+            "Specify a verbosity level for assertions, overriding the main level. "
+            "Higher levels will provide more detailed explanation when an assertion fails."
+        ),
+    )
 
 
 def register_assert_rewrite(*names: str) -> None:

src/_pytest/assertion/rewrite.py

@@ -426,7 +426,10 @@ def _saferepr(obj: object) -> str:
 
 def _get_maxsize_for_saferepr(config: Optional[Config]) -> Optional[int]:
     """Get `maxsize` configuration for saferepr based on the given config object."""
-    verbosity = config.getoption("verbose") if config is not None else 0
+    if config is None:
+        verbosity = 0
+    else:
+        verbosity = config.get_verbosity(Config.VERBOSITY_ASSERTIONS)
     if verbosity >= 2:
         return None
     if verbosity >= 1:

src/_pytest/assertion/truncate.py

@@ -1,12 +1,13 @@
 """Utilities for truncating assertion output.
 
 Current default behaviour is to truncate assertion explanations at
-~8 terminal lines, unless running in "-vv" mode or running on CI.
+terminal lines, unless running with an assertions verbosity level of at least 2 or running on CI.
 """
 from typing import List
 from typing import Optional
 
 from _pytest.assertion import util
+from _pytest.config import Config
 from _pytest.nodes import Item
 
 
@@ -26,7 +27,7 @@ def truncate_if_required(
 
 def _should_truncate_item(item: Item) -> bool:
     """Whether or not this test item is eligible for truncation."""
-    verbose = item.config.option.verbose
+    verbose = item.config.get_verbosity(Config.VERBOSITY_ASSERTIONS)
     return verbose < 2 and not util.running_on_ci()
 
 

src/_pytest/assertion/util.py

@@ -168,7 +168,7 @@ def assertrepr_compare(
     config, op: str, left: Any, right: Any, use_ascii: bool = False
 ) -> Optional[List[str]]:
     """Return specialised explanations for some operators/operands."""
-    verbose = config.getoption("verbose")
+    verbose = config.get_verbosity(Config.VERBOSITY_ASSERTIONS)
 
     # Strings which normalize equal are often hard to distinguish when printed; use ascii() to make this easier.
     # See issue #3246.

src/_pytest/config/__init__.py

@@ -22,6 +22,7 @@ from typing import Any
 from typing import Callable
 from typing import cast
 from typing import Dict
+from typing import Final
 from typing import final
 from typing import Generator
 from typing import IO
@@ -69,7 +70,7 @@ from _pytest.warning_types import warn_explicit_for
 if TYPE_CHECKING:
     from _pytest._code.code import _TracebackStyle
     from _pytest.terminal import TerminalReporter
-    from .argparsing import Argument
+    from .argparsing import Argument, Parser
 
 
 _PluggyPlugin = object
@@ -1650,6 +1651,78 @@ class Config:
         """Deprecated, use getoption(skip=True) instead."""
         return self.getoption(name, skip=True)
 
+    #: Verbosity type for failed assertions (see :confval:`verbosity_assertions`).
+    VERBOSITY_ASSERTIONS: Final = "assertions"
+    _VERBOSITY_INI_DEFAULT: Final = "auto"
+
+    def get_verbosity(self, verbosity_type: Optional[str] = None) -> int:
+        r"""Retrieve the verbosity level for a fine-grained verbosity type.
+
+        :param verbosity_type: Verbosity type to get level for. If a level is
+            configured for the given type, that value will be returned. If the
+            given type is not a known verbosity type, the global verbosity
+            level will be returned. If the given type is None (default), the
+            global verbosity level will be returned.
+
+        To configure a level for a fine-grained verbosity type, the
+        configuration file should have a setting for the configuration name
+        and a numeric value for the verbosity level. A special value of "auto"
+        can be used to explicitly use the global verbosity level.
+
+        Example:
+
+        .. code-block:: ini
+
+            # content of pytest.ini
+            [pytest]
+            verbosity_assertions = 2
+
+        .. code-block:: console
+
+            pytest -v
+
+        .. code-block:: python
+
+            print(config.get_verbosity())  # 1
+            print(config.get_verbosity(Config.VERBOSITY_ASSERTIONS))  # 2
+        """
+        global_level = self.option.verbose
+        assert isinstance(global_level, int)
+        if verbosity_type is None:
+            return global_level
+
+        ini_name = Config._verbosity_ini_name(verbosity_type)
+        if ini_name not in self._parser._inidict:
+            return global_level
+
+        level = self.getini(ini_name)
+        if level == Config._VERBOSITY_INI_DEFAULT:
+            return global_level
+
+        return int(level)
+
+    @staticmethod
+    def _verbosity_ini_name(verbosity_type: str) -> str:
+        return f"verbosity_{verbosity_type}"
+
+    @staticmethod
+    def _add_verbosity_ini(parser: "Parser", verbosity_type: str, help: str) -> None:
+        """Add a output verbosity configuration option for the given output type.
+
+        :param parser: Parser for command line arguments and ini-file values.
+        :param verbosity_type: Fine-grained verbosity category.
+        :param help: Description of the output this type controls.
+
+        The value should be retrieved via a call to
+        :py:func:`config.get_verbosity(type) <pytest.Config.get_verbosity>`.
+        """
+        parser.addini(
+            Config._verbosity_ini_name(verbosity_type),
+            help=help,
+            type="string",
+            default=Config._VERBOSITY_INI_DEFAULT,
+        )
+
     def _warn_about_missing_assertion(self, mode: str) -> None:
         if not _assertion_supported():
             if mode == "plain":

testing/test_assertion.py

@@ -13,27 +13,68 @@ import pytest
 from _pytest import outcomes
 from _pytest.assertion import truncate
 from _pytest.assertion import util
+from _pytest.config import Config as _Config
 from _pytest.monkeypatch import MonkeyPatch
 from _pytest.pytester import Pytester
 
 
-def mock_config(verbose=0):
+def mock_config(verbose: int = 0, assertion_override: Optional[int] = None):
     class TerminalWriter:
         def _highlight(self, source, lexer):
             return source
 
     class Config:
-        def getoption(self, name):
-            if name == "verbose":
-                return verbose
-            raise KeyError("Not mocked out: %s" % name)
-
         def get_terminal_writer(self):
             return TerminalWriter()
 
+        def get_verbosity(self, verbosity_type: Optional[str] = None) -> int:
+            if verbosity_type is None:
+                return verbose
+            if verbosity_type == _Config.VERBOSITY_ASSERTIONS:
+                if assertion_override is not None:
+                    return assertion_override
+                return verbose
+
+            raise KeyError(f"Not mocked out: {verbosity_type}")
+
     return Config()
 
 
+class TestMockConfig:
+    SOME_VERBOSITY_LEVEL = 3
+    SOME_OTHER_VERBOSITY_LEVEL = 10
+
+    def test_verbose_exposes_value(self):
+        config = mock_config(verbose=TestMockConfig.SOME_VERBOSITY_LEVEL)
+
+        assert config.get_verbosity() == TestMockConfig.SOME_VERBOSITY_LEVEL
+
+    def test_get_assertion_override_not_set_verbose_value(self):
+        config = mock_config(verbose=TestMockConfig.SOME_VERBOSITY_LEVEL)
+
+        assert (
+            config.get_verbosity(_Config.VERBOSITY_ASSERTIONS)
+            == TestMockConfig.SOME_VERBOSITY_LEVEL
+        )
+
+    def test_get_assertion_override_set_custom_value(self):
+        config = mock_config(
+            verbose=TestMockConfig.SOME_VERBOSITY_LEVEL,
+            assertion_override=TestMockConfig.SOME_OTHER_VERBOSITY_LEVEL,
+        )
+
+        assert (
+            config.get_verbosity(_Config.VERBOSITY_ASSERTIONS)
+            == TestMockConfig.SOME_OTHER_VERBOSITY_LEVEL
+        )
+
+    def test_get_unsupported_type_error(self):
+        config = mock_config(verbose=TestMockConfig.SOME_VERBOSITY_LEVEL)
+
+        with pytest.raises(KeyError):
+            config.get_verbosity("--- NOT A VERBOSITY LEVEL ---")
+
+
 class TestImportHookInstallation:
     @pytest.mark.parametrize("initial_conftest", [True, False])
     @pytest.mark.parametrize("mode", ["plain", "rewrite"])
@@ -1836,3 +1877,54 @@ def test_comparisons_handle_colors(
     )
 
     result.stdout.fnmatch_lines(formatter(expected_lines), consecutive=False)
+
+
+def test_fine_grained_assertion_verbosity(pytester: Pytester):
+    long_text = "Lorem ipsum dolor sit amet " * 10
+    p = pytester.makepyfile(
+        f"""
+        def test_ok():
+            pass
+
+
+        def test_words_fail():
+            fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"]
+            fruits2 = ["banana", "apple", "orange", "melon", "kiwi"]
+            assert fruits1 == fruits2
+
+
+        def test_numbers_fail():
+            number_to_text1 = {{str(x): x for x in range(5)}}
+            number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}}
+            assert number_to_text1 == number_to_text2
+
+
+        def test_long_text_fail():
+            long_text = "{long_text}"
+            assert "hello world" in long_text
+        """
+    )
+    pytester.makeini(
+        """
+        [pytest]
+        verbosity_assertions = 2
+        """
+    )
+    result = pytester.runpytest(p)
+
+    result.stdout.fnmatch_lines(
+        [
+            f"{p.name} .FFF                            [100%]",
+            "E         At index 2 diff: 'grapes' != 'orange'",
+            "E         Full diff:",
+            "E         - ['banana', 'apple', 'orange', 'melon', 'kiwi']",
+            "E         ?                      ^  ^^",
+            "E         + ['banana', 'apple', 'grapes', 'melon', 'kiwi']",
+            "E         ?                      ^  ^ +",
+            "E         Full diff:",
+            "E         - {'0': 0, '10': 10, '20': 20, '30': 30, '40': 40}",
+            "E         ?            -    -    -    -    -    -    -    -",
+            "E         + {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4}",
+            f"E       AssertionError: assert 'hello world' in '{long_text}'",
+        ]
+    )

testing/test_assertrewrite.py

@@ -2056,13 +2056,15 @@ class TestReprSizeVerbosity:
     )
     def test_get_maxsize_for_saferepr(self, verbose: int, expected_size) -> None:
         class FakeConfig:
-            def getoption(self, name: str) -> int:
-                assert name == "verbose"
+            def get_verbosity(self, verbosity_type: Optional[str] = None) -> int:
                 return verbose
 
         config = FakeConfig()
         assert _get_maxsize_for_saferepr(cast(Config, config)) == expected_size
 
+    def test_get_maxsize_for_saferepr_no_config(self) -> None:
+        assert _get_maxsize_for_saferepr(None) == DEFAULT_REPR_MAX_SIZE
+
     def create_test_file(self, pytester: Pytester, size: int) -> None:
         pytester.makepyfile(
             f"""

testing/test_config.py

@@ -23,6 +23,7 @@ from _pytest.config import ConftestImportFailure
 from _pytest.config import ExitCode
 from _pytest.config import parse_warning_filter
 from _pytest.config.argparsing import get_ini_default_for_type
+from _pytest.config.argparsing import Parser
 from _pytest.config.exceptions import UsageError
 from _pytest.config.findpaths import determine_setup
 from _pytest.config.findpaths import get_common_ancestor
@@ -2245,3 +2246,76 @@ class TestDebugOptions:
                 "*Default: pytestdebug.log.",
             ]
         )
+
+
+class TestVerbosity:
+    SOME_OUTPUT_TYPE = Config.VERBOSITY_ASSERTIONS
+    SOME_OUTPUT_VERBOSITY_LEVEL = 5
+
+    class VerbosityIni:
+        def pytest_addoption(self, parser: Parser) -> None:
+            Config._add_verbosity_ini(
+                parser, TestVerbosity.SOME_OUTPUT_TYPE, help="some help text"
+            )
+
+    def test_level_matches_verbose_when_not_specified(
+        self, pytester: Pytester, tmp_path: Path
+    ) -> None:
+        tmp_path.joinpath("pytest.ini").write_text(
+            textwrap.dedent(
+                """\
+                [pytest]
+                addopts = --verbose
+                """
+            ),
+            encoding="utf-8",
+        )
+        pytester.plugins = [TestVerbosity.VerbosityIni()]
+
+        config = pytester.parseconfig(tmp_path)
+
+        assert (
+            config.get_verbosity(TestVerbosity.SOME_OUTPUT_TYPE)
+            == config.option.verbose
+        )
+
+    def test_level_matches_verbose_when_not_known_type(
+        self, pytester: Pytester, tmp_path: Path
+    ) -> None:
+        tmp_path.joinpath("pytest.ini").write_text(
+            textwrap.dedent(
+                """\
+                [pytest]
+                addopts = --verbose
+                """
+            ),
+            encoding="utf-8",
+        )
+        pytester.plugins = [TestVerbosity.VerbosityIni()]
+
+        config = pytester.parseconfig(tmp_path)
+
+        assert config.get_verbosity("some fake verbosity type") == config.option.verbose
+
+    def test_level_matches_specified_override(
+        self, pytester: Pytester, tmp_path: Path
+    ) -> None:
+        setting_name = f"verbosity_{TestVerbosity.SOME_OUTPUT_TYPE}"
+        tmp_path.joinpath("pytest.ini").write_text(
+            textwrap.dedent(
+                f"""\
+                [pytest]
+                addopts = --verbose
+                {setting_name} = {TestVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL}
+                """
+            ),
+            encoding="utf-8",
+        )
+        pytester.plugins = [TestVerbosity.VerbosityIni()]
+
+        config = pytester.parseconfig(tmp_path)
+
+        assert (
+            config.get_verbosity(TestVerbosity.SOME_OUTPUT_TYPE)
+            == TestVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL
+        )