From 9dc1fc4523e60f0c8795e9693e47f35769a66471 Mon Sep 17 00:00:00 2001 From: Patrick Lannigan Date: Sun, 19 Nov 2023 09:56:29 -0500 Subject: [PATCH] Add verbosity_assertions and config.get_verbosity Fixes #11387 --- AUTHORS | 1 + changelog/11387.feature.rst | 5 ++ doc/en/how-to/output.rst | 14 ++++ doc/en/reference/reference.rst | 13 ++++ src/_pytest/assertion/__init__.py | 8 +++ src/_pytest/assertion/rewrite.py | 5 +- src/_pytest/assertion/truncate.py | 5 +- src/_pytest/assertion/util.py | 2 +- src/_pytest/config/__init__.py | 75 ++++++++++++++++++++- testing/test_assertion.py | 104 ++++++++++++++++++++++++++++-- testing/test_assertrewrite.py | 6 +- testing/test_config.py | 74 +++++++++++++++++++++ 12 files changed, 299 insertions(+), 13 deletions(-) create mode 100644 changelog/11387.feature.rst diff --git a/AUTHORS b/AUTHORS index d8e901715..e30131d1a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -293,6 +293,7 @@ Ondřej Súkup Oscar Benjamin Parth Patel Patrick Hayes +Patrick Lannigan Paul Müller Paul Reece Pauli Virtanen diff --git a/changelog/11387.feature.rst b/changelog/11387.feature.rst new file mode 100644 index 000000000..90f20885b --- /dev/null +++ b/changelog/11387.feature.rst @@ -0,0 +1,5 @@ +Added the new :confval:`verbosity_assertions` configuration option for fine-grained control of failed assertions verbosity. + +See :ref:`Fine-grained verbosity ` for more details. + +For plugin authors, :attr:`config.get_verbosity ` can be used to retrieve the verbosity level for a specific verbosity type. diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index 04f201610..8af9a38b7 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -286,6 +286,20 @@ situations, for example you are shown even fixtures that start with ``_`` if you Using higher verbosity levels (``-vvv``, ``-vvvv``, ...) is supported, but has no effect in pytest itself at the moment, however some plugins might make use of higher verbosity. +.. _`pytest.fine_grained_verbosity`: + +Fine-grained verbosity +~~~~~~~~~~~~~~~~~~~~~~ + +In addition to specifying the application wide verbosity level, it is possible to control specific aspects independently. +This is done by setting a verbosity level in the configuration file for the specific aspect of the output. + +:confval:`verbosity_assertions`: Controls how verbose the assertion output should be when pytest is executed. Running +``pytest --no-header`` with a value of ``2`` would have the same output as the previous example, but each test inside +the file is shown by a single character in the output. + +(Note: currently this is the only option available, but more might be added in the future). + .. _`pytest.detailed_failed_tests_usage`: Producing a detailed summary report diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index f6dabb245..254973709 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1822,6 +1822,19 @@ passed multiple times. The expected format is ``name=value``. For example:: clean_db +.. confval:: verbosity_assertions + + Set a verbosity level specifically for assertion related output, overriding the application wide level. + + .. code-block:: ini + + [pytest] + verbosity_assertions = 2 + + Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of + "auto" can be used to explicitly use the global verbosity level. + + .. confval:: xfail_strict If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index 64ad4b0e6..e1e7a5e66 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -42,6 +42,14 @@ def pytest_addoption(parser: Parser) -> None: help="Enables the pytest_assertion_pass hook. " "Make sure to delete any previously generated pyc cache files.", ) + Config._add_verbosity_ini( + parser, + Config.VERBOSITY_ASSERTIONS, + help=( + "Specify a verbosity level for assertions, overriding the main level. " + "Higher levels will provide more detailed explanation when an assertion fails." + ), + ) def register_assert_rewrite(*names: str) -> None: diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py index 258ed9f9a..149101e71 100644 --- a/src/_pytest/assertion/rewrite.py +++ b/src/_pytest/assertion/rewrite.py @@ -426,7 +426,10 @@ def _saferepr(obj: object) -> str: def _get_maxsize_for_saferepr(config: Optional[Config]) -> Optional[int]: """Get `maxsize` configuration for saferepr based on the given config object.""" - verbosity = config.getoption("verbose") if config is not None else 0 + if config is None: + verbosity = 0 + else: + verbosity = config.get_verbosity(Config.VERBOSITY_ASSERTIONS) if verbosity >= 2: return None if verbosity >= 1: diff --git a/src/_pytest/assertion/truncate.py b/src/_pytest/assertion/truncate.py index dfd6f65d2..16de27f25 100644 --- a/src/_pytest/assertion/truncate.py +++ b/src/_pytest/assertion/truncate.py @@ -1,12 +1,13 @@ """Utilities for truncating assertion output. Current default behaviour is to truncate assertion explanations at -~8 terminal lines, unless running in "-vv" mode or running on CI. +terminal lines, unless running with an assertions verbosity level of at least 2 or running on CI. """ from typing import List from typing import Optional from _pytest.assertion import util +from _pytest.config import Config from _pytest.nodes import Item @@ -26,7 +27,7 @@ def truncate_if_required( def _should_truncate_item(item: Item) -> bool: """Whether or not this test item is eligible for truncation.""" - verbose = item.config.option.verbose + verbose = item.config.get_verbosity(Config.VERBOSITY_ASSERTIONS) return verbose < 2 and not util.running_on_ci() diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index b9123c97d..65abe8d23 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -168,7 +168,7 @@ def assertrepr_compare( config, op: str, left: Any, right: Any, use_ascii: bool = False ) -> Optional[List[str]]: """Return specialised explanations for some operators/operands.""" - verbose = config.getoption("verbose") + verbose = config.get_verbosity(Config.VERBOSITY_ASSERTIONS) # Strings which normalize equal are often hard to distinguish when printed; use ascii() to make this easier. # See issue #3246. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 03f69ed31..ea23c7742 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -22,6 +22,7 @@ from typing import Any from typing import Callable from typing import cast from typing import Dict +from typing import Final from typing import final from typing import Generator from typing import IO @@ -69,7 +70,7 @@ from _pytest.warning_types import warn_explicit_for if TYPE_CHECKING: from _pytest._code.code import _TracebackStyle from _pytest.terminal import TerminalReporter - from .argparsing import Argument + from .argparsing import Argument, Parser _PluggyPlugin = object @@ -1650,6 +1651,78 @@ class Config: """Deprecated, use getoption(skip=True) instead.""" return self.getoption(name, skip=True) + #: Verbosity type for failed assertions (see :confval:`verbosity_assertions`). + VERBOSITY_ASSERTIONS: Final = "assertions" + _VERBOSITY_INI_DEFAULT: Final = "auto" + + def get_verbosity(self, verbosity_type: Optional[str] = None) -> int: + r"""Retrieve the verbosity level for a fine-grained verbosity type. + + :param verbosity_type: Verbosity type to get level for. If a level is + configured for the given type, that value will be returned. If the + given type is not a known verbosity type, the global verbosity + level will be returned. If the given type is None (default), the + global verbosity level will be returned. + + To configure a level for a fine-grained verbosity type, the + configuration file should have a setting for the configuration name + and a numeric value for the verbosity level. A special value of "auto" + can be used to explicitly use the global verbosity level. + + Example: + + .. code-block:: ini + + # content of pytest.ini + [pytest] + verbosity_assertions = 2 + + .. code-block:: console + + pytest -v + + .. code-block:: python + + print(config.get_verbosity()) # 1 + print(config.get_verbosity(Config.VERBOSITY_ASSERTIONS)) # 2 + """ + global_level = self.option.verbose + assert isinstance(global_level, int) + if verbosity_type is None: + return global_level + + ini_name = Config._verbosity_ini_name(verbosity_type) + if ini_name not in self._parser._inidict: + return global_level + + level = self.getini(ini_name) + if level == Config._VERBOSITY_INI_DEFAULT: + return global_level + + return int(level) + + @staticmethod + def _verbosity_ini_name(verbosity_type: str) -> str: + return f"verbosity_{verbosity_type}" + + @staticmethod + def _add_verbosity_ini(parser: "Parser", verbosity_type: str, help: str) -> None: + """Add a output verbosity configuration option for the given output type. + + :param parser: Parser for command line arguments and ini-file values. + :param verbosity_type: Fine-grained verbosity category. + :param help: Description of the output this type controls. + + The value should be retrieved via a call to + :py:func:`config.get_verbosity(type) `. + """ + parser.addini( + Config._verbosity_ini_name(verbosity_type), + help=help, + type="string", + default=Config._VERBOSITY_INI_DEFAULT, + ) + def _warn_about_missing_assertion(self, mode: str) -> None: if not _assertion_supported(): if mode == "plain": diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 62c465d8a..7c8c01556 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -13,27 +13,68 @@ import pytest from _pytest import outcomes from _pytest.assertion import truncate from _pytest.assertion import util +from _pytest.config import Config as _Config from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester -def mock_config(verbose=0): +def mock_config(verbose: int = 0, assertion_override: Optional[int] = None): class TerminalWriter: def _highlight(self, source, lexer): return source class Config: - def getoption(self, name): - if name == "verbose": - return verbose - raise KeyError("Not mocked out: %s" % name) - def get_terminal_writer(self): return TerminalWriter() + def get_verbosity(self, verbosity_type: Optional[str] = None) -> int: + if verbosity_type is None: + return verbose + if verbosity_type == _Config.VERBOSITY_ASSERTIONS: + if assertion_override is not None: + return assertion_override + return verbose + + raise KeyError(f"Not mocked out: {verbosity_type}") + return Config() +class TestMockConfig: + SOME_VERBOSITY_LEVEL = 3 + SOME_OTHER_VERBOSITY_LEVEL = 10 + + def test_verbose_exposes_value(self): + config = mock_config(verbose=TestMockConfig.SOME_VERBOSITY_LEVEL) + + assert config.get_verbosity() == TestMockConfig.SOME_VERBOSITY_LEVEL + + def test_get_assertion_override_not_set_verbose_value(self): + config = mock_config(verbose=TestMockConfig.SOME_VERBOSITY_LEVEL) + + assert ( + config.get_verbosity(_Config.VERBOSITY_ASSERTIONS) + == TestMockConfig.SOME_VERBOSITY_LEVEL + ) + + def test_get_assertion_override_set_custom_value(self): + config = mock_config( + verbose=TestMockConfig.SOME_VERBOSITY_LEVEL, + assertion_override=TestMockConfig.SOME_OTHER_VERBOSITY_LEVEL, + ) + + assert ( + config.get_verbosity(_Config.VERBOSITY_ASSERTIONS) + == TestMockConfig.SOME_OTHER_VERBOSITY_LEVEL + ) + + def test_get_unsupported_type_error(self): + config = mock_config(verbose=TestMockConfig.SOME_VERBOSITY_LEVEL) + + with pytest.raises(KeyError): + config.get_verbosity("--- NOT A VERBOSITY LEVEL ---") + + class TestImportHookInstallation: @pytest.mark.parametrize("initial_conftest", [True, False]) @pytest.mark.parametrize("mode", ["plain", "rewrite"]) @@ -1836,3 +1877,54 @@ def test_comparisons_handle_colors( ) result.stdout.fnmatch_lines(formatter(expected_lines), consecutive=False) + + +def test_fine_grained_assertion_verbosity(pytester: Pytester): + long_text = "Lorem ipsum dolor sit amet " * 10 + p = pytester.makepyfile( + f""" + def test_ok(): + pass + + + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + assert fruits1 == fruits2 + + + def test_numbers_fail(): + number_to_text1 = {{str(x): x for x in range(5)}} + number_to_text2 = {{str(x * 10): x * 10 for x in range(5)}} + assert number_to_text1 == number_to_text2 + + + def test_long_text_fail(): + long_text = "{long_text}" + assert "hello world" in long_text + """ + ) + pytester.makeini( + """ + [pytest] + verbosity_assertions = 2 + """ + ) + result = pytester.runpytest(p) + + result.stdout.fnmatch_lines( + [ + f"{p.name} .FFF [100%]", + "E At index 2 diff: 'grapes' != 'orange'", + "E Full diff:", + "E - ['banana', 'apple', 'orange', 'melon', 'kiwi']", + "E ? ^ ^^", + "E + ['banana', 'apple', 'grapes', 'melon', 'kiwi']", + "E ? ^ ^ +", + "E Full diff:", + "E - {'0': 0, '10': 10, '20': 20, '30': 30, '40': 40}", + "E ? - - - - - - - -", + "E + {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4}", + f"E AssertionError: assert 'hello world' in '{long_text}'", + ] + ) diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index d3cd61444..a4d48b6fe 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -2056,13 +2056,15 @@ class TestReprSizeVerbosity: ) def test_get_maxsize_for_saferepr(self, verbose: int, expected_size) -> None: class FakeConfig: - def getoption(self, name: str) -> int: - assert name == "verbose" + def get_verbosity(self, verbosity_type: Optional[str] = None) -> int: return verbose config = FakeConfig() assert _get_maxsize_for_saferepr(cast(Config, config)) == expected_size + def test_get_maxsize_for_saferepr_no_config(self) -> None: + assert _get_maxsize_for_saferepr(None) == DEFAULT_REPR_MAX_SIZE + def create_test_file(self, pytester: Pytester, size: int) -> None: pytester.makepyfile( f""" diff --git a/testing/test_config.py b/testing/test_config.py index b7c61feea..58671e6ed 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -23,6 +23,7 @@ from _pytest.config import ConftestImportFailure from _pytest.config import ExitCode from _pytest.config import parse_warning_filter from _pytest.config.argparsing import get_ini_default_for_type +from _pytest.config.argparsing import Parser from _pytest.config.exceptions import UsageError from _pytest.config.findpaths import determine_setup from _pytest.config.findpaths import get_common_ancestor @@ -2245,3 +2246,76 @@ class TestDebugOptions: "*Default: pytestdebug.log.", ] ) + + +class TestVerbosity: + SOME_OUTPUT_TYPE = Config.VERBOSITY_ASSERTIONS + SOME_OUTPUT_VERBOSITY_LEVEL = 5 + + class VerbosityIni: + def pytest_addoption(self, parser: Parser) -> None: + Config._add_verbosity_ini( + parser, TestVerbosity.SOME_OUTPUT_TYPE, help="some help text" + ) + + def test_level_matches_verbose_when_not_specified( + self, pytester: Pytester, tmp_path: Path + ) -> None: + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + """\ + [pytest] + addopts = --verbose + """ + ), + encoding="utf-8", + ) + pytester.plugins = [TestVerbosity.VerbosityIni()] + + config = pytester.parseconfig(tmp_path) + + assert ( + config.get_verbosity(TestVerbosity.SOME_OUTPUT_TYPE) + == config.option.verbose + ) + + def test_level_matches_verbose_when_not_known_type( + self, pytester: Pytester, tmp_path: Path + ) -> None: + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + """\ + [pytest] + addopts = --verbose + """ + ), + encoding="utf-8", + ) + pytester.plugins = [TestVerbosity.VerbosityIni()] + + config = pytester.parseconfig(tmp_path) + + assert config.get_verbosity("some fake verbosity type") == config.option.verbose + + def test_level_matches_specified_override( + self, pytester: Pytester, tmp_path: Path + ) -> None: + setting_name = f"verbosity_{TestVerbosity.SOME_OUTPUT_TYPE}" + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + f"""\ + [pytest] + addopts = --verbose + {setting_name} = {TestVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL} + """ + ), + encoding="utf-8", + ) + pytester.plugins = [TestVerbosity.VerbosityIni()] + + config = pytester.parseconfig(tmp_path) + + assert ( + config.get_verbosity(TestVerbosity.SOME_OUTPUT_TYPE) + == TestVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL + )