Allow tests to override "global" `log_level` (rebased) (#7340)

Co-authored-by: Ruaridh Williamson <ruaridh.williamson@flexciton.com>

AUTHORS

@@ -245,6 +245,7 @@ Romain Dorgueil
 Roman Bolshakov
 Ronny Pfannschmidt
 Ross Lawley
+Ruaridh Williamson
 Russel Winder
 Ryan Wooden
 Samuel Dion-Girardeau

changelog/7133.improvement.rst

@@ -0,0 +1 @@
+``caplog.set_level()`` will now override any :confval:`log_level` set via the CLI or ``.ini``.

doc/en/logging.rst

@@ -250,6 +250,9 @@ made in ``3.4`` after community feedback:
 
 * Log levels are no longer changed unless explicitly requested by the :confval:`log_level` configuration
   or ``--log-level`` command-line options. This allows users to configure logger objects themselves.
+  Setting :confval:`log_level` will set the level that is captured globally so if a specific test requires
+  a lower level than this, use the ``caplog.set_level()`` functionality otherwise that test will be prone to
+  failure.
 * :ref:`Live Logs <live_logs>` is now disabled by default and can be enabled setting the
   :confval:`log_cli` configuration option to ``true``. When enabled, the verbosity is increased so logging for each
   test is visible.

src/_pytest/logging.py

@@ -343,7 +343,7 @@ class LogCaptureFixture:
         """Creates a new funcarg."""
         self._item = item
         # dict of log name -> log level
-        self._initial_log_levels = {}  # type: Dict[Optional[str], int]
+        self._initial_logger_levels = {}  # type: Dict[Optional[str], int]
 
     def _finalize(self) -> None:
         """Finalizes the fixture.
@@ -351,7 +351,7 @@ class LogCaptureFixture:
         This restores the log levels changed by :meth:`set_level`.
         """
         # restore log levels
-        for logger_name, level in self._initial_log_levels.items():
+        for logger_name, level in self._initial_logger_levels.items():
             logger = logging.getLogger(logger_name)
             logger.setLevel(level)
 
@@ -430,8 +430,9 @@ class LogCaptureFixture:
         """
         logger_obj = logging.getLogger(logger)
         # save the original log-level to restore it during teardown
-        self._initial_log_levels.setdefault(logger, logger_obj.level)
+        self._initial_logger_levels.setdefault(logger, logger_obj.level)
         logger_obj.setLevel(level)
+        self.handler.setLevel(level)
 
     @contextmanager
     def at_level(
@@ -446,10 +447,13 @@ class LogCaptureFixture:
         logger_obj = logging.getLogger(logger)
         orig_level = logger_obj.level
         logger_obj.setLevel(level)
+        handler_orig_level = self.handler.level
+        self.handler.setLevel(level)
         try:
             yield
         finally:
             logger_obj.setLevel(orig_level)
+            self.handler.setLevel(handler_orig_level)
 
 
 @pytest.fixture

testing/logging/test_fixture.py

@@ -138,3 +138,100 @@ def test_caplog_captures_for_all_stages(caplog, logging_during_setup_and_teardow
 
     # This reaches into private API, don't use this type of thing in real tests!
     assert set(caplog._item._store[catch_log_records_key]) == {"setup", "call"}
+
+
+def test_ini_controls_global_log_level(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+        import logging
+        def test_log_level_override(request, caplog):
+            plugin = request.config.pluginmanager.getplugin('logging-plugin')
+            assert plugin.log_level == logging.ERROR
+            logger = logging.getLogger('catchlog')
+            logger.warning("WARNING message won't be shown")
+            logger.error("ERROR message will be shown")
+            assert 'WARNING' not in caplog.text
+            assert 'ERROR' in caplog.text
+    """
+    )
+    testdir.makeini(
+        """
+        [pytest]
+        log_level=ERROR
+    """
+    )
+
+    result = testdir.runpytest()
+    # make sure that that we get a '0' exit code for the testsuite
+    assert result.ret == 0
+
+
+def test_caplog_can_override_global_log_level(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+        import logging
+        def test_log_level_override(request, caplog):
+            logger = logging.getLogger('catchlog')
+            plugin = request.config.pluginmanager.getplugin('logging-plugin')
+            assert plugin.log_level == logging.WARNING
+
+            logger.info("INFO message won't be shown")
+
+            caplog.set_level(logging.INFO, logger.name)
+
+            with caplog.at_level(logging.DEBUG, logger.name):
+                logger.debug("DEBUG message will be shown")
+
+            logger.debug("DEBUG message won't be shown")
+
+            with caplog.at_level(logging.CRITICAL, logger.name):
+                logger.warning("WARNING message won't be shown")
+
+            logger.debug("DEBUG message won't be shown")
+            logger.info("INFO message will be shown")
+
+            assert "message won't be shown" not in caplog.text
+    """
+    )
+    testdir.makeini(
+        """
+        [pytest]
+        log_level=WARNING
+    """
+    )
+
+    result = testdir.runpytest()
+    assert result.ret == 0
+
+
+def test_caplog_captures_despite_exception(testdir):
+    testdir.makepyfile(
+        """
+        import pytest
+        import logging
+        def test_log_level_override(request, caplog):
+            logger = logging.getLogger('catchlog')
+            plugin = request.config.pluginmanager.getplugin('logging-plugin')
+            assert plugin.log_level == logging.WARNING
+
+            logger.info("INFO message won't be shown")
+
+            caplog.set_level(logging.INFO, logger.name)
+
+            with caplog.at_level(logging.DEBUG, logger.name):
+                logger.debug("DEBUG message will be shown")
+                raise Exception()
+    """
+    )
+    testdir.makeini(
+        """
+        [pytest]
+        log_level=WARNING
+    """
+    )
+
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines(["*DEBUG message will be shown*"])
+    assert result.ret == 1