diff --git a/changelog/7672.bugfix.rst b/changelog/7672.bugfix.rst
new file mode 100644
index 000000000..88608e161
--- /dev/null
+++ b/changelog/7672.bugfix.rst
@@ -0,0 +1 @@
+Fixed log-capturing level restored incorrectly if ``caplog.set_level`` is called more than once.
diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py
index 5dfd47887..95226e8cc 100644
--- a/src/_pytest/logging.py
+++ b/src/_pytest/logging.py
@@ -439,7 +439,8 @@ class LogCaptureFixture:
         # Save the original log-level to restore it during teardown.
         self._initial_logger_levels.setdefault(logger, logger_obj.level)
         logger_obj.setLevel(level)
-        self._initial_handler_level = self.handler.level
+        if self._initial_handler_level is None:
+            self._initial_handler_level = self.handler.level
         self.handler.setLevel(level)
 
     @contextmanager
diff --git a/testing/logging/test_fixture.py b/testing/logging/test_fixture.py
index cbd28f798..ffd51bcad 100644
--- a/testing/logging/test_fixture.py
+++ b/testing/logging/test_fixture.py
@@ -65,6 +65,7 @@ def test_change_level_undos_handler_level(testdir: Testdir) -> None:
 
         def test1(caplog):
             assert caplog.handler.level == 0
+            caplog.set_level(9999)
             caplog.set_level(41)
             assert caplog.handler.level == 41