pdb: fix capturing with recursive debugging and pdb++

While I think that pdb++ should be fixed in this regard (by using
`pdb.Pdb`, and not `self.__class__` maybe), this ensures that custom
debuggers like this are working.

changelog/4347.bugfix.rst

@@ -0,0 +1 @@
+Fix output capturing when using pdb++ with recursive debugging.

src/_pytest/debugging.py

@@ -75,6 +75,7 @@ class pytestPDB(object):
     _config = None
     _pdb_cls = pdb.Pdb
     _saved = []
+    _recursive_debug = 0
 
     @classmethod
     def _init_pdb(cls, *args, **kwargs):
@@ -87,6 +88,7 @@ class pytestPDB(object):
                 capman.suspend_global_capture(in_=True)
             tw = _pytest.config.create_terminal_writer(cls._config)
             tw.line()
+            if cls._recursive_debug == 0:
                 # Handle header similar to pdb.set_trace in py37+.
                 header = kwargs.pop("header", None)
                 if header is not None:
@@ -100,11 +102,18 @@ class pytestPDB(object):
                 _pytest_capman = capman
                 _continued = False
 
+                def do_debug(self, arg):
+                    cls._recursive_debug += 1
+                    ret = super(_PdbWrapper, self).do_debug(arg)
+                    cls._recursive_debug -= 1
+                    return ret
+
                 def do_continue(self, arg):
                     ret = super(_PdbWrapper, self).do_continue(arg)
                     if self._pytest_capman:
                         tw = _pytest.config.create_terminal_writer(cls._config)
                         tw.line()
+                        if cls._recursive_debug == 0:
                             if self._pytest_capman.is_globally_capturing():
                                 tw.sep(">", "PDB continue (IO-capturing resumed)")
                             else:

testing/test_pdb.py

@@ -513,6 +513,76 @@ class TestPDB(object):
         assert "1 failed" in rest
         self.flush(child)
 
+    def test_pdb_interaction_continue_recursive(self, testdir):
+        p1 = testdir.makepyfile(
+            mytest="""
+            import pdb
+            import pytest
+
+            count_continue = 0
+
+            # Simulates pdbpp, which injects Pdb into do_debug, and uses
+            # self.__class__ in do_continue.
+            class CustomPdb(pdb.Pdb, object):
+                def do_debug(self, arg):
+                    import sys
+                    import types
+
+                    newglobals = {
+                        'Pdb': self.__class__,  # NOTE: different with pdb.Pdb
+                        'sys': sys,
+                    }
+                    if sys.version_info < (3, ):
+                        do_debug_func = pdb.Pdb.do_debug.im_func
+                    else:
+                        do_debug_func = pdb.Pdb.do_debug
+
+                    orig_do_debug = types.FunctionType(
+                        do_debug_func.__code__, newglobals,
+                        do_debug_func.__name__, do_debug_func.__defaults__,
+                    )
+                    return orig_do_debug(self, arg)
+                do_debug.__doc__ = pdb.Pdb.do_debug.__doc__
+
+                def do_continue(self, *args, **kwargs):
+                    global count_continue
+                    count_continue += 1
+                    return super(CustomPdb, self).do_continue(*args, **kwargs)
+
+            def foo():
+                print("print_from_foo")
+
+            def test_1():
+                i = 0
+                print("hello17")
+                pytest.set_trace()
+                x = 3
+                print("hello18")
+
+                assert count_continue == 2, "unexpected_failure: %d != 2" % count_continue
+                pytest.fail("expected_failure")
+        """
+        )
+        child = testdir.spawn_pytest("--pdbcls=mytest:CustomPdb %s" % str(p1))
+        child.expect(r"PDB set_trace \(IO-capturing turned off\)")
+        child.expect(r"\n\(Pdb")
+        child.sendline("debug foo()")
+        child.expect("ENTERING RECURSIVE DEBUGGER")
+        child.expect(r"\n\(\(Pdb")
+        child.sendline("c")
+        child.expect("LEAVING RECURSIVE DEBUGGER")
+        assert b"PDB continue" not in child.before
+        assert b"print_from_foo" in child.before
+        child.sendline("c")
+        child.expect(r"PDB continue \(IO-capturing resumed\)")
+        rest = child.read().decode("utf8")
+        assert "hello17" in rest  # out is captured
+        assert "hello18" in rest  # out is captured
+        assert "1 failed" in rest
+        assert "Failed: expected_failure" in rest
+        assert "AssertionError: unexpected_failure" not in rest
+        self.flush(child)
+
     def test_pdb_without_capture(self, testdir):
         p1 = testdir.makepyfile(
             """