testing: fix flaky tests on pypy3 due to resource warnings in stderr (#7405)

testing/test_stepwise.py

@@ -75,6 +75,16 @@ def broken_testdir(testdir):
     return testdir
 
 
+def _strip_resource_warnings(lines):
+    # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work.
+    # (https://github.com/pytest-dev/pytest/issues/5088)
+    return [
+        x
+        for x in lines
+        if not x.startswith(("Exception ignored in:", "ResourceWarning"))
+    ]
+
+
 def test_run_without_stepwise(stepwise_testdir):
     result = stepwise_testdir.runpytest("-v", "--strict-markers", "--fail")
 
@@ -88,7 +98,7 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir):
     result = stepwise_testdir.runpytest(
         "-v", "--strict-markers", "--stepwise", "--fail"
     )
-    assert not result.stderr.str()
+    assert _strip_resource_warnings(result.stderr.lines) == []
 
     stdout = result.stdout.str()
     # Make sure we stop after first failing test.
@@ -98,7 +108,7 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir):
 
     # "Fix" the test that failed in the last run and run it again.
     result = stepwise_testdir.runpytest("-v", "--strict-markers", "--stepwise")
-    assert not result.stderr.str()
+    assert _strip_resource_warnings(result.stderr.lines) == []
 
     stdout = result.stdout.str()
     # Make sure the latest failing test runs and then continues.
@@ -116,7 +126,7 @@ def test_run_with_skip_option(stepwise_testdir):
         "--fail",
         "--fail-last",
     )
-    assert not result.stderr.str()
+    assert _strip_resource_warnings(result.stderr.lines) == []
 
     stdout = result.stdout.str()
     # Make sure first fail is ignore and second fail stops the test run.
@@ -129,7 +139,7 @@ def test_run_with_skip_option(stepwise_testdir):
 def test_fail_on_errors(error_testdir):
     result = error_testdir.runpytest("-v", "--strict-markers", "--stepwise")
 
-    assert not result.stderr.str()
+    assert _strip_resource_warnings(result.stderr.lines) == []
     stdout = result.stdout.str()
 
     assert "test_error ERROR" in stdout
@@ -140,7 +150,7 @@ def test_change_testfile(stepwise_testdir):
     result = stepwise_testdir.runpytest(
         "-v", "--strict-markers", "--stepwise", "--fail", "test_a.py"
     )
-    assert not result.stderr.str()
+    assert _strip_resource_warnings(result.stderr.lines) == []
 
     stdout = result.stdout.str()
     assert "test_fail_on_flag FAILED" in stdout
@@ -150,7 +160,7 @@ def test_change_testfile(stepwise_testdir):
     result = stepwise_testdir.runpytest(
         "-v", "--strict-markers", "--stepwise", "test_b.py"
     )
-    assert not result.stderr.str()
+    assert _strip_resource_warnings(result.stderr.lines) == []
 
     stdout = result.stdout.str()
     assert "test_success PASSED" in stdout