From 1ae4182e1836000eb35a40ec4c3dbb2689e0c5ae Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Fri, 26 Jun 2020 15:50:19 +0300 Subject: [PATCH] testing: fix flaky tests on pypy3 due to resource warnings in stderr (#7405) --- testing/test_stepwise.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index 3bc77857d..df66d798b 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -75,6 +75,16 @@ def broken_testdir(testdir): return testdir +def _strip_resource_warnings(lines): + # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work. + # (https://github.com/pytest-dev/pytest/issues/5088) + return [ + x + for x in lines + if not x.startswith(("Exception ignored in:", "ResourceWarning")) + ] + + def test_run_without_stepwise(stepwise_testdir): result = stepwise_testdir.runpytest("-v", "--strict-markers", "--fail") @@ -88,7 +98,7 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir): result = stepwise_testdir.runpytest( "-v", "--strict-markers", "--stepwise", "--fail" ) - assert not result.stderr.str() + assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() # Make sure we stop after first failing test. @@ -98,7 +108,7 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir): # "Fix" the test that failed in the last run and run it again. result = stepwise_testdir.runpytest("-v", "--strict-markers", "--stepwise") - assert not result.stderr.str() + assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() # Make sure the latest failing test runs and then continues. @@ -116,7 +126,7 @@ def test_run_with_skip_option(stepwise_testdir): "--fail", "--fail-last", ) - assert not result.stderr.str() + assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() # Make sure first fail is ignore and second fail stops the test run. @@ -129,7 +139,7 @@ def test_run_with_skip_option(stepwise_testdir): def test_fail_on_errors(error_testdir): result = error_testdir.runpytest("-v", "--strict-markers", "--stepwise") - assert not result.stderr.str() + assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() assert "test_error ERROR" in stdout @@ -140,7 +150,7 @@ def test_change_testfile(stepwise_testdir): result = stepwise_testdir.runpytest( "-v", "--strict-markers", "--stepwise", "--fail", "test_a.py" ) - assert not result.stderr.str() + assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() assert "test_fail_on_flag FAILED" in stdout @@ -150,7 +160,7 @@ def test_change_testfile(stepwise_testdir): result = stepwise_testdir.runpytest( "-v", "--strict-markers", "--stepwise", "test_b.py" ) - assert not result.stderr.str() + assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() assert "test_success PASSED" in stdout