diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py
index dfc3c5320..3f03b5ff9 100644
--- a/testing/test_cacheprovider.py
+++ b/testing/test_cacheprovider.py
@@ -1,5 +1,6 @@
 import os
 import shutil
+import stat
 import sys
 import textwrap
 
@@ -45,26 +46,40 @@ class TestNewAPI:
     )
     def test_cache_writefail_permissions(self, testdir):
         testdir.makeini("[pytest]")
+        cache_dir = str(testdir.tmpdir.ensure_dir(".pytest_cache"))
+        mode = os.stat(cache_dir)[stat.ST_MODE]
         testdir.tmpdir.ensure_dir(".pytest_cache").chmod(0)
-        config = testdir.parseconfigure()
-        cache = config.cache
-        cache.set("test/broken", [])
+        try:
+            config = testdir.parseconfigure()
+            cache = config.cache
+            cache.set("test/broken", [])
+        finally:
+            testdir.tmpdir.ensure_dir(".pytest_cache").chmod(mode)
 
     @pytest.mark.skipif(sys.platform.startswith("win"), reason="no chmod on windows")
-    @pytest.mark.filterwarnings("default")
+    @pytest.mark.filterwarnings(
+        "ignore:could not create cache path:pytest.PytestWarning"
+    )
     def test_cache_failure_warns(self, testdir):
+        cache_dir = str(testdir.tmpdir.ensure_dir(".pytest_cache"))
+        mode = os.stat(cache_dir)[stat.ST_MODE]
         testdir.tmpdir.ensure_dir(".pytest_cache").chmod(0)
-        testdir.makepyfile(
-            """
-            def test_error():
-                raise Exception
+        try:
+            testdir.makepyfile(
+                """
+                def test_error():
+                    raise Exception
 
-        """
-        )
-        result = testdir.runpytest("-rw")
-        assert result.ret == 1
-        # warnings from nodeids, lastfailed, and stepwise
-        result.stdout.fnmatch_lines(["*could not create cache path*", "*3 warnings*"])
+            """
+            )
+            result = testdir.runpytest("-rw")
+            assert result.ret == 1
+            # warnings from nodeids, lastfailed, and stepwise
+            result.stdout.fnmatch_lines(
+                ["*could not create cache path*", "*3 warnings*"]
+            )
+        finally:
+            testdir.tmpdir.ensure_dir(".pytest_cache").chmod(mode)
 
     def test_config_cache(self, testdir):
         testdir.makeconftest(
diff --git a/tox.ini b/tox.ini
index edc9a5667..93889322f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -142,8 +142,6 @@ filterwarnings =
     error
     default:Using or importing the ABCs:DeprecationWarning:unittest2.*
     ignore:Module already imported so cannot be rewritten:pytest.PytestWarning
-    # https://github.com/pytest-dev/pytest/issues/5974
-    default:\(rm_rf\) error removing.*:pytest.PytestWarning
     # produced by python3.6/site.py itself (3.6.7 on Travis, could not trigger it with 3.6.8).
     ignore:.*U.*mode is deprecated:DeprecationWarning:(?!(pytest|_pytest))
     # produced by pytest-xdist