diff --git a/src/_pytest/pytester.py b/src/_pytest/pytester.py
index 8d5a6b141..24ddf4e9e 100644
--- a/src/_pytest/pytester.py
+++ b/src/_pytest/pytester.py
@@ -502,6 +502,7 @@ class Testdir(object):
         self.tmpdir = tmpdir_factory.mktemp(name, numbered=True)
         self.test_tmproot = tmpdir_factory.mktemp("tmp-" + name, numbered=True)
         os.environ["PYTEST_DEBUG_TEMPROOT"] = str(self.test_tmproot)
+        os.environ.pop("TOX_ENV_DIR", None)  # Ensure that it is not used for caching.
         self.plugins = []
         self._cwd_snapshot = CwdSnapshot()
         self._sys_path_snapshot = SysPathsSnapshot()
diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py
index 55090a2f6..2b8ca2e18 100644
--- a/testing/test_cacheprovider.py
+++ b/testing/test_cacheprovider.py
@@ -14,17 +14,6 @@ import pytest
 pytest_plugins = ("pytester",)
 
 
-@pytest.fixture(scope="module", autouse=True)
-def handle_env():
-    """Ensure env is like most of the tests expect it, i.e. not using tox."""
-    orig_env = os.environ.pop("TOX_ENV_DIR", None)
-
-    yield
-
-    if orig_env is not None:
-        os.environ["TOX_ENV_DIR"] = orig_env
-
-
 class TestNewAPI(object):
     def test_config_cache_makedir(self, testdir):
         testdir.makeini("[pytest]")