pytester: pop TOX_ENV_DIR from os.environ

Closes: https://github.com/pytest-dev/pytest/pull/4378
Fixes: https://github.com/pytest-dev/pytest/issues/4366

src/_pytest/pytester.py

@@ -502,6 +502,7 @@ class Testdir(object):
         self.tmpdir = tmpdir_factory.mktemp(name, numbered=True)
         self.test_tmproot = tmpdir_factory.mktemp("tmp-" + name, numbered=True)
         os.environ["PYTEST_DEBUG_TEMPROOT"] = str(self.test_tmproot)
+        os.environ.pop("TOX_ENV_DIR", None)  # Ensure that it is not used for caching.
         self.plugins = []
         self._cwd_snapshot = CwdSnapshot()
         self._sys_path_snapshot = SysPathsSnapshot()

testing/test_cacheprovider.py

@@ -14,17 +14,6 @@ import pytest
 pytest_plugins = ("pytester",)
 
 
-@pytest.fixture(scope="module", autouse=True)
-def handle_env():
-    """Ensure env is like most of the tests expect it, i.e. not using tox."""
-    orig_env = os.environ.pop("TOX_ENV_DIR", None)
-
-    yield
-
-    if orig_env is not None:
-        os.environ["TOX_ENV_DIR"] = orig_env
-
-
 class TestNewAPI(object):
     def test_config_cache_makedir(self, testdir):
         testdir.makeini("[pytest]")