From 0f52856f995e42ac5f337bd82dd5179cea5214a1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 15 Jul 2015 20:03:58 -0300 Subject: [PATCH 1/2] Use a subdirectory in the TEMP directory to speed up tmpdir creation Fix #105 --- CHANGELOG | 7 +++++++ _pytest/pytester.py | 14 +++++++------ _pytest/tmpdir.py | 35 +++++++++++++++++++++++++++---- doc/en/tmpdir.txt | 44 ++++++++++++++++++++++++++++++++++++--- testing/python/fixture.py | 3 ++- testing/test_conftest.py | 4 ++-- testing/test_tmpdir.py | 35 ++++++++++++++++--------------- 7 files changed, 109 insertions(+), 33 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index e5221df04..b0707a5b0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,13 @@ - fix issue768: docstrings found in python modules were not setting up session fixtures. Thanks Jason R. Coombs for reporting and Bruno Oliveira for the PR. +- added `tmpdir_factory`, a session-scoped fixture that can be used to create + directories under the base temporary directory. Previously this object was + installed as a `_tmpdirhandler` attribute of the `config` object, but now it + is part of the official API and using `config._tmpdirhandler` is + deprecated. + Thanks Bruno Oliveira for the PR. + - fix issue 808: pytest's internal assertion rewrite hook now implements the optional PEP302 get_data API so tests can access data files next to them. Thanks xmo-odoo for request and example and Bruno Oliveira for diff --git a/_pytest/pytester.py b/_pytest/pytester.py index eedbbd39c..2048cf563 100644 --- a/_pytest/pytester.py +++ b/_pytest/pytester.py @@ -309,16 +309,18 @@ class HookRecorder: self.calls[:] = [] -def pytest_funcarg__linecomp(request): +@pytest.fixture +def linecomp(request): return LineComp() + def pytest_funcarg__LineMatcher(request): return LineMatcher -def pytest_funcarg__testdir(request): - tmptestdir = Testdir(request) - return tmptestdir +@pytest.fixture +def testdir(request, tmpdir_factory): + return Testdir(request, tmpdir_factory) rex_outcome = re.compile("(\d+) (\w+)") @@ -388,10 +390,10 @@ class Testdir: """ - def __init__(self, request): + def __init__(self, request, tmpdir_factory): self.request = request # XXX remove duplication with tmpdir plugin - basetmp = request.config._tmpdirhandler.ensuretemp("testdir") + basetmp = tmpdir_factory.ensuretemp("testdir") name = request.function.__name__ for i in range(100): try: diff --git a/_pytest/tmpdir.py b/_pytest/tmpdir.py index 53c396b76..6acac97b7 100644 --- a/_pytest/tmpdir.py +++ b/_pytest/tmpdir.py @@ -6,7 +6,12 @@ import py from _pytest.monkeypatch import monkeypatch -class TempdirHandler: +class TempdirFactory: + """Factory for temporary directories under the common base temp directory. + + The base directory can be configured using the ``--basetemp`` option. + """ + def __init__(self, config): self.config = config self.trace = config.trace.get("tmpdir") @@ -22,6 +27,10 @@ class TempdirHandler: return self.getbasetemp().ensure(string, dir=dir) def mktemp(self, basename, numbered=True): + """Create a subdirectory of the base temporary directory and return it. + If ``numbered``, ensure the directory is unique by adding a number + prefix greater than any existing one. + """ basetemp = self.getbasetemp() if not numbered: p = basetemp.mkdir(basename) @@ -51,15 +60,33 @@ class TempdirHandler: def finish(self): self.trace("finish") +# backward compatibility +TempdirHandler = TempdirFactory + + def pytest_configure(config): + """Create a TempdirFactory and attach it to the config object. + + This is to comply with existing plugins which expect the handler to be + available at pytest_configure time, but ideally should be moved entirely + to the tmpdir_factory session fixture. + """ mp = monkeypatch() - t = TempdirHandler(config) + t = TempdirFactory(config) config._cleanup.extend([mp.undo, t.finish]) mp.setattr(config, '_tmpdirhandler', t, raising=False) mp.setattr(pytest, 'ensuretemp', t.ensuretemp, raising=False) + +@pytest.fixture(scope='session') +def tmpdir_factory(request): + """Return a TempdirFactory instance for the test session. + """ + return request.config._tmpdirhandler + + @pytest.fixture -def tmpdir(request): +def tmpdir(request, tmpdir_factory): """return a temporary directory path object which is unique to each test function invocation, created as a sub directory of the base temporary @@ -71,5 +98,5 @@ def tmpdir(request): MAXVAL = 30 if len(name) > MAXVAL: name = name[:MAXVAL] - x = request.config._tmpdirhandler.mktemp(name, numbered=True) + x = tmpdir_factory.mktemp(name, numbered=True) return x diff --git a/doc/en/tmpdir.txt b/doc/en/tmpdir.txt index 8c535beba..c396346a1 100644 --- a/doc/en/tmpdir.txt +++ b/doc/en/tmpdir.txt @@ -5,10 +5,10 @@ Temporary directories and files ================================================ -The 'tmpdir' test function argument ------------------------------------ +The 'tmpdir' fixture +-------------------- -You can use the ``tmpdir`` function argument which will +You can use the ``tmpdir`` fixture which will provide a temporary directory unique to the test invocation, created in the `base temporary directory`_. @@ -51,6 +51,44 @@ Running this would result in a passed test except for the last test_tmpdir.py:7: AssertionError ======= 1 failed in 0.12 seconds ======== + +The 'tmpdir_factory' fixture +---------------------------- + +.. versionadded:: 2.8 + +The ``tmpdir_factory`` is a session-scoped fixture which can be used +to create arbitrary temporary directories from any other fixture or test. + +For example, suppose your test suite needs a large image on disk, which is +generated procedurally. Instead of computing the same image for each test +that uses it into its own ``tmpdir``, you can generate it once per-session +to save time: + +.. code-block:: python + + # contents of conftest.py + import pytest + + @pytest.fixture(scope='session') + def image_file(tmpdir_factory): + img = compute_expensive_image() + fn = tmpdir_factory.mktemp('data').join('img.png') + img.save(str(fn)) + return fn + + # contents of test_image.py + def test_histogram(image_file): + img = load_image(image_file) + # compute and test histogram + +``tmpdir_factory`` instances have the following methods: + +.. currentmodule:: _pytest.tmpdir + +.. automethod:: TempdirFactory.mktemp +.. automethod:: TempdirFactory.getbasetemp + .. _`base temporary directory`: The default base temporary directory diff --git a/testing/python/fixture.py b/testing/python/fixture.py index 679caa927..2678a07e7 100644 --- a/testing/python/fixture.py +++ b/testing/python/fixture.py @@ -555,7 +555,8 @@ class TestRequestBasic: pass def test_function(request, farg): assert set(get_public_names(request.fixturenames)) == \ - set(["tmpdir", "sarg", "arg1", "request", "farg"]) + set(["tmpdir", "sarg", "arg1", "request", "farg", + "tmpdir_factory"]) """) reprec = testdir.inline_run() reprec.assertoutcome(passed=1) diff --git a/testing/test_conftest.py b/testing/test_conftest.py index 4791d6f61..513ba43a5 100644 --- a/testing/test_conftest.py +++ b/testing/test_conftest.py @@ -4,9 +4,9 @@ from _pytest.config import PytestPluginManager @pytest.fixture(scope="module", params=["global", "inpackage"]) -def basedir(request): +def basedir(request, tmpdir_factory): from _pytest.tmpdir import tmpdir - tmpdir = tmpdir(request) + tmpdir = tmpdir(request, tmpdir_factory) tmpdir.ensure("adir/conftest.py").write("a=1 ; Directory = 3") tmpdir.ensure("adir/b/conftest.py").write("b=2 ; a = 1.5") if request.param == "inpackage": diff --git a/testing/test_tmpdir.py b/testing/test_tmpdir.py index e6c87dd5b..edc010906 100644 --- a/testing/test_tmpdir.py +++ b/testing/test_tmpdir.py @@ -1,7 +1,7 @@ import py import pytest -from _pytest.tmpdir import tmpdir, TempdirHandler +from _pytest.tmpdir import tmpdir, TempdirFactory def test_funcarg(testdir): testdir.makepyfile(""" @@ -13,16 +13,15 @@ def test_funcarg(testdir): reprec = testdir.inline_run() calls = reprec.getcalls("pytest_runtest_setup") item = calls[0].item - # pytest_unconfigure has deleted the TempdirHandler already config = item.config - config._tmpdirhandler = TempdirHandler(config) + tmpdirhandler = TempdirFactory(config) item._initrequest() - p = tmpdir(item._request) + p = tmpdir(item._request, tmpdirhandler) assert p.check() bn = p.basename.strip("0123456789") assert bn.endswith("test_func_a_") item.name = "qwe/\\abc" - p = tmpdir(item._request) + p = tmpdir(item._request, tmpdirhandler) assert p.check() bn = p.basename.strip("0123456789") assert bn == "qwe__abc" @@ -38,7 +37,7 @@ class TestTempdirHandler: def test_mktemp(self, testdir): config = testdir.parseconfig() config.option.basetemp = testdir.mkdir("hello") - t = TempdirHandler(config) + t = TempdirFactory(config) tmp = t.mktemp("world") assert tmp.relto(t.getbasetemp()) == "world0" tmp = t.mktemp("this") @@ -49,17 +48,19 @@ class TestTempdirHandler: class TestConfigTmpdir: def test_getbasetemp_custom_removes_old(self, testdir): - p = testdir.tmpdir.join("xyz") - config = testdir.parseconfigure("--basetemp=xyz") - b = config._tmpdirhandler.getbasetemp() - assert b == p - h = b.ensure("hello") - config._tmpdirhandler.getbasetemp() - assert h.check() - config = testdir.parseconfigure("--basetemp=xyz") - b2 = config._tmpdirhandler.getbasetemp() - assert b2.check() - assert not h.check() + mytemp = testdir.tmpdir.join("xyz") + p = testdir.makepyfile(""" + def test_1(tmpdir): + pass + """) + testdir.runpytest(p, '--basetemp=%s' % mytemp) + mytemp.check() + mytemp.ensure("hello") + + testdir.runpytest(p, '--basetemp=%s' % mytemp) + mytemp.check() + assert not mytemp.join("hello").check() + def test_basetemp(testdir): mytemp = testdir.tmpdir.mkdir("mytemp") From 74bc50e97cc8b1d85bec8f11ce84df698270f07a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 15 Jul 2015 20:45:35 -0300 Subject: [PATCH 2/2] Fix flakes --- testing/test_tmpdir.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/test_tmpdir.py b/testing/test_tmpdir.py index edc010906..7c2095e6e 100644 --- a/testing/test_tmpdir.py +++ b/testing/test_tmpdir.py @@ -1,7 +1,7 @@ import py import pytest -from _pytest.tmpdir import tmpdir, TempdirFactory +from _pytest.tmpdir import tmpdir def test_funcarg(testdir): testdir.makepyfile(""" @@ -10,6 +10,7 @@ def test_funcarg(testdir): metafunc.addcall(id='b') def test_func(tmpdir): pass """) + from _pytest.tmpdir import TempdirFactory reprec = testdir.inline_run() calls = reprec.getcalls("pytest_runtest_setup") item = calls[0].item @@ -35,6 +36,7 @@ def test_ensuretemp(recwarn): class TestTempdirHandler: def test_mktemp(self, testdir): + from _pytest.tmpdir import TempdirFactory config = testdir.parseconfig() config.option.basetemp = testdir.mkdir("hello") t = TempdirFactory(config)